From 4a611588ddb2b8a055f865329f14c3b5de126cb4 Mon Sep 17 00:00:00 2001 From: jatinchowdhury18 Date: Tue, 12 Nov 2024 18:41:00 -0800 Subject: [PATCH] Dense layers with optionally no bias (#155) * Add has_bias template parameter for DenseT * Implementing other backends and tests * More tests * Apply clang-format * Fixing Windows test --------- Co-authored-by: github-actions[bot] --- RTNeural/ModelT.h | 4 +- RTNeural/common.h | 4 + RTNeural/config.h | 13 + RTNeural/dense/dense.h | 20 +- RTNeural/dense/dense_eigen.h | 36 +- RTNeural/dense/dense_xsimd.h | 119 ++++- RTNeural/lstm/lstm_xsimd.tpp | 2 +- RTNeural/model_loader.h | 14 +- RTNeural/torch_helpers.h | 19 +- models/dense.json | 716 ++++++++++++++++----------- python/dense.png | Bin 18026 -> 16361 bytes python/dense.py | 7 +- test_data/dense_x_python.csv | 22 +- test_data/dense_y_python.csv | 200 ++++---- tests/functional/templated_tests.cpp | 10 +- 15 files changed, 744 insertions(+), 442 deletions(-) diff --git a/RTNeural/ModelT.h b/RTNeural/ModelT.h index b7cd6e9e..226218b2 100644 --- a/RTNeural/ModelT.h +++ b/RTNeural/ModelT.h @@ -77,8 +77,8 @@ namespace modelt_detail json_parser::debug_print("Loading a no-op layer!", debug); } - template - void loadLayer(DenseT& dense, int& json_stream_idx, const nlohmann::json& l, + template + void loadLayer(DenseT& dense, int& json_stream_idx, const nlohmann::json& l, const std::string& type, int layerDims, bool debug) { using namespace json_parser; diff --git a/RTNeural/common.h b/RTNeural/common.h index 29750e98..68594536 100644 --- a/RTNeural/common.h +++ b/RTNeural/common.h @@ -29,6 +29,10 @@ constexpr T ceil_div(T num, T den) { return (num + den - 1) / den; } + +struct Empty +{ +}; } // namespace RTNEURAL_NAMESPACE #if RTNEURAL_USE_EIGEN diff --git a/RTNeural/config.h b/RTNeural/config.h index ac85f37f..7743a216 100644 --- a/RTNeural/config.h +++ b/RTNeural/config.h @@ -13,6 +13,19 @@ #define RTNEURAL_DEFAULT_ALIGNMENT 16 #endif +#if defined(_MSVC_LANG) +#define RTNEURAL_CPLUSPLUS _MSVC_LANG +#elif defined(__cplusplus) +#define RTNEURAL_CPLUSPLUS __cplusplus +#endif + +#if defined(RTNEURAL_CPLUSPLUS) && RTNEURAL_CPLUSPLUS >= 201703L +#define RTNEURAL_HAS_CPP17 1 +#define RTNEURAL_IF_CONSTEXPR if constexpr +#else +#define RTNEURAL_IF_CONSTEXPR if +#endif + /** Facilitate testing real-time safety with RealtimeSanitizer (RADSan) diff --git a/RTNeural/dense/dense.h b/RTNeural/dense/dense.h index 9ab5973e..cc17ace1 100644 --- a/RTNeural/dense/dense.h +++ b/RTNeural/dense/dense.h @@ -49,7 +49,7 @@ class Dense1 private: const int in_size; - T bias; + T bias {}; T* weights; }; @@ -63,6 +63,8 @@ template class Dense final : public Layer { public: + static constexpr bool dense_has_bias = true; + /** Constructs a dense layer for a given input and output size. */ Dense(int in_size, int out_size) : Layer(in_size, out_size) @@ -157,7 +159,7 @@ class Dense final : public Layer * Static implementation of a fully-connected (dense) layer, * with no activation. */ -template +template class DenseT { static constexpr auto weights_size = in_sizet * out_sizet; @@ -165,6 +167,7 @@ class DenseT public: static constexpr auto in_size = in_sizet; static constexpr auto out_size = out_sizet; + static constexpr bool dense_has_bias = has_bias; DenseT() { @@ -188,10 +191,19 @@ class DenseT RTNEURAL_REALTIME void reset() { } /** Performs forward propagation for this layer. */ - RTNEURAL_REALTIME inline void forward(const T (&ins)[in_size]) noexcept + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < out_size; ++i) + outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], bias[i]); + } + + /** Performs forward propagation for this layer (no bias). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const T (&ins)[in_size]) noexcept { for(int i = 0; i < out_size; ++i) - outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], (T)0) + bias[i]; + outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], (T)0); } /** diff --git a/RTNeural/dense/dense_eigen.h b/RTNeural/dense/dense_eigen.h index 18b540e7..a96c8de7 100644 --- a/RTNeural/dense/dense_eigen.h +++ b/RTNeural/dense/dense_eigen.h @@ -2,6 +2,7 @@ #define DENSEEIGEN_H_INCLUDED #include "../Layer.h" +#include "../common.h" #include "../config.h" #include @@ -16,6 +17,8 @@ template class Dense : public Layer { public: + static constexpr bool dense_has_bias = true; + /** Constructs a dense layer for a given input and output size. */ Dense(int in_size, int out_size) : Layer(in_size, out_size) @@ -118,23 +121,31 @@ class Dense : public Layer * Static implementation of a fully-connected (dense) layer, * with no activation. */ -template +template class DenseT { using out_vec_type = Eigen::Matrix; +#if RTNEURAL_HAS_CPP17 + using in_vec_type = typename std::conditional, Empty>::type; +#else using in_vec_type = Eigen::Matrix; - using mat_type = Eigen::Matrix; +#endif + using mat_type = Eigen::Matrix; public: static constexpr auto in_size = in_sizet; static constexpr auto out_size = out_sizet; + static constexpr bool dense_has_bias = has_bias; DenseT() : outs(outs_internal) { weights = mat_type::Zero(); - ins_internal = in_vec_type::Zero(); - ins_internal(in_size, 0) = (T)1; + RTNEURAL_IF_CONSTEXPR(has_bias) + { + ins_internal = in_vec_type::Zero(); + ins_internal(in_size, 0) = (T)1; + } outs = out_vec_type::Zero(); } @@ -148,7 +159,8 @@ class DenseT RTNEURAL_REALTIME void reset() { } /** Performs forward propagation for this layer. */ - RTNEURAL_REALTIME inline void forward(const Eigen::Matrix& ins) noexcept + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const Eigen::Matrix& ins) noexcept { for(int i = 0; i < in_size; ++i) ins_internal(i, 0) = ins(i, 0); @@ -160,6 +172,13 @@ class DenseT outs.noalias() = weights * ins_internal; } + /** Performs forward propagation for this layer (no bias). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const Eigen::Matrix& ins) noexcept + { + outs.noalias() = weights * ins; + } + /** * Sets the layer weights from a given vector. * @@ -190,7 +209,12 @@ class DenseT * Sets the layer bias from a given array of size * bias[out_size] */ - RTNEURAL_REALTIME void setBias(const T* b) +#if RTNEURAL_HAS_CPP17 + template + RTNEURAL_REALTIME inline typename std::enable_if::type setBias(const T* b) +#else + RTNEURAL_REALTIME inline void setBias(const T* b) +#endif { for(int i = 0; i < out_size; ++i) weights(i, in_size) = b[i]; diff --git a/RTNeural/dense/dense_xsimd.h b/RTNeural/dense/dense_xsimd.h index 73c904b4..45cfc704 100644 --- a/RTNeural/dense/dense_xsimd.h +++ b/RTNeural/dense/dense_xsimd.h @@ -16,6 +16,8 @@ template class Dense : public Layer { public: + static constexpr bool dense_has_bias = true; + /** Constructs a dense layer for a given input and output size. */ Dense(int in_size, int out_size) : Layer(in_size, out_size) @@ -118,7 +120,7 @@ class Dense : public Layer * Static implementation of a fully-connected (dense) layer, * with no activation. */ -template +template class DenseT { using v_type = xsimd::simd_type; @@ -129,6 +131,7 @@ class DenseT public: static constexpr auto in_size = in_sizet; static constexpr auto out_size = out_sizet; + static constexpr bool dense_has_bias = has_bias; DenseT() { @@ -136,8 +139,11 @@ class DenseT for(int k = 0; k < in_size; ++k) weights[k][i] = v_type((T)0.0); - for(int i = 0; i < v_out_size; ++i) - bias[i] = v_type((T)0.0); + RTNEURAL_IF_CONSTEXPR(has_bias) + { + for(int i = 0; i < v_out_size; ++i) + bias[i] = v_type((T)0.0); + } for(int i = 0; i < v_out_size; ++i) outs[i] = v_type((T)0.0); @@ -153,7 +159,8 @@ class DenseT RTNEURAL_REALTIME void reset() { } /** Performs forward propagation for this layer. */ - RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_in_size]) noexcept + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const v_type (&ins)[v_in_size]) noexcept { static constexpr auto v_size_inner = std::min(v_size, in_size); @@ -172,6 +179,27 @@ class DenseT } } + /** Performs forward propagation for this layer (no bias). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const v_type (&ins)[v_in_size]) noexcept + { + static constexpr auto v_size_inner = std::min(v_size, in_size); + + for(int i = 0; i < v_out_size; ++i) + outs[i] = v_type((T)0.0); + + T scalar_in alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_size] { (T)0 }; + for(int k = 0; k < v_in_size; ++k) + { + ins[k].store_aligned(scalar_in); + for(int i = 0; i < v_out_size; ++i) + { + for(int j = 0; j < v_size_inner; ++j) + outs[i] += scalar_in[j] * weights[k * v_size + j][i]; + } + } + } + /** * Sets the layer weights from a given vector. * @@ -210,16 +238,25 @@ class DenseT * Sets the layer bias from a given array of size * bias[out_size] */ - RTNEURAL_REALTIME void setBias(const T* b) +#if RTNEURAL_HAS_CPP17 + template + RTNEURAL_REALTIME inline typename std::enable_if::type setBias(const T* bias_vals) +#else + RTNEURAL_REALTIME inline void setBias(const T* bias_vals) +#endif { for(int i = 0; i < out_size; ++i) - bias[i / v_size] = set_value(bias[i / v_size], i % v_size, b[i]); + bias[i / v_size] = set_value(bias[i / v_size], i % v_size, bias_vals[i]); } v_type outs[v_out_size]; private: +#if RTNEURAL_HAS_CPP17 + std::conditional_t bias; +#else v_type bias[v_out_size]; +#endif v_type weights[in_size][v_out_size]; }; @@ -227,8 +264,8 @@ class DenseT * Static implementation of a fully-connected (dense) layer, * optimized for out_size=1. */ -template -class DenseT +template +class DenseT { using v_type = xsimd::simd_type; static constexpr auto v_size = (int)v_type::size; @@ -237,6 +274,7 @@ class DenseT public: static constexpr auto in_size = in_sizet; static constexpr auto out_size = 1; + static constexpr bool dense_has_bias = has_bias; DenseT() { @@ -251,7 +289,8 @@ class DenseT RTNEURAL_REALTIME void reset() { } - RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_in_size]) noexcept + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const v_type (&ins)[v_in_size]) noexcept { v_type y {}; for(int k = 0; k < v_in_size; ++k) @@ -260,6 +299,16 @@ class DenseT outs[0] = v_type(xsimd::reduce_add(y) + bias); } + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const v_type (&ins)[v_in_size]) noexcept + { + v_type y {}; + for(int k = 0; k < v_in_size; ++k) + y += ins[k] * weights[k]; + + outs[0] = v_type(xsimd::reduce_add(y)); + } + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) { for(int i = 0; i < out_size; ++i) @@ -284,15 +333,24 @@ class DenseT } } - RTNEURAL_REALTIME void setBias(const T* b) +#if RTNEURAL_HAS_CPP17 + template + RTNEURAL_REALTIME inline typename std::enable_if::type setBias(const T* bias_vals) +#else + RTNEURAL_REALTIME inline void setBias(const T* bias_vals) +#endif { - bias = b[0]; + bias = bias_vals[0]; } v_type outs[1]; private: - T bias; +#if RTNEURAL_HAS_CPP17 + std::conditional_t bias {}; +#else + T bias {}; +#endif v_type weights[v_in_size]; }; @@ -300,8 +358,8 @@ class DenseT * Static implementation of a fully-connected (dense) layer, * optimized for in_size=1. */ -template -class DenseT +template +class DenseT { using v_type = xsimd::simd_type; static constexpr auto v_size = (int)v_type::size; @@ -310,14 +368,18 @@ class DenseT public: static constexpr auto in_size = 1; static constexpr auto out_size = out_sizet; + static constexpr bool dense_has_bias = has_bias; DenseT() { for(int i = 0; i < v_out_size; ++i) weights[i] = v_type((T)0.0); - for(int i = 0; i < v_out_size; ++i) - bias[i] = v_type((T)0.0); + RTNEURAL_IF_CONSTEXPR(has_bias) + { + for(int i = 0; i < v_out_size; ++i) + bias[i] = v_type((T)0.0); + } for(int i = 0; i < v_out_size; ++i) outs[i] = v_type((T)0.0); @@ -333,7 +395,8 @@ class DenseT RTNEURAL_REALTIME void reset() { } /** Performs forward propagation for this layer. */ - RTNEURAL_REALTIME inline void forward(const v_type (&ins)[1]) noexcept + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const v_type (&ins)[1]) noexcept { for(int i = 0; i < v_out_size; ++i) outs[i] = bias[i]; @@ -343,6 +406,15 @@ class DenseT outs[i] += in * weights[i]; } + /** Performs forward propagation for this layer (no bias). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type forward(const v_type (&ins)[1]) noexcept + { + const auto in = ins[0].get(0); + for(int i = 0; i < v_out_size; ++i) + outs[i] = in * weights[i]; + } + /** * Sets the layer weights from a given vector. * @@ -371,16 +443,25 @@ class DenseT * Sets the layer bias from a given array of size * bias[out_size] */ - RTNEURAL_REALTIME void setBias(const T* b) +#if RTNEURAL_HAS_CPP17 + template + RTNEURAL_REALTIME inline typename std::enable_if::type setBias(const T* bias_vals) +#else + RTNEURAL_REALTIME inline void setBias(const T* bias_vals) +#endif { for(int i = 0; i < out_size; ++i) - bias[i / v_size] = set_value(bias[i / v_size], i % v_size, b[i]); + bias[i / v_size] = set_value(bias[i / v_size], i % v_size, bias_vals[i]); } v_type outs[v_out_size]; private: +#if RTNEURAL_HAS_CPP17 + std::conditional_t bias; +#else v_type bias[v_out_size]; +#endif v_type weights[v_out_size]; }; diff --git a/RTNeural/lstm/lstm_xsimd.tpp b/RTNeural/lstm/lstm_xsimd.tpp index ad1f1b96..dd17f25e 100644 --- a/RTNeural/lstm/lstm_xsimd.tpp +++ b/RTNeural/lstm/lstm_xsimd.tpp @@ -189,7 +189,7 @@ LSTMLayerT::prepare(T del template void LSTMLayerT::reset() { - if constexpr(sampleRateCorr != SampleRateCorrectionMode::None) + RTNEURAL_IF_CONSTEXPR(sampleRateCorr != SampleRateCorrectionMode::None) { for(auto& x : ct_delayed) std::fill(x.begin(), x.end(), v_type {}); diff --git a/RTNeural/model_loader.h b/RTNeural/model_loader.h index 7d66b683..46f42d09 100644 --- a/RTNeural/model_loader.h +++ b/RTNeural/model_loader.h @@ -42,7 +42,7 @@ namespace json_parser for(auto& w : denseWeights) w.resize(dense.in_size, (T)0); - auto layerWeights = weights.at(0); + const auto& layerWeights = weights.at(0); for(size_t i = 0; i < layerWeights.size(); ++i) { auto lw = layerWeights.at(i); @@ -53,8 +53,14 @@ namespace json_parser dense.setWeights(denseWeights); // load biases - std::vector denseBias = weights.at(1).get>(); - dense.setBias(denseBias.data()); + RTNEURAL_IF_CONSTEXPR(DenseType::dense_has_bias) + { + if(weights.size() >= 2) + { + std::vector denseBias = weights.at(1).get>(); + dense.setBias(denseBias.data()); + } + } } /** Creates a Dense layer from a json representation of the layer weights. */ @@ -385,7 +391,7 @@ namespace json_parser } /** Creates a LSTMLayer from a json representation of the layer weights. */ - template + template std::unique_ptr> createLSTM(int in_size, int out_size, const nlohmann::json& weights) { auto lstm = std::make_unique>(in_size, out_size); diff --git a/RTNeural/torch_helpers.h b/RTNeural/torch_helpers.h index fec52148..a4acec6b 100644 --- a/RTNeural/torch_helpers.h +++ b/RTNeural/torch_helpers.h @@ -77,15 +77,18 @@ namespace torch_helpers const std::vector> dense_weights = modelJson.at(layerPrefix + "weight"); dense.setWeights(dense_weights); - if(hasBias) - { - const std::vector dense_bias = modelJson.at(layerPrefix + "bias"); - dense.setBias(dense_bias.data()); - } - else + RTNEURAL_IF_CONSTEXPR(DenseType::dense_has_bias) { - const std::vector dense_bias((size_t)dense.out_size, (T)0); - dense.setBias(dense_bias.data()); + if(hasBias) + { + const std::vector dense_bias = modelJson.at(layerPrefix + "bias"); + dense.setBias(dense_bias.data()); + } + else + { + const std::vector dense_bias((size_t)dense.out_size, (T)0); + dense.setBias(dense_bias.data()); + } } } diff --git a/models/dense.json b/models/dense.json index 141f111e..768ac275 100644 --- a/models/dense.json +++ b/models/dense.json @@ -14,25 +14,25 @@ "weights": [ [ [ - 0.02959759347140789, - 0.013856622390449047, - 0.032424140721559525, - -0.05226530507206917, - -0.0022389169316738844, - -0.0022201465908437967, - 0.14792291820049286, - -0.0340598002076149 + -0.03534340858459473, + -0.052705056965351105, + 0.052596330642700195, + -0.006017900072038174, + 0.0628681406378746, + 0.011521938256919384, + 0.021455300971865654, + -0.08634442090988159 ] ], [ - -0.033835519105196, - -0.004419860430061817, - -0.04077084735035896, - 0.06576187163591385, - 0.02637501060962677, - 0.015842854976654053, - -0.06899886578321457, - 0.014413068071007729 + 0.04202878102660179, + -0.0012000653659924865, + 0.011545690707862377, + -0.05244330316781998, + 0.03159698098897934, + 0.010745985433459282, + -0.03388917073607445, + 0.029600143432617188 ] ] }, @@ -55,95 +55,95 @@ "weights": [ [ [ - -0.11824238300323486, - 0.18984776735305786, - -0.32243698835372925, - -0.053159601986408234, - -0.234708771109581, - -0.5278862714767456, - -0.3106524348258972, - -0.6425948143005371 - ], - [ - -0.053796082735061646, - -0.007236320525407791, - 0.2985859513282776, - -0.4773643910884857, - -0.557999312877655, - -0.37290218472480774, - 0.43713968992233276, - 0.19624699652194977 - ], - [ - 0.2779940664768219, - -0.49355548620224, - -0.5186429023742676, - 0.27592599391937256, - -0.545677125453949, - 0.15460051596164703, - 0.08471795916557312, - 0.07179676741361618 - ], - [ - -0.4904208779335022, - -0.12053998559713364, - -0.22945156693458557, - 0.431242436170578, - 0.23868657648563385, - -0.5095962285995483, - 0.2889946699142456, - 0.32582491636276245 - ], - [ - -0.6140166521072388, - 0.061950989067554474, - -0.46456414461135864, - -0.46851322054862976, - -0.03316175937652588, - 0.4230864644050598, - 0.024708256125450134, - 0.05575639009475708 - ], - [ - 0.4344633221626282, - 0.616168737411499, - -0.44833940267562866, - -0.04195869714021683, - 0.14244748651981354, - -0.024230584502220154, - 0.4486488699913025, - 0.08151553571224213 - ], - [ - -0.10237322747707367, - -0.23776470124721527, - 0.1324652135372162, - 0.09194865077733994, - 0.15291863679885864, - 0.193818137049675, - 0.6473000049591064, - -0.6534823179244995 - ], - [ - -0.29845649003982544, - 0.5155042409896851, - 0.22553829848766327, - 0.5267564058303833, - -0.48377537727355957, - 0.2855444550514221, - 0.028553344309329987, - -0.021203696727752686 + -0.003918290138244629, + -0.34485363960266113, + -0.34457308053970337, + -0.10131333768367767, + 0.7012238502502441, + -0.04543594643473625, + 0.3385898768901825, + -0.3790031671524048 + ], + [ + 0.07936839759349823, + 0.14327313005924225, + -0.5227841138839722, + 0.35828861594200134, + 0.07653595507144928, + -0.2009020447731018, + -0.6946932077407837, + -0.2065955549478531 + ], + [ + 0.689213216304779, + -0.5692605972290039, + 0.21413308382034302, + 0.12099577486515045, + -0.25283142924308777, + 0.0012526456266641617, + -0.0875929594039917, + -0.262367844581604 + ], + [ + 0.41735997796058655, + 0.6434321403503418, + 0.09785640239715576, + 0.3696126937866211, + 0.06955072283744812, + -0.13385352492332458, + 0.42027747631073, + -0.2573500871658325 + ], + [ + 0.12645135819911957, + 0.3164733946323395, + 0.32790112495422363, + -0.6105164289474487, + 0.19617542624473572, + 0.1823834478855133, + -0.38583192229270935, + -0.4277736246585846 + ], + [ + -0.27132299542427063, + -0.11044679582118988, + 0.45485758781433105, + 0.059217073023319244, + 0.08727099001407623, + -0.8194010257720947, + -0.07838885486125946, + -0.13639730215072632 + ], + [ + -0.3447718024253845, + -0.012321121990680695, + -0.2535732388496399, + -0.10530606657266617, + -0.5970247387886047, + -0.01188876386731863, + 0.2204296737909317, + -0.6327897310256958 + ], + [ + 0.3687921166419983, + 0.0999026671051979, + -0.4169214367866516, + -0.5681415796279907, + -0.17642810940742493, + -0.4845951795578003, + 0.12187013775110245, + 0.2767520546913147 ] ], [ - 0.027708694338798523, - -0.11620844900608063, - 0.010310920886695385, - -0.02356245182454586, - -0.04637200012803078, - 0.014046723954379559, - -0.030229294672608376, - 0.050620585680007935 + -0.14227238297462463, + -0.046730611473321915, + -0.016264744102954865, + 0.12445727735757828, + 0.008557003922760487, + 0.014110793359577656, + 0.05469542741775513, + -0.01889052987098694 ] ] }, @@ -166,95 +166,95 @@ "weights": [ [ [ - -0.20817959308624268, - -0.022461755201220512, - -0.028169117867946625, - 0.3681483268737793, - -0.6971794366836548, - 0.21201810240745544, - -0.11270102858543396, - -0.5254690647125244 - ], - [ - 0.43870025873184204, - -0.047398097813129425, - -0.2160155028104782, - -0.5335911512374878, - -0.19222524762153625, - -0.2507162094116211, - 0.39698728919029236, - -0.4653013348579407 - ], - [ - -0.26814693212509155, - -0.5439682006835938, - -0.27317094802856445, - -0.04124649614095688, - 0.3320295512676239, - 0.5519460439682007, - 0.332748681306839, - -0.17396140098571777 - ], - [ - 0.0069047934375703335, - 0.4070618450641632, - -0.27469736337661743, - -0.3182283043861389, - -0.3814701735973358, - 0.5282473564147949, - 0.1502683162689209, - 0.45867177844047546 - ], - [ - -0.6643510460853577, - -0.05349980294704437, - 0.3391300439834595, - -0.22969183325767517, - -0.21785585582256317, - -0.32640543580055237, - 0.4615830183029175, - 0.14473211765289307 - ], - [ - -0.44933491945266724, - 0.42264944314956665, - -0.1401549130678177, - -0.4198732376098633, - 0.29233378171920776, - 0.03885771334171295, - -0.4093458354473114, - -0.41109082102775574 - ], - [ - 0.20841355621814728, - -0.10478948056697845, - 0.7941262722015381, - -0.3240814805030823, - -0.022079207003116608, - 0.42153286933898926, - -0.13157229125499725, - -0.12012029439210892 - ], - [ - -0.0745147168636322, - -0.5858094096183777, - -0.1928049921989441, - -0.37775135040283203, - -0.29634371399879456, - -0.15086306631565094, - -0.5463178157806396, - 0.24972525238990784 + 0.03138935565948486, + -0.25379428267478943, + 0.021432332694530487, + -0.10971544682979584, + -0.38787373900413513, + -0.16265034675598145, + 0.5014432668685913, + 0.7026806473731995 + ], + [ + 0.6687290072441101, + -0.1225038468837738, + -0.4222104847431183, + 0.012664854526519775, + -0.35956957936286926, + 0.41438084840774536, + 0.0883433073759079, + -0.22486859560012817 + ], + [ + 0.3078693449497223, + 0.27866658568382263, + 0.03669373691082001, + -0.8103352785110474, + 0.38431358337402344, + 0.02902795746922493, + 0.08127378672361374, + 0.1201113760471344 + ], + [ + 0.3174535632133484, + -0.7608171105384827, + 0.4355405867099762, + -0.09939200431108475, + 0.14800995588302612, + -0.2040300816297531, + -0.18602214753627777, + -0.15055477619171143 + ], + [ + 0.48993900418281555, + 0.5000723600387573, + 0.41257500648498535, + 0.23876293003559113, + -0.23757822811603546, + -0.4714456796646118, + 0.007814973592758179, + -0.06241774559020996 + ], + [ + 0.020556680858135223, + 0.04478450492024422, + 0.466820627450943, + 0.23737166821956635, + 0.2820282280445099, + 0.4843200743198395, + 0.6242638826370239, + -0.13961918652057648 + ], + [ + 0.3072773814201355, + -0.04788568243384361, + -0.3972523510456085, + 0.4314861595630646, + 0.6439536213874817, + -0.22335284948349, + 0.07541020214557648, + 0.29840973019599915 + ], + [ + 0.1461978405714035, + 0.09855207055807114, + 0.2903235852718353, + 0.14746803045272827, + 0.00036875903606414795, + 0.5026419162750244, + -0.551440954208374, + 0.5533016920089722 ] ], [ - 0.08585952967405319, - -0.00612999452278018, - 0.011831352487206459, - -0.01632610522210598, - -0.021000942215323448, - 0.0771828219294548, - 0.04144418239593506, - -0.08865813165903091 + -0.07373789697885513, + 0.06499330699443817, + 0.003997496794909239, + 0.06643050909042358, + 0.030990097671747208, + 0.019048111513257027, + -0.024648671969771385, + 0.05282875895500183 ] ] }, @@ -277,95 +277,95 @@ "weights": [ [ [ - 0.05400395393371582, - -0.15698114037513733, - -0.12308583408594131, - 0.4648529887199402, - -0.742836594581604, - -0.43209171295166016, - -0.05043284595012665, - -0.012207024730741978 - ], - [ - 0.5854129791259766, - 0.1987193524837494, - -0.3993404805660248, - 0.02702420949935913, - -0.054016679525375366, - 0.1597747653722763, - 0.6549045443534851, - 0.015919221565127373 - ], - [ - 0.19693134725093842, - 0.5775699019432068, - 0.3521740436553955, - 0.22699564695358276, - 0.11732374131679535, - -0.24965617060661316, - -0.09006930142641068, - 0.6065244674682617 - ], - [ - 0.07257136702537537, - 0.27656614780426025, - 0.13267970085144043, - 0.5824130177497864, - -0.04584095627069473, - 0.6138626933097839, - -0.2368190884590149, - -0.3555615544319153 - ], - [ - 0.32703906297683716, - -0.20853662490844727, - 0.8106528520584106, - -0.16368092596530914, - -0.1715278923511505, - -0.019745364785194397, - 0.2688019573688507, - -0.25207605957984924 - ], - [ - 0.6684560775756836, - -0.09408790618181229, - -0.15061809122562408, - -0.2961271405220032, - -0.05506795644760132, - 0.013608694076538086, - -0.6562104225158691, - -0.010354566387832165 - ], - [ - 0.23102398216724396, - -0.5718412399291992, - 0.002263445407152176, - 0.5259236693382263, - 0.5539560914039612, - -0.15677398443222046, - 0.028096608817577362, - 0.10387344658374786 - ], - [ - 0.056272219866514206, - 0.3826865255832672, - -0.06243092939257622, - 0.037029776722192764, - 0.30014652013778687, - -0.5687355995178223, - -0.026552969589829445, - -0.6564012765884399 + -0.16288280487060547, + 0.30220749974250793, + -0.0768054723739624, + -0.002265450544655323, + 0.6957710385322571, + 0.22263841331005096, + 0.5587034821510315, + 0.1744154840707779 + ], + [ + -0.09981019794940948, + -0.020481636747717857, + -0.17108102142810822, + 0.17606449127197266, + -0.24456611275672913, + 0.16554871201515198, + 0.4478242099285126, + -0.8009903430938721 + ], + [ + -0.1700940579175949, + 0.7128530144691467, + -0.2377501130104065, + -0.5241463780403137, + -0.35995081067085266, + 0.005054648965597153, + -0.03336644172668457, + 0.030829260125756264 + ], + [ + 0.29436230659484863, + 0.06995957344770432, + -0.6804766654968262, + 0.03464367985725403, + 0.4033496379852295, + -0.3069074749946594, + -0.3430510461330414, + -0.26389506459236145 + ], + [ + 0.4082008898258209, + 0.5367173552513123, + 0.37801751494407654, + 0.4418834149837494, + -0.045446962118148804, + -0.4314388632774353, + 0.12730294466018677, + -0.05231919512152672 + ], + [ + 0.2724139392375946, + 0.22808748483657837, + -0.1881563663482666, + 0.4315183758735657, + -0.13492807745933533, + 0.7422537207603455, + -0.2437678873538971, + 0.15358063578605652 + ], + [ + 0.2557021975517273, + 0.09555301070213318, + 0.4984159469604492, + -0.41643184423446655, + 0.3402327597141266, + 0.2974848449230194, + -0.31229642033576965, + -0.44929659366607666 + ], + [ + -0.7360060811042786, + 0.21442362666130066, + 0.1361616849899292, + 0.37172824144363403, + 0.1667485535144806, + -0.05684076249599457, + -0.44198277592658997, + -0.1709120273590088 ] ], [ - -0.007640047464519739, - 0.03396986424922943, - 0.023156268522143364, - 0.03924748674035072, - -0.08401282131671906, - 0.007182068657130003, - -0.043575387448072433, - -0.05071526765823364 + 0.04880663380026817, + -0.020496316254138947, + 0.01028179470449686, + 0.038034942001104355, + -0.005797349847853184, + 0.007567500229924917, + 0.02594216726720333, + -0.014992105774581432 ] ] }, @@ -388,32 +388,182 @@ "weights": [ [ [ - 0.5878281593322754 + -0.14135122299194336 ], [ - -0.12552057206630707 + -0.09697845578193665 ], [ - -0.402028888463974 + 0.058321475982666016 ], [ - 0.15968173742294312 + -0.7096578478813171 ], [ - -0.15990637242794037 + 0.5546531677246094 ], [ - 0.07232145965099335 + 0.09789115935564041 ], [ - -0.5600761771202087 + -0.3475968539714813 ], [ - -0.3272480368614197 + -0.15985368192195892 ] ], [ - -0.024063842371106148 + 0.0 + ] + ] + }, + { + "type": "dense", + "activation": "", + "shape": [ + null, + 8 + ], + "weights": [ + [ + [ + -0.5305205583572388, + -0.3290325701236725, + -0.543561577796936, + 0.42929863929748535, + -0.17142745852470398, + 0.2421179711818695, + -0.03362809866666794, + 0.20344458520412445 + ] + ] + ] + }, + { + "type": "dense", + "activation": "", + "shape": [ + null, + 8 + ], + "weights": [ + [ + [ + -0.06177079677581787, + -0.23814216256141663, + -0.7005293965339661, + -0.49466732144355774, + 0.30031296610832214, + 0.17563900351524353, + -0.24616742134094238, + 0.14966678619384766 + ], + [ + -0.05930411443114281, + -0.33255618810653687, + -0.25533393025398254, + 0.2986767888069153, + -0.10642198473215103, + 0.2810288667678833, + 0.04773542284965515, + -0.7993151545524597 + ], + [ + -0.1486942321062088, + -0.5120373368263245, + -0.04606333374977112, + 0.55389803647995, + 0.006042502820491791, + -0.3016049265861511, + -0.4680682420730591, + 0.3109550178050995 + ], + [ + -0.20093034207820892, + -0.12035319209098816, + 0.3936957120895386, + -0.5204573273658752, + -0.11970169842243195, + -0.3155381977558136, + -0.5103104114532471, + -0.38073667883872986 + ], + [ + 0.40579211711883545, + -0.08158302307128906, + -0.26265230774879456, + -0.04430527985095978, + 0.1374949961900711, + -0.7785845994949341, + 0.30249178409576416, + -0.2027994841337204 + ], + [ + 0.7071669101715088, + 0.18410181999206543, + -0.15025553107261658, + 0.033111751079559326, + -0.46437638998031616, + 0.1352890431880951, + -0.45631009340286255, + 0.01344994455575943 + ], + [ + 0.5039335489273071, + -0.32378825545310974, + 0.42154166102409363, + -0.018352121114730835, + 0.6269726753234863, + 0.2561192512512207, + -0.05312468856573105, + -0.040791891515254974 + ], + [ + -0.1071593314409256, + 0.6408118605613708, + -0.13298353552818298, + 0.29158440232276917, + 0.5064350962638855, + -0.09772287309169769, + -0.39379656314849854, + -0.23252765834331512 + ] + ] + ] + }, + { + "type": "dense", + "activation": "", + "shape": [ + null, + 1 + ], + "weights": [ + [ + [ + -0.15197956562042236 + ], + [ + -0.5088167786598206 + ], + [ + 0.671139657497406 + ], + [ + -0.18995320796966553 + ], + [ + -0.12430261075496674 + ], + [ + 0.22196348011493683 + ], + [ + -0.042059820145368576 + ], + [ + -0.4062132239341736 + ] ] ] } diff --git a/python/dense.png b/python/dense.png index 29d664d42f8428da39a7bd4247f199a09e87eb24..3cc442a76ed51f291192e022a3d5c763c3750a66 100644 GIT binary patch literal 16361 zcmeHuXH*kx)@`sWB6d1h00jg@no6|*(xod^K2)m?rakO?5C{a5>b0x7 z2*j2^1cG63*A93_vaSCYJX~;7y5XjWvT^gYbh(4jv~;`cgmQDTw>su=$Hmni=viX#8_3q(*ZwxY+iw)Vh>?7n;LrYi!$Y)Svikgbqqk3c9Gs$RXM z@AYPifHu{4AE3=q_DXNf4t#X(y4OsU@xWstohh4oz8IVt0omsyfa;!Yxc%KuoFG(M zlFCuszVytYcwWuOv3$});8SN_XzSVXGsXR-67I7ioDAFVG|z;$-B-P?e{0msyrphf z_IC4X;^#3x>ba`1eoOJapZCL{&V1$`-vW=Vm|cPJaP$gd8v^l;b;~gXBJk^e1OvPx z2(b%+xFxnt5q=+X909Msv6lhfTVwVA-|{c7n1)?4<5}_Wy0iot&s?)gkD?x@`yx9L z2#NcatQWmzlOrM`ZkL-Z++byxJu}%~p;upDe_v!P0`cblR9`vy0i(z+H#UY#2wBFd z-qOflT#9XNZB1naTiT3;&pcI_UGkVJg;sK!E$xN6GJAEOwSBmUm5=S_|S6O$xa@m0q!S zQ6e>TQu4S%Ux9a>xw=Eg@;i9TW2G`N&8CRMM|+e#>U2u&`nUH(nr1E%fUO{^h6Nki| zhQkEEg(e>g!4r*qEoxEF-*4I!C#g=x7aGwvsZlRqrVuJU%R8MRWVdbC+Ylc7`8n8e zW%j40b5c@LVS!oY#QQq#NZzGsf(eiRN(y(t#=SeYy=XTmRo)A)5)$&~hU?b5z1OG8 zOYPcDXvRzBuX;k@PX;P~;6DD%sgAHb&XM&t+$4f5QXX51QV2hPa`llfuLyW%oIVDWWReyXT$d!!l$a zf4#4RM`Mf_cDcOYC1jI26W}<}nX@*LSGcG^TTt*>9f@|YCFW_k!ba; zJN90`bm`-6>7Z>9O-IM~?0?J3p?LTG+?CytVoC_kbGM)DO`8iRGEt+#k6#HI@6c9| zBYC*GxV(ytbS)bJh>v*&<1uMcEh-R*$?-oF)q6~QNwW1eKRcdZ@W|{H#MNT41-AWK z#1iiK#xA@KH+v?TtjKcfxD7Ynj{E<8UPyLlg_WCRwIMPfvV#%3VqOThZb!_1ZsBbV zLa=^us>*;}KB|cLi_G=J#x}Q-PgCFFibfaSPBN%I_;GA)DQUfTQYKJb`S-*$g+}xH z{hav8PLWaMIyW@3$fxN!cE`UK`k8wI8CgX1qeaXPA2^WG*{PpzR$10ob|q-<%O)Ec zOLj4e2+`_@#uWaVxFxU z$!|}@qC=h2Fc@R!@;($y>nw-fwwgbJ+Hm*o-Tn)Q4j%mFQlV9`*<0#ZdT&^n4p{yR z;^N|^UUOD37ShjPp6-dRJW0QmyItMgQXdkks${=@`LdGv>(?)Fq5$kELaa$L&~~_D zEKv`>PSR`k7H$;=w2UT{qb!!lcx?bVe6B&Bgxh%Qp)zswn%l_e=$d&Y!TItr7!j8q zC&TmbaHDJ2o-{NxY>Y-nMb(qs65R+54Qj$BWg3V0HO`MvNZtAIL&L+RcfP;g5iP^H zPf0~(T@zN;#`+n8m7BZZ)vLoW+SO5SS#Ed~5Eh6C12@1O5%{)oN;R@a5ME|_h)Dxkc7j)%(7OyKS zE9XsibmEaWi@8_IbLHl~tIEm~1WzKec^;NZ|a)d z53gHG7sNb{Z5ZouJI^9kV8V=MDn5ZoU5#0}a6|TU-cn3Ko~G>iMx%;VK`GI--ig2b zl>bpF8fa*_jJx2lyvC3$zXi^MZGA1V`*Hn##KV}Uv>kE?#4f9U2e)dD-CkC>a3^z~ zp9Z1_dJdPPr04d1@CLLFe@;|Y7V zrB;IB?m(4yLvwTY%G_{QmNpL^iv$JLLXHYMxN*fd@zZhR;K74_-}ioQJ)|w00w+}E zq)M1R&YxNWF+2rjs*vGZyT@`6{=4=<_{YVuN4NHM4znUroV%bVuB<gcpB% zd%J{9BP$c@Sw$mr?+kPFl7ahp>m_Xk>LMG;_3w)eal^fEjxRz1Hat8`uPKU(&f!+? z+_~d&F+sCe`NoZy9s4-50X?oopsG81GC%*k4$Ds5mTbQgvKz5q7T);zcV&?Z5lfVX zf*&5C;E=TEC@n2r+gNCciitU`I?-F2_eLv=C`E8Chq#NY(mZzE>zTs-+bgk;>L2aE z1ZFlgHFf=XZ!KxpdW?x%_69vc{5@v4ZLW?9iHd4RNcb*0nZaUAw50_%eE+^Qo)$h) zIfLzJk|v(1_YIzXC7|aj(rRT`|(sBIkLA3%pD z=WpmR_f%X6=JCJo;ZX*-0@*kL zXOEWGEeU86E51lDT#s=1U2Sv>78&Fju2%bR%+)Qe42LJV`AWrW7TdOTPxchY=Us)d zB-=i7q<5XIe*ZHNIJ+yB8s5wiY9v_z)ra@@|uDlY)w^!(7+ z$O>sPt?Y@(&tv1r4=!+)Ta^44z9|3Gq;~w^OBqBn-!{dXM_v-U_nWS~eAEMY!ixLF zfKXIF_Mh*IV+0W*rXQ}GVcc}CUVWsir&kKmt=uQSK3Tj*(hkU7p6bg(uP>`%RmdbeMs_*cEK0&RjWxw*O-?$6@+g=?j??k?$3j??*tMnZ=o7tJbhUiI zCy$<_%&UD!EdiUwvfSZNxNP)7lLQ4mu@j44@BYCBry>yQ*ta=3I`;NC?HwKHg-MS} zcr-9I?N3MQ8yJwjy*vpe%R)TPp-T|zCmnj0;@6BzhvkvPgicI2R1lnTyWuoCL3~oC z33{H?K3L)g6PhLVz!&&to>QlBLUCLVcH z2wo=VlMrxXVq#c<3l~-!J%KSgrg(X)nWCfDcuH`IQ_nd_k!lAH9Kim|MM~?{QQQq~ zL^D_B`7NwGJcTb_?El@?hzbcsc93LA7}v4pt}D!P>68IqZR*;RjD4Ro0V5zUZ*EfN zWPi8fztCsfgILrDhGwOHeTHIKs=dt2Mrvx$h)kSysPSv+>W*&JD%{>zT}K;rE?v66 zgNfbkzZES`uQ4r0XHFpy$$T*<5Nf*4&IO6+<(t(jLm|B-_L203go=s2i4k-Z)l8Bn z;5X^j+jBN?*vu2|q;@4}L*Ne-ND=AJp6wyHmTAL*eDimlvVKB7J4C(D(MP0OCZ#>BrfoSaFqo!D$8*CYXj~EI&*`ODhauhuyRc z4h0TR^k1J+EkDA>mg7l8m&Xc1LOsje@n#G3eTobJspx%_!FCSf;ig!lnY{6np5M(z ztON2%54kS(Bx1+$;-G`^!mUT%ToA`QOWBYe;-Bse?olg`b#{wkK{IVR24zcgxTO1e z-7SNrml4}EzB#FQ1iEsmbl#HMR&%UY)Ej{D;J<)Z{~l2%W`HpLa5n|xMoNA1WLHLJ zrUQ^or;xM;p~7sVLi1-&o?M42cd1!sW<^-~ugVlYPeATtgZ~_2Hq*l8+nJ>MlXoBD zr}v^=**ak5unpC*`oPN&7JwA$+x{FT)?Nj+dP<^ZWIpWP`un`+euCi-7mL!v{Lze7SD$ zDDWW#B67n+$Kv+PxRuoQ)#=GcrVdphiLrv{`TnQF#`9rQ#KP!a&t>f9;3d+qp8#*8 zOeWXVM6E~NK>MIm4m#SfuPHD$=M~msj0-O#eT`7?s-53A+F&Yrl=+1$H|;%=HDWI* z)j|!w+0)lO{BnBcu2Y)SOiWp@S6yk-G+}y)W7iig5VTr#cP}xsVh%_67nO1;S?sb`0#Ldrig4J<38%K{8%BX zV@AokI_=I=>I8OQ$n3J7^~OLI6K#{iX^WZdDGvvmwW;gEMQ8%vQm6fA^zHjOMQCA)-`N-t!rn!?Ku zCwQ7O=a5^{lQg4J4{1AkK8UM3neS^j;cG@pbV)SszOe7k$I_Di)y80eU zZfFG2wk6DlR;&?a1ur)*e&ne2KlgEcw{b+f^O8S%_>&9EPYd%YO)|XwzWO=XmjMxB z^2@{dc(i??w4eBV`tgox)S0?)Y^T0jlyCa{lsn2U(U>$MHgM7qU)?$wY~U)!OVOIq zsh*#iu&J{cm$SOJ$*0?1gIN`;*VW3Amc=VfSQe`mss7Y)NA_LJ=+~^nhU$3U{;Eet zPuUsarDWCC0<^Em$j@=7^w3cCOR=776I`S7CNP}Q>wub-l`mtC z@^u`)`}4g46KRRle$0o2VY$O$c{)3*p88!rX@wOaQK{i)ZcNxrmzwjn5o9)4XUbx^|uF$8B|U!~V6tM)DX9LT#2 zX1@ZE(#yEmjhlQ0#jq+JSk-HGW_J~$W~JJb=LS?G z`YY4(^D;igwkK*1D4w1Xb&79#gHD)5-(RfVJ9CE-d2uZOL3%Zj5O?BSw~VAx{6bYn zj{bC%2l7QjxoXI{!wO!R=>?-4_@(bmA@khGzT%r|Nh5M`oP9ZSM}~c+sPPi+rhQWb z*Ii|QaIRhIU4CCO?~h`Yovju{#5vtqpK#~yKRwd&qMzvd(B)dS_{rvwYCXNHSADZG z&-b(_28iGsgn=ZRo15FSd$-Qbo3X?V=m{?*H1ELQyO?ka8L&ZwpZWOt*Gk#W-co$n z2Mn9bc}`mlhvw_{ifY|v)G&>)cAJ&kb?qQweW$ss0XAxYP^ecUEn7F)eUgugcOXT* zZ!ZX#n9$}YEx)iJaSXp!H1aS`eJ-dp{$pg9N9E%8tZ*)Shk*GbChBgfl+dca{WCg` zY2RK*VR+e!I7+jOWwIZ!OU4~pOWonWQQ)Zz3vQt>CMS6COs~fa{({$r{enn;lZ23& zwqeU?A>y~)Zg;Z2I&&IuaLr@i^;FI97->ZfJ^1y+=C&{C(@$NzP==^Q^p5^xQ-u+{ zkH47?_u6<#@c%M1qpcpSKQH*zJ<`88#&PD%m&3bH%F3Dw2nYxY3bsQY=T+qnSAoj8 zq27LlNkmv!6aF7}?#?rmIpkny!g*DJ-j3=Z=>runItxWPFcWq_kuy2Hk3w%2EmA70 zS$Km)o z*HQ*F+3G~I<^6kke-2^N4mpla(3~hjU!1?~y7x`Ws(`YVerw|fC*EL<;H&$QP!xqt zmie@q8JCYzQ!jk&Us+W^D5uR%XEhmm3X;yNl+aEMBl$utXhU}70L;X?c2!m`S!O`n z^5tfh6Has~kvm83#1vt&I;efAV&H9WI{s6^i|0bsQxBFfg&tb=gQ3{yF~`Ehp7@h; zgnAi~1ZtIv_vKH#JypXqD_G7mk;}C2>+V;EF;@yPb4Kq~de_~D-M>&?^>WLU8bigf zkfyE@Mm&4Vs|p#;b9~M1>82+Ixrb41Il%WTSGxhF(`=nZtwXkqU z-)u^4@-RR_c&4`l*-T`%f}6Nnu_-!vVoD;BQpOm*Cvn!u!t0gv2G^ledXQLym#_KryJL*dNmq}t; zk2Kp=O~pu8d45!FA1%))pK+?1VgjTVueMM*CGf>&+_7cVL3+)bZ`omp^iFYi$P53? z^i7?PD=g^OSvL$4y;4@xar2w0PS>K7)P@IgeW}rHOcsr2ybGD)Z_Lz28`Y}rER*gp zyMnTxuX8W7SQoV?K5fa|`|_1TUph5-Sge*F@1*n-NjVRhfFo^=Kg2!G6KHr6IcaI3#ura*re?ukYoF<~`uSBlYt<;k2QLWbi*A>^=Ljf9H^aERj0-%ABn%>C;TqwM5lMQF}X z`Oyq1Nv`SYwLS`MT)3FcTd{sBgA*2=obHKk1F8i9yYoGJFBVlFrtxfkCkO0d%JalX zoml?i5sua-9(ZU?>&6uq<7pjH+?Cio`z%SWM%O)-b-JnaJI=*}`MS!9wF1 zIO!ggHGM6ax1L`qX@m!t?(DCmjGq@SvdEK^{< zo(^L+o6auLa6`ThmkOg7zwEdY*)o%H{E>os5nIse%)`Bh%lqg{7DBtJ6D2h+a?6iAvR3E2!btFX82vUS9mF zWZ%@8NB1{q`sC;b5Xal!=&+D8)n#^qm!SuXQ&T|y+N$(8c>H@$?0u1}dK)ZOCnN;d zfxO}?NKt4y@*#dJ4yU;otq45&KhQk>M`{HyTz~W0sEcngLj)dT+qk_&I25rf*#H=} z8vCNy{6U{aQ<=cmN{Byo&bWk+PrD4Q3&iha3NCM3lw`LmxYvg0m_aljX4v0UYcsT< zX(L2vL^Da$Hh%7P%81W5mr$UKA%FK}{{drN*9Kcy%zPJPQr&dbIlxB=~#n z%QzZ!#{K7G7Qu7pbQ3PRwaXO^|0PaX4z(Vc%(5lFCSMcV|n;;cN zgHq}K>)Qc3-E`=bRwqO_hZD|qOp7iz03BwKB#&N%hJ*Cl#JLSrg(W<0v#W3l(nKLJ zcSQ1!A3t9AAAw0G@9$)|3u>F->C>$kcd`T!>8kAywllglN#gCM`zwHUzi62NI{l4+ z^**9sVWsCx!A@>3MUeE;!noyfTU#+^VWFW=tvIc6(MNgW5_-=E;6&fQe~+C#rAi`^ ziu$Gm1qAHpJ@#>)PaYq)!@<*$r?d_I{Ho}QOey4O{n_V{&RKt|qz~#7hG?9MP8auS*@AT8RyJUv7m1uf6S$O2sotm)!_?+mHi z1ZAM4^w_y_s!X{&DmprAeY$dH(Q{j1I9>PG3fJTp`oO(lxGy$PtCH5WcRlCzR z7rTrs5|$|UDhYHw8q|(7-bC9;x}w1?a$EfBtFrE)@Q8V}t#JDoaWvU_N;z zCHhcJMx;jx88r;g{T7Cnz%T;RIM_KBYdKKhl^J?PbOd*X0}(670B)H#3lG-VYX+)) zg@B*#>FH@`X~6@xL)R_a(p1)`5d-h7gM;>Q<$eueahx6e2)-0LrJb9byV$NSx=MK4 zmU3n1<42FiK{3&X@?NBRQ5TfB?04_3Gs~|eZ_WlQY;*B!Ve2<)^J(ntvf*K|5hPBc zk^Dh9%C%1$&E%SuukTb;H7>gS6zE2fqObQ%d6@Z5mBuLqU{R|u6uPQ!U|^sc(OqaU zQ+eeBESP^^FUCEO&a}OIcNRQ8*FF311@HZQ9Y|7%Kgg}}fv|*!jeZ|1R-H8v&2pL8 zpS8D9Cv5UqnncI9-ZC{cO=CvCP6zkUiX8UNw-?8={NAYB*kls7Cc{~M%bT6F6(roi zQ5D3WXaPU`4P*X|u>TX5ST}BM5f1weOSc)_c4yVy9_h9%j{V?;5PtTr`UK;+!e<@i zY<$e)o~j}DqeFvY>`f)UOfC&Kf|ly1vGGkTQFG@uhC|_whobj|ocMQ@m46IO?m>Im zPS6!sg3EVde{kK6`?kmb9GkQ;<{*mMv?NaXaveD` zvW_V$Gd!vM{6z<;Q-uee4(*OT*oFYpIvz61|FBx88$k9xxQKA>El~xw+qm300r+@c zRpWQJ?iW;yvNt8j7=dprw1DHt5x%UBp1~qI^>5+7nlTVSLxW~vR};t(CvF=?7c2-6 z$9tHWQ7@^NW`8oLZ2ed_xbu+Ry?=mH7f2ZO*JpG9oJ6oIR=}s3!J+i;1Ak#5UFBV% z2bo9Z67cF0n5O^IOd#`Y#{_m*7r9VlDm)OvLfPR6jHw8SENv)rJG<G&~*yPz=8br$Vj z8^CJYDk_MGYuIV}m<}D1gkYkQEoizR*|d07c4TQAfi^Ky+BaqB;xf;Mg6KcCf6GC{ zOx1!`D2McPIud{$Wa}L8g{(Iz_$+2G&k%{@OK2ulP0c7?WKWS2{3aUq1`-Xm_f=Gs zYP;&6Ua`0hGz88A9mOrTg7gr$>JE3Y^;gSIom1w8rS1{r6 zsy==C6gwq9TeG#d+P8e>uq*VqscC7Tq$hRsbHMyucTdDC`cS6xAu-YEYGBjX>J&Cp z-n`)q<(BL9+&LS@CDWN_BuYF`9RTyBZ!3HkT{M{hXGtqqLl%pZs2U*rMDl_sO#?Kv zHce?ldcfvF65abpFMaLif7NsT1T|x~aK>X)TX`4& zc@%%ku5*WJ-PV~{DV-dqbGH+@&iY!!KkcdP)fi@P3PC)vv7W$Pyj3)$X(-alo5_vm z7aFiB@@d=nw{(to65>=orW(POoNdP34Eq)H&h_yeJG*3|an^MTVGM=Qf5x{Q#})9c zdMPHSkV((L_r&L`BOEq@-M< z|GgpPFKpB?$;^T4lo=yK!`E<@j&N|~flQJO`M+%$6s&wFDU*jCLWNaBj`-Gja4eP9 ziomJerVsAo^HFc#o`EB#{`m3Zi?G|Z;XH*9Ob(>>45l(|Ev;^FeQu1(Ei{VzfN)QD zUEx++6lhiF#KqBt*4EbX)&AS0x$ke^#slbELO)F4`_cuyush!*>(av=su4%e+(>pT zs8FU?VY<5QyUC9pJ^Ds+TWf-3hhK~*ZYV=*wiM69!Qp$^1R?z0DcnbVVWGXzwIb9Qty0WJYEgI(7OP=N@2a7{4`p#}EVHV(-!Z42~L*Uf;W zE5ZBh>5IlqjTf^H62Wb_Fmay0PFD>xwGwgx)fdzI;^om8DEbED%TPrBZu*HYDGTIe zs9I>2k#z3=5Lrfdm4U&-V`gAVnl|2+)(c+V$O6JTpczyif}w2Uqpy&9dU}%JwlV%T37>P<6JOZsJ1sm360Eoy^4$TX;@0~ zoxVUFV!_$9Q+mOu3dRiWnm?cbyW^eb%s@k9qdugk`Go~~Nj>uGm)+uc8&)RJbMi+D zAe_V8&{uX@uXiW{7+UPWKifNR`-vzMqpnFH)f(eU}Z zUeTeZ_t02+7J5%ndujntaQi9=x>9- zKXV|+weKR5{r4S%-HYS-C*j;S$^JorL->mZ8kuR#VU;=Gq@ULlTV2>>uF%Z+yFX%d z+iu%zQ#90MAlxbeR@O{kW-+4*JBGiE(03BW#2Cbzkd^m;;!08Mq;7z?#Tir?$00-s z_kX^x`U`M?YZ=CY2gbLY`FvK#B0j4h-dxtbA|*aME%Kk6qZxK}WECMMq_zGu&l zxSGb{UZ+y9Wt1TP7~wWF!f){YPP%OhNLw}NWi*ZJ&hDi?4Q`xQ4XBc zU&7G#WyHtl#XO3$VjU!pnzs++5#61gt?b_|8G z!fj>N(jp$<2|f3fMexj-P7_phhNNS+-O!h(!tWUo6Jms%bCF#pNl(>xwtfOjMNLjW z#9Kr<-N&Ii#q{6oXRmna_6)F?(nUYzaPE9?w&eh~1L|K#@}-Bos*Fd^v`XSl(<)rY zpgYS^THX$20_!&FftrreKU`-SSy}6&U^S&%@Bl+1(uanuzSh>ReM;hmUwwBj&r45F zZ$ldWog>Y)dAnltcRp3MYu9GulU}?SG#PcN=zMRjf*VOmNuirhO6{+in{F65WDB9{V7SXsRH3qBaMUxrj-wH?M+!AcsC*R9Qr@p zJB94XHz^nIE$bk?;spjvKuoMn8u#|C2DrRuOM9TwAo~2-aiPi5Uw+>G_Vex=9Ww}r zk0*{XyaVu>Xid2i;#3LVLh!5sF+aau9%vR&(}@XR?CtFZMMaIkN29B&%d0A6RPd^9 z7?e_nkWN^o{bwF*vey8J+e3fVTZW8xaC9uS{JaDCkmK$TS7ataOiL?-}q8 zUAS<8{i0hoIDrbPmyMf}6maeybVp7g97!zr1hf2h42CDMqktF&hg?&lTxFX(6ssj} zr1sI%(9zcDKiE8cmX0DU{{%%~cmC@BlSV6z@5L>T4bc}L2-66ogPab`Mh$vs{hb8o zij&Xv3iof@XTbd`3!^4!a3pgrK-TAjZsLTX0fxC@Y9BM)zR@-KIYmAhd;(1AD*L?G zF}Xv@_7erN<5T3cU}R>_&4}DQ!CyHA?I~&ti%TCL4#YW5rO=<=$1_6~Tv&8>y@+j5 z*i^SNv=V}ZqYLzeAroiT zAe-hlIbXh9RZ><4GpiZVZ$T3wB$WlGv8EWYE6~#!+`gTT0lw&h>*xwq_~zPLf!=`FuTOfneU4Eq@ujw5!&j(m?6l9o0q zwrOf#TO!kM``o#G+YWl2`yx%9hK1(&^^vE@U;sCS1RTbBA=UD85TxaH{3ztlR(g&G zIYcSXsIZ$(gXLLJ1p^J@iO>NpE_P_=S>XKr7f9sMxzEfB60g~1T*vOyEQVpF#DOAoRgK zSjXRSd;dBheB)gWJ_su>4of>h#Npr`(~rUV%tc42TCpobj7N?hoyadAXorhX68DC_ zboXv|J-1&UC^09EMA;q+^vK=U)YQ(wCT_XWRbdM=4&t?5n&=9$WL1WK>%h^YHdWxT z$4Ws3dXAS3Ao{*wN9R6nS-Zu_o{rOJ&PWtg&8rPB_o1ZsKEyg}(QC7N(CSEF`iJLO zwX!S_`N) z>PvX#PlbhIQmutG^h44Yi$f~D2~4;ud0oy@oD6$U{N+uFtv7lq#sI`3(m#ASuc`~! z3Y#^ArvO~)W@#miuZv5-i3G{&Q=vBB8s-8WGU-}ZLFM#Cx>*qz(az!)d9n%iPfs*9 z?6-SG5RHx#shm6qraBV25cisw3YW?{!G@a+hg?0i&_qpdt@IB*lKydpnvl5x4uRJ+ zuC^)O1xBgt;y+B(g7mvr{&4pS3*9)vzt<0x^AdVO(A8mi-@BU{EKmSJ!-$prg3GrwI&c;m+ zX=nNjSphCo`^r&xXoU;l#ulLdHq0B>IDj%p@r#2GnbjibK5S_E1)U2w~bRFaUea2Hr%-0S(GPfJMXL%KJv_9>-X>tXo7$h6{{ zc*VqYpgEt5$5(oyAb)B`_)+V98+t0igC>FOm4J5Nn2s;>=3Qp`!xg%qIZl2}Ukb1g z^q=-#9M_p5;*zimySLohzxxmSCLo31!XK8{SFc|mt!Zd%IlC7M?s4eWJD>~bDs?o% zjR0K=?9u{xy(wPW5zN|)RC{_}l&5XD($lsy6T8GUKp<=(V8K0Ldy{6*rxoHhjiFtC z{ri>~giB%JA2#%HatF9+`6>xCBOBN__-@`cOol{;MMrzPzPWh;*sv_(|3=MrIZA~p z5+82~0484?pnmQXqbMt#suH_5boHc^RQKHqQ#zD^f%Nv5FJD;m4D-+ZzMufDfgjSj zV&rI=UY1rZOeIuXe)S6&RmH(0u;s>5McSrW4LvlYOC9m-u;8^@nPqRm{UPbv2tM`a z+1aOuGosDsOTEE&`~1g`8^cR@3u*90i(CKt_EH&NU=|8x(DrvO3v4nTZ|{Lqo�A z^!h!7*Dz4wl5pE6G)FIMt+y3#J39FBVQp5Dzk}ZzDGN{tZbba?BhV{BmJ<0EaA7I* z{??uBP~6nQ$^4B#AHX5KrBppr5wZ6&RGVinx=-AG6dcU)a3`~r{N~EAV)SjF2=+IL z0h`_y@h}uNy3x4}t}vby6l_?b!_j1KsnA`eq$~cM2-uf}(RdU)T#I=B^X~cQz{#++ z$Syo5x`fwR19$lO+O=zEVRglT46}y5iXpeV3Mlb=a5<9$M3r+VPMmlSreHQ2l}JPC z8yT(to)#QtyIHC?$m+jm0k_C(`|o~cQic-}3QC1FSSM?Nt>5}idel<5g=NuIQ1&I=rERs`!3KVECEH8(WETC6qan0@pa zAjl`oca7)bBR4lES8;xR`+xZbK1UZTerZ1~F1X1FCk1_11cKU>_#a8Wbe=T=A&XVK zdrQkJb#c_&PYd64xVmMYa{A(l3vr5UzvE?pkUe+Sby1Ar%CCC(=D2L_rMmF)C);PW z)SP8c$WC_)bA`8i9Ckc{~WdC35HgUGiUEv5!ZLgA%-(OcsZ#l<%jABs5PPcN0S( zGLKO-_k0K@Pl%7tjydhSbm}r9{jqU_2d9{rSdRs4MSq$9u1A&KkLx`aWMm{O6hihx zT5)wpWaZ_tLw?BZWmQ6ZYl={a!>HeLVCsWRf(^bfxO2Fb0&n7ZxTVrxz z*2U)cmaEG~vqSWO$ZrYbp>o;G_JUXz52l{wj9rb@`x^>Cm4>Z`{DwW!IM&(k8 z%VX25BBk4WCBoc4Yqrz;jQJ)N!wh$_Ycs^nMWQ_CHv^c2ru#M@^W}|sOE9x@aO@sL z4gUcz#whulfr-huh@?z?GWVpj+*i86Y@HcH?Cn(bldJSL^h_m>BFNt7U$I2W3%%1p zhRKTPC|Ktw3zdwzqxgg!T=ZDGu7-%(*u73>=WJ9r3%SugSgWYnJ}~+#^UZ0i_}iD$ z5)1U>F3}6CKdaDFKR>_F9Dc3FpGrU1Qqe^YzA2~_NpF~kT6mJCihDQn*bxGp;qre! z=F9IQ_BQ%e-SLCE+?RAF}I#Ud>=0&&`$=MO|gc=*6udFsw|$p*PaQOB|3$E4)*;oPM?$-=fm z0efDfF73R8m1b&xdlP@WwuObo+`;ZLc7CvE5U$F}%j1@o#+05XMIbI-)%&Ucp7a<+ zPY?;I%}793s+i4ibxFF%hN zzN)S>U4xQaBbvLX01{5K@syLVXife4b(NR5)6vMsr{2ZIg-tOhC#S8m6K{L=%$cmp zN+D!QVSIf2m&w&_N?1mn{)?oU!;3n%Z$H7+K7C3C`_3fprr^7^q%A8en_xfU;90ev zCgorM@Zm$hJ8-|W0aygJBzkj1G55{bCu#QUAi>T;OU4FF+u76`0)cR0?$gBz<%U`sp*?<#rkiBZNDj8cU*sG*-39d zTzx;z`;o`?$^?3|X3VE3;9$q<{;O-Z7O*HR?qE}Ne*t;em*zP(XZtcCA&1t)>(=^o zXMer>f{@2=JsKLCozj!X)EZvM&^qPvVBqDxo(T)W5azj!oj=o{xAgV(v^r7EcAo4OH73Ho*qeuXo+VCt>zvc@3)vaEAw;INk?rb1e*x1L=~!LKbc3>P8*# zzkg^((DF+yeT00o};OB&GV-ah}MgUBjih*C*-4uKxuQnqblnu8x&^S8$zeQ$jkk@Zb)CyQo5 z$3c=R!oB))8{VuAU9H!oBZvGizLxUQ_dhDEiU8EZU%b##QVQ28HPW{qZs*4sQH4#` znvTDJj25{p_u|IV`imDY2zv0OK|v=@xseFt=ZjEW+}z5p(;ez?UAp%)9UNTS=1|J; z=xFKEeFn$IH+_8?xLyC<%>wyo=K6_HhP@#RbMx7pB$Kr!Pw_PO8c4^8Tko&?BCI-I zz3F``Z~N;@?EFww(R9b_xB2;3JJW8~v~6~#h$gj9tx+l`@~CTR4LFnZ%()S&Y$3?~ zd8iNwnfH2^5Q&M2T%w}7*4EZ&XI55LC1qtz6&01p>D8YR{5D@=SZzjX%W>Nl?TMUH zQYLBQ?tS0O&8i%6Xqy>z4UJ3Dt&^hgBzlqTN5;pRe6`ip-@r-7te!Y=!lhx9BQ0R6 zErGDqu` zB}K%(aLqNZzhHLu5e5M7y$zh7%YOoeg_YHA3#4%P%q=X8f7*}U9J1x&7!+)!BV@w1^qXA7pEVWH3P^#?uJGA_WC=#b^$&ytPv(Npz9AGyQuTs0=2zsVwpnf}k2rW$|o zL5F$~mKxD|2{O4;D5HcS?fE+hiihU|j!7aA#~=RhP}}zx&GGUo`UQRjddfc*-1pz0 zOtr~m2Iy_2Yd;MQ8is#nb~Y=q*%vGRhW*Bc?VGjzCo-LQGN{6;TLtx`YN3#Tk1_Hh`{mg9>Nc}MFM1* z-DM22NEf9O%`6$8o?g1Qw`Y_0JSytj(rAN!^S2VieW}2`)mCBG>3f~81?9PQiq$4t zA`{>@o5QHF_@A&x|CL$mWR{B5uGb zMt1{I73`LH3-PELe2h{94xVt}fgh320>jafgmZ$BdGPN7NmW11`SXSD3*VGfRER;m zvA>1el^SmJt{8Es!AY1sd>9`Wr?DuZtfcg7)oYI^yNZBIjWl?oxVX6XHpdk;fIk3L zNa%N)A5bK+lE8y?mZr5f?lqsL`jwBTs?1uc6xZ&1a;>NSFD|k6nvWiytEu8HLV^hx z0KJr7tmbGlio)j ze>C2Ee6e|%jpqta&(Q%bfHDX4jwODsUmn0^khlX`S(k{L6({7k?Ud%f`6C%#m(6;9 zprC&I$tfZ@(40Fr`NK--KqIryEUWwP@9IEi27i5hNt7I*JjiVX z0z-H~_~!d3o(K{h^Lo4a`^Q^O&D<}G9z|B&=?0Bn<;$0%;37`8Y?rWUh}0~|GwtwG z{R+$Z(S{oI_Rmx9i$jB0%a2b_z-INm5-=k|Nd!My?{BOmv=?@h^0@^O1T8JO1O+SF zx)XT~r?(D(c7E$umG=L*BgRN|KTTX)Gf$TWvi~l|0Nj#llT@0b_ ziSNY|bUKy%BMD~ss*g%VRloq^5(xBF5r}67lVvM!BT9@K2K7pc^hM{Xjg15U%{l#R zWS9>bTo(Dj>xmFzBa?amq*nOEX`}6gr-P>v!50F5ksxG@ft&nK%Va$+L|3|`q5V*$ zy199_hK5F^%T$}qFdAaM{#TsxK(W3cP}O^oxm&AP1Rg#m;z&$vtU5NLcbrZxlRICG zd(eO;A}Lz@p`PDo5uQrRet_O~viG`3e{GW$%B$eS{r#tXl>(A5}?HUd$u_MMGtt%X+ zzD<_tSs4}^NhdDYMhx0oYzXt5{SV2>sq;KN_BErqKxYf#SQB<}hkSn?7>tKXo+e*X zyQmOQ6{+(;Q!hRMm{C?%PDn{90-b@Yjfpu8+JlpWBTsJqDj%PYzW(cEQO70G0LWB8 zqIC0A(ufRvakvIG{Rt>?0dwwBgaR@M)Wet&{X^mV}Uc8{_ z%#cQcGVu9b-dgFm<3f4inhYFY?JhII#$5RG=c|-ozkbcbs6lj-oH;*xWWM=rv{a#N z#CdDtUw`ppBa-F1`LkvB8i&z=CSOlrOD2%TH8BDEQ7PToiZA065~h5YSG|rumZ8)? zH1|e4x&iX)8qkn8Z{D=Ex34o~BqebqwYz_`&hT4PlqAf)g@4g-udTDy_Sn+2pFe+A zTKDI2NdpxOkBZu42$CsVJ1Q!3=aVc=8ND*u`)ALN zt%%-?mfU=GMK!$`5=J3#U|@UjkBf&Vo(%wh9w^_pBxFDV@UyEoZyFL4M;H|oq$}na z3pm~R)I@Igmttnq_`*B;VUduhTGJp<{dHiQ38TvaGol&9yhkmu5_J2{ohpw-10c7m zM~@y=Qc}`5JlGeu9})uH=VJ_NE{I_M8gW(e_3PJd9UaAM6+m8hns5-=CuWFx(SLg) ztV#>?VUWAt^yzD`i#Fu{=cMD$BAQ+2ztBH*`C_zzo^?~TMu~R0&;AX6GQg-owVzI8 z^-ub(riMydY>XGnR@zLf`j`?8Zx%CM*5sB-FTWnnds-}p~3h8F4fCdHv%@vU5B9~K!{0@=uRW&8tCC-~Vj z2ktp{riy8TIO+|j;U1`UHUrYTurkq%uhN6NqnGrll#604C^c@dfzn2J zczEAPookWZAyG5a(a{kJ^k9xMEyU*inKaMz0|FMKbbaeUa=k;aei^+h^r+{e`|_Ca z`b;+#?y>RB&~b1w4l>XHKO7*CUsKc5Tu|%~^=pAka%w6gW-|`rc*z#y zy>1B_93vtiAfTkESm2Qk__5|C5dgK9pg|x+z{hj}Hk1?;Rv0p3VlLJU8&3&2)a*W8 zj6U%FXSQO^t9r-k!sF7+>0&M0!Qu+UhWmcG57MyFPx%dMksBC+-q(Kx9P?~sCVk~0 z$v2t@*ahmp%@`*!KIR*zxWS{-R8@$dandXDvDIGac3G&Z0$p1*RHDOglL6es$S4&K zPeM}CjHnbpzbe#A+N_D1XqaVR6ut)H8DI}ANqZ8x$G=p7dK~u1nePWhTY-?-l z>lDQmOZgM>K}Y&bzqZoS(#np#EUy$7O3hMQ(Jf`zpQoC{HtO8URR8;3`^JE#sx_!- z;B2vIsGUe2?ybjlG^- zt8Qi@uXd<0l=R&k`1v`)pVrw9Dm$#K2{(oK8&_`g8r7<8je2G*HEo#?nGz7gD# zBV@{nvsd!-^A*fmUZ~%?^_UO6{gBtBQ4bCR9_>dA#?RMD)Z(rS3MRaMtv_W2D1Xyo zgi>@Qlfj|(&Cq|3+?W+a$T%GtJ86wm?K{ZG{^>nI{w8U7q#P+##}gs zZl82bJ`5F&bMVVq`?g|;iQ#I?-saPIxgEr#qPFgngA_^GH-5!IGo?&3)i-sPd)Ut@ zy3%Yg7EP6OJ559H=?c3#U4Zpe{b$O)F;C45nH zQL8^V>N=W`t(p-p;-^+ec_Q)}cH+wFw9{Cp#$@e#KGibrA?_)EekM2FP4`~rIkm@H z>X#^!?H$gtS>vU%g->zUu>PU0F?n8N(yKu6R6)PCU6FI!@gSXQ0!v?1Lfr06&cHOS zfc&+JSLTO^THnaZg@wYI{!deftxj`y`o48{Vk0Nnv2_|B>*9(U;=db*??y@I!Y*x+Zaq^Z@1ci7HqHu9;OUqoO54!B;c59+GTx;M@4N?erjF-f0 z+Q*IHl+pqXcRus`7FPP0;QN_2wT`_k+hO;#Xn#gxaxk0hXGKrP!U%Q>W<#l$E;EWp4RfUZ^>)bn_U$bK!^Ar; zjtp2hcrnym6Ww5aGd+_u#+w!xaqi3&@AmAqGZozcV|GDQ5kiNmREtR>fi_g~vO)=0 z9Aa-X%%bK?Q8@|x3Jn;6t$dXcqdF-c?v54OGkj@-ANI_=Dp7$=HIrGxIZL9Z0H+QSp?G7A^u4uuQ zLP4i2zLylA_n=|p^F7!7u3Zx@Zjsh#eLi-)5RzjvCb=f*VVBf94*m}#DcJnib^g}L zYGN1%(O3TNZcTi=5`9?t&1do%#y>Xma{A_XTF*_wC4$Wlwit3$Naua#O#h(VKq--~ zk23Tv1YSpOrPwjs#C9O3MUu3Zgl)#EuLtcJgars6dIuj`9vB*!AG~o2G!zqhj-ud) zVMqtl+2SS9S?ck$L^`U z^hP}Pd|JnwD}?*8^+x_39_A(bt?78nM~%NqHIeBLJg%uGZyCCxM%5?Or#;?gRl1Dq zyoF!?c{AnCeK#gH*%qzMtq;c21-EtsuWx-Ci{q{OZQP%8i&PFl;gKSpFt&$O4E5fA zZ>YYbe_xb2k2|Y;;9azsmc{B_>p@)3eHP`*W2j++`Q|0ZY}w|Hm8}$tg6^xFh2B#$ zi>UmD3?^nycKr3Qw~Z!qJ^TsFqKCSK6%*^(1$SL}{~w1HmE)e4+qUj+=%kEB_qU8% z-Sbm2n9i$vKij|Be69K5!)U@JS6jz=WpU{>B%!s+a#m^fiZQ{;)ltok;gJ7qtbqIa zdbf=u)o6efT9jh_^x9IDM#VQ_>Np)WjYp+ejGX|c5uf5>u8M4!$E0GbtCL{H2fb_r(~e)5qTW`YK1{VC58iMXz4jhz&~gXc@bpNv9CHe{;omfr)B; z;CU|J;i{pOrqPc8#aoq;H?=1svlC~O(~bLBa6-Y3nZi*ohmh9H?`kINhVv7J4-JMt z@Eb<`W)-_yh5p|7Q{dId4D)~@CPKY;z~Hp1@TheGa<|>3=hHhvCq}bb zPaR=72;gPYqJP&TLEhh_(@!N`=#)TSU)b;RW5RBoYyP`VUm2E^<)Bc*;8ExzvU+yx z38zi*9gd7EdX!%Glzv+ui<1e%%-a$4U|R?nKtQNj)8x4J2s}ioI`iJ{*gxHz8zL_mzo8<4&F9N&b}bn3GFo zv0cTCHF9Xb}(j~4OdBPHZ`d- z)ScNLOsp7EzJu&%mCxoov9Iz~`jESaW>9Y+9pxTnrablRf%FD;fSuA;ll;!Z0bcWn zt{;0EncRit&qk5fI-iij?W@_|W2H1k-WdVU6|{z9B#bNZj*m-^k=%yGqzZHUL1z7X zM#Jqszdc2n1EznWg@)@IJ$JE_Rc28^`P<#@lOGMpwwUA0sxTT6A5jTxb?7v+$fh8g zDVrLHD1-cqJqfAF72W+GuUuwRv=2zNHQ8bQO>w6L?I*J=wmhgk#X?@6oE))`DZA>a z5GQWGpXYrmyF_2M?Zi)k9k5ZkgOS*t;(T z_JpukXziugk{T;@`c)_})$A*)B&0Xm$9PNb)vR5Jbi0uoykj2mV5`}X(7C0$Al(f_nQeL|_foTe;E>e2P69`C}q@wqKF!X4sBupG1bfwXSe&Ke}jxT+O)L{ghb@`T`y?RW> zmTrhuJ4ygqoNO!ggsS>B;%h{1u5fC4w4udI!~X{NGR`6y07+M?a*>HJRk^SKpJGkPL)148^R$?o8k3IZ2!Sz>Qr?cIMorR9pf}=&<>4|Lij$6J=F~S+|CaT(U z*S?Hl_U!|UnB?}J4;H+V;xmZqn>T#=DY7#?oug+ z-j45giFfN|(&&EknC}F>=d?9qcsNS6qv+q(+FK0TX9f`m@6DyJ@&?``ENeb4{pAH@ z?8*s^)&uc@M&-;RG1sY_J}l~$>h%-Vu1t(n)}I>sGNgoRJSi2+@WS)p&#CI3h~Y!` z`!)Tn8?R4m{??t7hE#l`GWVqXacQmxo%UQP(#BCo%sY5rm(g<`HzI$72C(Q$(urLUaRf^Vf?UG(-K z{#4VSln)+bx_-CJuoLDQ7PkIl1C|(o3O^$TkH99AF5?^`yN2#iQc}Q zqr5DFM5S9f2$5P0CnYcl>59%#11)2wHj=FBgjGs=fIgUz?nBlqBahB1o@~Ixny|3@ zi{-hbR#-oS zAQ*;MIz6it&O2DR`u&RkHFX~X<6Kiic3yXa2dd;yd|m_Bt6^ST>G*czBTZjpQKyv< zc8hf)*t@TlT1l`gR;pgLZGt((ogFlOCdtAMIcH<6_l4~}L$21q-TGfY*z39y&eSX3 zHy9Y}-tftJ!b_Av%Qx2<7NfK^hLv+ta0Pa>4@-P|)3jQJtY8&By4uM zfDTy(irz17x}y`y%Jkjj2iEX_+x}o+&(1JvCy;pvAu3>%!4c`DO4v9tmuENE5U9Mk z3##T-2FrKT^p|umjT*wpsp~s?Vz)|max5p$?4(<0mT9l)2sPsCy$!7ga2Wp;XjzolCTmpP1W#jh|EYznb^bn z8wY%sND^E;Xxr&(7^3iRb$O~VV`Md^lf@c}d25iT6zJF7cxIz0DceC*DkRtK?7<09)3vF4%c{wYx#_}BwJX&$B_M-yki&W<+s&9s}@EP?pnGb*~T z{YU4Y5cHR16n?XgZiiQTz64bw=;T?KySC1MXlWAP-#EJJ&je?6?APr2fakAdq>kT$ z$Mc=3{6`sDjk@%E#bRk}Q)qLOr!HE6*o{Dl5KWmoVFs{9;qdn z@hta%q|7UW?f{Anepc%#MMWufx8C*hQbkb)#+~ceu<89FVXoKM%^2nKROwK#WNp7~ zCI`Q)BfZ?tp&j~p+P+iLpI2P@#7weuP5Yqb_v%w{c54LpS`*GIv6@bMKWxA~^6RS`1OU+K~zhD|Kf!xDQs8~daRtn7J0gg;$Trt9zQ_d*Bu2d2DR z=N7!H%TecOff%X0Yd}!L+f&zblRjv56O^gO^hxB^y$St&5t8kG!?=!XOh=QFuZTj3 zgR3eI)h%kB@Nh}7;HvNzUx?f*-@CUjcbpw-8f$ZWW=*ZI&vKobLFiYLY97}jiR;yy z=9^VkNp&@EAED}M=Dt&w6gB+l`okqVBYML?s@$Pcxhpwz-<(cR;mLzeN z`4#EbE%zU$t~dRKWRx|GIM$UXdeKoY3MAG}(^v&~OBA{nie-+i=$?5zWp@dr-TsU< z^?;uV2_(2vsAA6CpMpH@d|ZNc&rN7 zZkeJ?6Lil{ZhWe~9+o`o6DfA_sOmzAdI$jFj*StuPJKY5D@fh{r2`y4@QJ6NaG%0FLLfLVo3>|lbfNdcIMh(?k6 z#p+=#J-z&4M@-X@_x%hhV`{1G@dEzi2$rJJt}fOH8b68!9jeUIqvq7(M8Dva?1fe; zu*XSHW=yVn2|10+K#y3P*tfx1z2s2+CQECaZPMgp{qxiF6iiOd zb@}ogsPPX1EiPDDSvgy+s;JltZ=(h7q`ZE83kLYGXa%)eoj+m_-?L;-RXytcnY{?b zYHjFMkBk`Mm+YnZ?B@D%z)$eEI=*;x>)VYDH*jn=?Qaf4Kd-oYQ5`yw>J^rq$GxDA z{o&NSE26<|E(=}f0WiJ^UD_nUj27>qb1g!9qi3Wle;)GP?Q$sOY6+r z6rpH!xN!6S_UxfK-(F1WH~6GJ&!h^9t9@w@k2*~eHo9LPad)wNg}1d5*k+R;7mA8@kAU^YFMKJ z#*A-;+C>9-sw|3q#rhR)gW9@tUpX@1xM)=M%gM@qopRCM`618ZpUo1meTyYDXxSq_x}Qfi6JXV0FA>ob$FFh8XtV+lsy%j`OeXB~&;*yolO!p96YVs9SW zX5c-#F;NMPel_Sk5mE1cvXJ8E&z}R<7om>`_w}3X>+hdh^&+~0bZ>UY#hn3@Foxtk zrwdtqkfvF=%f4Qn2g02@e+yAR>hQU>bvN?VsZ*K|$WtB-MQKaaf!o0bH4Y`=n^ME(Xemba&UA>aM-4b0k6`8#OM@ew_bYR452jbQ!R_-+=RR@MntT@~L0=RjtE(e=Y}l;-1SYha?pc7KST^X^6Q*~; z{|!EGoaZy|uW{_)0MLP?r0NVKC5U47eEuVFO~m!<*NOHnu$gIO1Rm7=GdRt3rTP1A z1Q6yph`S1ZYv`Rt#8|+tJtd2jQId&ILA~cPpW-|YgQ@peER;T2jV4T|IBraYvCN}) zm-=g+C%-p?ErY(1h(q4Oj&MKXvKhM%YS)945x=+Iz55Rw^3&asyL)#Q{0BOao(Jn( ztw!Ck#`On;1w0tzw{uTWQVupOql*9#;oWV%mzfgHS~!1FSZMmfO!P^;IIwmEd`yw5 zvF>kKPFDx>g#p@cM{#Dj-rcr|XkyX047ig(w$CgBD=@$Z?m{NdRTN;SE%CJ9`KFI3 z+Nw&>Ybq+KG-I_fiPaFkW*e z-xusd_%<8ri)9;U&lVfq3=;pJ8n1a~&>B3l5S>66D$}ZJ739%Eo8~;^?SibZy5@ZB zcu>ADRhWB*gkx1($p}`)y%>P3#3gKsOV0j38)hK$M~q+sF`-mUgzC7wLPirX03aI*rI3}6)ctwDj4vY-@*hJK*DwbMhb?#o118f054Y$Oy?%d>R z*e8U4hde!EhKM()u#5!fg8PUSxFaji{x6uqTMBK{7nEOwDk~_O=(s z+0@k3W~{Le+=zerqYIUj_=pz7{T5a5u<4q3=VWDN0cm2Hs~&dHgmD4~%>Jg$jIck! zO>%oYI1hah;`0{UMPI~n-AuX-|G*9LVkG9^MPS#PJ7Gj*=h`~BpDv+KYwQ;5gP+fN zewM=qymdZ5pL3|(z`&c%y?;!KHr?G^v@(K_m+?RKh53EMQoS`Qr7++x##d(f-%X@o!6RnIygDfPy*;skz-i*V&86QKU zr$^?*Ja~GDxx~^kmw;c$0}GgO;66TC%r$p;{oT9k_TS5H5pzpFc;9uwcccYq6P;q^ zN5W#xw0;XgK}o77HRp)X`+|eg=C1fT8vndW!gmiJ5as z2qHB=H0T(3dtPRusfoL1!}1?c4)uk?+N%uipOpOVa}0_e=B$-y6%Fb)p-N1oRp7d+ z78w3In%8Yw{hxbpfceiQq}WHC zT!@LugfSrHJE0W4z#J;UU|(WTrQ}c8h-K?c7FO0Zc27tL%N_v8;MaGsrDVaGI8v`d z5yvqe0Fl307h%XDCp-Hp2ZwxI{L7bbLGBSX9Q-Kc>C->GG7WG#nWI}B;rg=VW3-4- zhZ;7{f4CKw1^IK-CR_7|6F|koJ2`I+~tX^%#jBX=XnQUszUs=Cs==` zfDNUkEuQiNqhpxakMFeP&&XbKW#q`{x;SY z9~=9(c^Km03D6KQ<`l5r33WOOKEFpRPYy(`!F)t73_bWQ`{L)>6$zlq-C)0!<3#Mf zX9A1f9=ZqfUBn3uLCY5>Po8vJ{Y+~MIf>}&tkN%g@EDjBk?X*2%@P(+*pAnN3s!Gn z8ID|?B^?Kn@7L#xI3Sd+!U!NyOAi!i7NzBSuTLA)I+@zX6DMmqZw3B2F%j|K=NngM zGsw<#L`gmVG`UJv8#k`zT&pk^=fbq%_3OEiiqx&sH?-W;#^N3(oToLIKP=yKifv^7 z%(ccC-+J~Bn8(bn?fJd%?oMptJbbi7($X?l-W!!A??m%S|1P`N8Kyd#<|&tXJ{C`e(00k;WRddxJr> z9o_8pBTo=s#~Mvg#PSK4<=ZBl2Id>V>ke+>Di~`e4zbQRPDGS^ni}Nh9Kb&;!BWM5Fm%k3PS@CEF^yypRk2EQPlR)pwSdBy zKsqBStex-zLNMiYgE+g1yFsYm{5?{q38P;9`c`Qr#(np`;LARy{b&=XIV6cQa3@9;aGLZDy zP=kriwe@vUOZdYV@(Y>FG9V?-da0<{cKjmIcDZks2Ryf@2vzAhfjeubYQ1fTUm%l zN7$;1u|%)@HmscvJOUMq6yy80l9}>W=6>)27GVzXDA0Fl%)Y+`BoDSHiF^=639k)y z&wotGwsqU~q5eIX$6nHV`8cz$#t~^YQtOP~0$h>P(1;TN#75{0t^ler#J!kmPhto) zy#QunIbaK`L$;`$p<=_Dt{C%QzkZPw-c!EmwUXIeZYFOAYOBQ}$=GrPtcjtfJ=iK@ z8LQ3@C>U?_Z1-mz{1RP)hDgJFK+@JF}NM!4$PgmL5L(o#PFw6SpT}1^RbemEfdS{veCQht~9G;C( zUpkPDbain#2ayFX&h4S>f06t7TgfGth^?%?e=`kefC=yqDzxzeRA`B2K}GP@D}V?u zW&675!F}n{ZQuj}_Ao{K%cFr*EvJgtpi%)woQRc*sNEOnQc3B0;G9=n!*dMB3wpx) zEWmQ~#<1QQC@~b>UyTkNAfDg3{ygO@)PmA|!r+t(ljj@rKc#{yaVf4nyLDilyn+$RJ8pbb^qGx!IF6REi) zW?u)Y6|0nR%zB|t?~j^W7;^5FrDgKG`4&n_>LrFXiELH&!#XhKfkOQ`1$7IVk@fTE z!;q)jhhHfuDHE^@33rj$gyaod8=FE%#PKO93$_(*a~d#;2?T1uw=?mN*&?-Z$KEX2 z(Ed3glYkwIN#ZF2bGX^3mKJ)4nwvd2`a>|U8VQ`s3}!Q0h$C`$pbm@bEU6i@sk9kv z)2nwYSOq+!gCQ|F;+qyZ+=yyA`Tg4L!q7XHDW%IiJRM#@ULyga=s|s41BLejl-_6T zH0V5GsPILCM3HV8snO#<;01`23LbR)}w)D$rN zNQPt}5H=Vkyn42QMMT16_E3y@$18#6zkLboLm#SM5zzVN%QX;!7R7qyAGaYmUqHg8 z7aeue%sl_~3?TUVY`I~;(y3Y2nPJ$Q}gmARH zD=SNfHSsMehl2lmn$e&=aYP@s3e53i0>t86^PC$!s5B9xV8?VfnVi?{pGR zlY?^h3+Pgi, DenseT, SoftmaxActivationT, - DenseT>; + DenseT, + DenseT, + DenseT, + DenseT>; runTestTemplated(tests.at("dense")); } @@ -69,7 +72,10 @@ TEST(TestTemplatedModels, modelOutputMatchesPythonImplementationForDenseWithMath ELuActivationT, DenseT, SoftmaxActivationT, - DenseT>; + DenseT, + DenseT, + DenseT, + DenseT>; runTestTemplated(tests.at("dense")); }