Skip to content

Commit

Permalink
[BREAKING] Remove parametric modeling code (#95)
Browse files Browse the repository at this point in the history 
* Get rid of parametric modeling code

* Virtual set condition array
  • Loading branch information
@sdatkinson
sdatkinson authored Dec 2, 2023
1 parent 4772bb2 commit 3d4a71f
Show file tree
Hide file tree
Showing 9 changed files with 120 additions and 225 deletions.
46 changes: 22 additions & 24 deletions NAM/convnet.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,24 @@
#include <unordered_set>

#include "dsp.h"
#include "json.hpp"
#include "util.h"
#include "convnet.h"

nam::convnet::BatchNorm::BatchNorm(const int dim, std::vector<float>::iterator& params)
nam::convnet::BatchNorm::BatchNorm(const int dim, std::vector<float>::iterator& weights)
{
// Extract from param buffer
Eigen::VectorXf running_mean(dim);
Eigen::VectorXf running_var(dim);
Eigen::VectorXf _weight(dim);
Eigen::VectorXf _bias(dim);
for (int i = 0; i < dim; i++)
running_mean(i) = *(params++);
running_mean(i) = *(weights++);
for (int i = 0; i < dim; i++)
running_var(i) = *(params++);
running_var(i) = *(weights++);
for (int i = 0; i < dim; i++)
_weight(i) = *(params++);
_weight(i) = *(weights++);
for (int i = 0; i < dim; i++)
_bias(i) = *(params++);
float eps = *(params++);
_bias(i) = *(weights++);
float eps = *(weights++);

// Convert to scale & loc
this->scale.resize(dim);
Expand All @@ -48,15 +46,15 @@ void nam::convnet::BatchNorm::process_(Eigen::MatrixXf& x, const long i_start, c
}
}

void nam::convnet::ConvNetBlock::set_params_(const int in_channels, const int out_channels, const int _dilation,
const bool batchnorm, const std::string activation,
std::vector<float>::iterator& params)
void nam::convnet::ConvNetBlock::set_weights_(const int in_channels, const int out_channels, const int _dilation,
const bool batchnorm, const std::string activation,
std::vector<float>::iterator& weights)
{
this->_batchnorm = batchnorm;
// HACK 2 kernel
this->conv.set_size_and_params_(in_channels, out_channels, 2, _dilation, !batchnorm, params);
this->conv.set_size_and_weights_(in_channels, out_channels, 2, _dilation, !batchnorm, weights);
if (this->_batchnorm)
this->batchnorm = BatchNorm(out_channels, params);
this->batchnorm = BatchNorm(out_channels, weights);
this->activation = activations::Activation::get_activation(activation);
}

Expand All @@ -76,12 +74,12 @@ long nam::convnet::ConvNetBlock::get_out_channels() const
return this->conv.get_out_channels();
}

nam::convnet::_Head::_Head(const int channels, std::vector<float>::iterator& params)
nam::convnet::_Head::_Head(const int channels, std::vector<float>::iterator& weights)
{
this->_weight.resize(channels);
for (int i = 0; i < channels; i++)
this->_weight[i] = *(params++);
this->_bias = *(params++);
this->_weight[i] = *(weights++);
this->_bias = *(weights++);
}

void nam::convnet::_Head::process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start,
Expand All @@ -94,22 +92,22 @@ void nam::convnet::_Head::process_(const Eigen::MatrixXf& input, Eigen::VectorXf
}

nam::convnet::ConvNet::ConvNet(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const std::string activation, std::vector<float>& params,
const std::string activation, std::vector<float>& weights,
const double expected_sample_rate)
: Buffer(*std::max_element(dilations.begin(), dilations.end()), expected_sample_rate)
{
this->_verify_params(channels, dilations, batchnorm, params.size());
this->_verify_weights(channels, dilations, batchnorm, weights.size());
this->_blocks.resize(dilations.size());
std::vector<float>::iterator it = params.begin();
std::vector<float>::iterator it = weights.begin();
for (size_t i = 0; i < dilations.size(); i++)
this->_blocks[i].set_params_(i == 0 ? 1 : channels, channels, dilations[i], batchnorm, activation, it);
this->_blocks[i].set_weights_(i == 0 ? 1 : channels, channels, dilations[i], batchnorm, activation, it);
this->_block_vals.resize(this->_blocks.size() + 1);
for (auto& matrix : this->_block_vals)
matrix.setZero();
std::fill(this->_input_buffer.begin(), this->_input_buffer.end(), 0.0f);
this->_head = _Head(channels, it);
if (it != params.end())
throw std::runtime_error("Didn't touch all the params when initializing ConvNet");
if (it != weights.end())
throw std::runtime_error("Didn't touch all the weights when initializing ConvNet");

_prewarm_samples = 1;
for (size_t i = 0; i < dilations.size(); i++)
Expand All @@ -136,8 +134,8 @@ void nam::convnet::ConvNet::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const
output[s] = this->_head_output(s);
}

void nam::convnet::ConvNet::_verify_params(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_params)
void nam::convnet::ConvNet::_verify_weights(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_weights)
{
// TODO
}
Expand Down
14 changes: 7 additions & 7 deletions NAM/convnet.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class BatchNorm
{
public:
BatchNorm(){};
BatchNorm(const int dim, std::vector<float>::iterator& params);
BatchNorm(const int dim, std::vector<float>::iterator& weights);
void process_(Eigen::MatrixXf& input, const long i_start, const long i_end) const;

private:
Expand All @@ -40,8 +40,8 @@ class ConvNetBlock
{
public:
ConvNetBlock(){};
void set_params_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm,
const std::string activation, std::vector<float>::iterator& params);
void set_weights_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm,
const std::string activation, std::vector<float>::iterator& weights);
void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end) const;
long get_out_channels() const;
Conv1D conv;
Expand All @@ -56,7 +56,7 @@ class _Head
{
public:
_Head(){};
_Head(const int channels, std::vector<float>::iterator& params);
_Head(const int channels, std::vector<float>::iterator& weights);
void process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start, const long i_end) const;

private:
Expand All @@ -68,16 +68,16 @@ class ConvNet : public Buffer
{
public:
ConvNet(const int channels, const std::vector<int>& dilations, const bool batchnorm, const std::string activation,
std::vector<float>& params, const double expected_sample_rate = -1.0);
std::vector<float>& weights, const double expected_sample_rate = -1.0);
~ConvNet() = default;

protected:
std::vector<ConvNetBlock> _blocks;
std::vector<Eigen::MatrixXf> _block_vals;
Eigen::VectorXf _head_output;
_Head _head;
void _verify_params(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_params);
void _verify_weights(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_weights);
void _update_buffers_(NAM_SAMPLE* input, const int num_frames) override;
void _rewind_buffers_() override;

Expand Down
51 changes: 17 additions & 34 deletions NAM/dsp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
#include <unordered_set>

#include "dsp.h"
#include "json.hpp"
#include "util.h"

#define tanh_impl_ std::tanh
// #define tanh_impl_ fast_tanh_
Expand Down Expand Up @@ -62,21 +60,6 @@ void nam::DSP::SetLoudness(const double loudness)

void nam::DSP::finalize_(const int num_frames) {}

void nam::DSP::_get_params_(const std::unordered_map<std::string, double>& input_params)
{
this->_stale_params = false;
for (auto it = input_params.begin(); it != input_params.end(); ++it)
{
const std::string key = util::lowercase(it->first);
const double value = it->second;
if (this->_params.find(key) == this->_params.end()) // Not contained
this->_stale_params = true;
else if (this->_params[key] != value) // Contained but new value
this->_stale_params = true;
this->_params[key] = value;
}
}

// Buffer =====================================================================

nam::Buffer::Buffer(const int receptive_field, const double expected_sample_rate)
Expand Down Expand Up @@ -153,20 +136,20 @@ void nam::Buffer::finalize_(const int num_frames)

// Linear =====================================================================

nam::Linear::Linear(const int receptive_field, const bool _bias, const std::vector<float>& params,
nam::Linear::Linear(const int receptive_field, const bool _bias, const std::vector<float>& weights,
const double expected_sample_rate)
: nam::Buffer(receptive_field, expected_sample_rate)
{
if ((int)params.size() != (receptive_field + (_bias ? 1 : 0)))
if ((int)weights.size() != (receptive_field + (_bias ? 1 : 0)))
throw std::runtime_error(
"Params vector does not match expected size based "
"on architecture parameters");

this->_weight.resize(this->_receptive_field);
// Pass in in reverse order so that dot products work out of the box.
for (int i = 0; i < this->_receptive_field; i++)
this->_weight(i) = params[receptive_field - 1 - i];
this->_bias = _bias ? params[receptive_field] : (float)0.0;
this->_weight(i) = weights[receptive_field - 1 - i];
this->_bias = _bias ? weights[receptive_field] : (float)0.0;
}

void nam::Linear::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames)
Expand All @@ -184,7 +167,7 @@ void nam::Linear::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_f

// NN modules =================================================================

void nam::Conv1D::set_params_(std::vector<float>::iterator& params)
void nam::Conv1D::set_weights_(std::vector<float>::iterator& weights)
{
if (this->_weight.size() > 0)
{
Expand All @@ -194,10 +177,10 @@ void nam::Conv1D::set_params_(std::vector<float>::iterator& params)
for (auto i = 0; i < out_channels; i++)
for (auto j = 0; j < in_channels; j++)
for (size_t k = 0; k < this->_weight.size(); k++)
this->_weight[k](i, j) = *(params++);
this->_weight[k](i, j) = *(weights++);
}
for (long i = 0; i < this->_bias.size(); i++)
this->_bias(i) = *(params++);
this->_bias(i) = *(weights++);
}

void nam::Conv1D::set_size_(const int in_channels, const int out_channels, const int kernel_size, const bool do_bias,
Expand All @@ -214,11 +197,11 @@ void nam::Conv1D::set_size_(const int in_channels, const int out_channels, const
this->_dilation = _dilation;
}

void nam::Conv1D::set_size_and_params_(const int in_channels, const int out_channels, const int kernel_size,
const int _dilation, const bool do_bias, std::vector<float>::iterator& params)
void nam::Conv1D::set_size_and_weights_(const int in_channels, const int out_channels, const int kernel_size,
const int _dilation, const bool do_bias, std::vector<float>::iterator& weights)
{
this->set_size_(in_channels, out_channels, kernel_size, do_bias, _dilation);
this->set_params_(params);
this->set_weights_(weights);
}

void nam::Conv1D::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long ncols,
Expand All @@ -237,12 +220,12 @@ void nam::Conv1D::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output
output.middleCols(j_start, ncols).colwise() += this->_bias;
}

long nam::Conv1D::get_num_params() const
long nam::Conv1D::get_num_weights() const
{
long num_params = this->_bias.size();
long num_weights = this->_bias.size();
for (size_t i = 0; i < this->_weight.size(); i++)
num_params += this->_weight[i].size();
return num_params;
num_weights += this->_weight[i].size();
return num_weights;
}

nam::Conv1x1::Conv1x1(const int in_channels, const int out_channels, const bool _bias)
Expand All @@ -253,14 +236,14 @@ nam::Conv1x1::Conv1x1(const int in_channels, const int out_channels, const bool
this->_bias.resize(out_channels);
}

void nam::Conv1x1::set_params_(std::vector<float>::iterator& params)
void nam::Conv1x1::set_weights_(std::vector<float>::iterator& weights)
{
for (int i = 0; i < this->_weight.rows(); i++)
for (int j = 0; j < this->_weight.cols(); j++)
this->_weight(i, j) = *(params++);
this->_weight(i, j) = *(weights++);
if (this->_do_bias)
for (int i = 0; i < this->_bias.size(); i++)
this->_bias(i) = *(params++);
this->_bias(i) = *(weights++);
}

Eigen::MatrixXf nam::Conv1x1::process(const Eigen::MatrixXf& input) const
Expand Down
41 changes: 9 additions & 32 deletions NAM/dsp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,6 @@ enum EArchitectures
kNumModels
};

// Class for providing params from the plugin to the DSP module
// For now, we'll work with doubles. Later, we'll add other types.
class DSPParam
{
public:
const char* name;
const double val;
};
// And the params shall be provided as a std::vector<DSPParam>.

class DSP
{
public:
Expand All @@ -65,9 +55,6 @@ class DSP
// Anything to take care of before next buffer comes in.
// For example:
// * Move the buffer index forward
// * Does NOT say that params aren't stale; that's the job of the routine
// that actually uses them, which varies depends on the particulars of the
// DSP subclass implementation.
virtual void finalize_(const int num_frames);
// Expected sample rate, in Hz.
// TODO throw if it doesn't know.
Expand All @@ -88,18 +75,8 @@ class DSP
double mLoudness = 0.0;
// What sample rate does the model expect?
double mExpectedSampleRate;
// Parameters (aka "knobs")
std::unordered_map<std::string, double> _params;
// If the params have changed since the last buffer was processed:
bool _stale_params = true;
// How many samples should be processed during "pre-warming"
int _prewarm_samples = 0;

// Methods

// Copy the parameters to the DSP module.
// If anything has changed, then set this->_stale_params to true.
// (TODO use "listener" approach)
void _get_params_(const std::unordered_map<std::string, double>& input_params);
};

// Class where an input buffer is kept so that long-time effects can be
Expand Down Expand Up @@ -132,7 +109,7 @@ class Buffer : public DSP
class Linear : public Buffer
{
public:
Linear(const int receptive_field, const bool _bias, const std::vector<float>& params,
Linear(const int receptive_field, const bool _bias, const std::vector<float>& weights,
const double expected_sample_rate = -1.0);
void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) override;

Expand All @@ -147,19 +124,19 @@ class Conv1D
{
public:
Conv1D() { this->_dilation = 1; };
void set_params_(std::vector<float>::iterator& params);
void set_weights_(std::vector<float>::iterator& weights);
void set_size_(const int in_channels, const int out_channels, const int kernel_size, const bool do_bias,
const int _dilation);
void set_size_and_params_(const int in_channels, const int out_channels, const int kernel_size, const int _dilation,
const bool do_bias, std::vector<float>::iterator& params);
void set_size_and_weights_(const int in_channels, const int out_channels, const int kernel_size, const int _dilation,
const bool do_bias, std::vector<float>::iterator& weights);
// Process from input to output
// Rightmost indices of input go from i_start to i_end,
// Indices on output for from j_start (to j_start + i_end - i_start)
void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end,
const long j_start) const;
long get_in_channels() const { return this->_weight.size() > 0 ? this->_weight[0].cols() : 0; };
long get_kernel_size() const { return this->_weight.size(); };
long get_num_params() const;
long get_num_weights() const;
long get_out_channels() const { return this->_weight.size() > 0 ? this->_weight[0].rows() : 0; };
int get_dilation() const { return this->_dilation; };

Expand All @@ -176,7 +153,7 @@ class Conv1x1
{
public:
Conv1x1(const int in_channels, const int out_channels, const bool _bias);
void set_params_(std::vector<float>::iterator& params);
void set_weights_(std::vector<float>::iterator& weights);
// :param input: (N,Cin) or (Cin,)
// :return: (N,Cout) or (Cout,), respectively
Eigen::MatrixXf process(const Eigen::MatrixXf& input) const;
Expand All @@ -203,7 +180,7 @@ class Conv1x1
// * "WaveNet"
// :param config:
// :param metadata:
// :param params: The model parameters ("weights")
// :param weights: The model weights
// :param expected_sample_rate: Most NAM models implicitly assume that data will be provided to them at some sample
// rate. This captures it for other components interfacing with the model to understand its needs. Use -1.0 for "I
// don't know".
Expand All @@ -213,7 +190,7 @@ struct dspData
std::string architecture;
nlohmann::json config;
nlohmann::json metadata;
std::vector<float> params;
std::vector<float> weights;
double expected_sample_rate;
};

Expand Down
Loading

0 comments on commit 3d4a71f

Please sign in to comment.