Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BREAKING] Remove parametric modeling code #95

Merged
merged 2 commits into from
Dec 2, 2023
Merged

[BREAKING] Remove parametric modeling code #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
61d209d
Get rid of parametric modeling code
sdatkinson Dec 2, 2023
75af3af
Virtual set condition array
sdatkinson Dec 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 22 additions & 24 deletions NAM/convnet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,24 @@
#include <unordered_set>

#include "dsp.h"
#include "json.hpp"
#include "util.h"
#include "convnet.h"

nam::convnet::BatchNorm::BatchNorm(const int dim, std::vector<float>::iterator& params)
nam::convnet::BatchNorm::BatchNorm(const int dim, std::vector<float>::iterator& weights)
{
// Extract from param buffer
Eigen::VectorXf running_mean(dim);
Eigen::VectorXf running_var(dim);
Eigen::VectorXf _weight(dim);
Eigen::VectorXf _bias(dim);
for (int i = 0; i < dim; i++)
running_mean(i) = *(params++);
running_mean(i) = *(weights++);
for (int i = 0; i < dim; i++)
running_var(i) = *(params++);
running_var(i) = *(weights++);
for (int i = 0; i < dim; i++)
_weight(i) = *(params++);
_weight(i) = *(weights++);
for (int i = 0; i < dim; i++)
_bias(i) = *(params++);
float eps = *(params++);
_bias(i) = *(weights++);
float eps = *(weights++);

// Convert to scale & loc
this->scale.resize(dim);
Expand All @@ -48,15 +46,15 @@ void nam::convnet::BatchNorm::process_(Eigen::MatrixXf& x, const long i_start, c
}
}

void nam::convnet::ConvNetBlock::set_params_(const int in_channels, const int out_channels, const int _dilation,
const bool batchnorm, const std::string activation,
std::vector<float>::iterator& params)
void nam::convnet::ConvNetBlock::set_weights_(const int in_channels, const int out_channels, const int _dilation,
const bool batchnorm, const std::string activation,
std::vector<float>::iterator& weights)
{
this->_batchnorm = batchnorm;
// HACK 2 kernel
this->conv.set_size_and_params_(in_channels, out_channels, 2, _dilation, !batchnorm, params);
this->conv.set_size_and_weights_(in_channels, out_channels, 2, _dilation, !batchnorm, weights);
if (this->_batchnorm)
this->batchnorm = BatchNorm(out_channels, params);
this->batchnorm = BatchNorm(out_channels, weights);
this->activation = activations::Activation::get_activation(activation);
}

Expand All @@ -76,12 +74,12 @@ long nam::convnet::ConvNetBlock::get_out_channels() const
return this->conv.get_out_channels();
}

nam::convnet::_Head::_Head(const int channels, std::vector<float>::iterator& params)
nam::convnet::_Head::_Head(const int channels, std::vector<float>::iterator& weights)
{
this->_weight.resize(channels);
for (int i = 0; i < channels; i++)
this->_weight[i] = *(params++);
this->_bias = *(params++);
this->_weight[i] = *(weights++);
this->_bias = *(weights++);
}

void nam::convnet::_Head::process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start,
Expand All @@ -94,22 +92,22 @@ void nam::convnet::_Head::process_(const Eigen::MatrixXf& input, Eigen::VectorXf
}

nam::convnet::ConvNet::ConvNet(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const std::string activation, std::vector<float>& params,
const std::string activation, std::vector<float>& weights,
const double expected_sample_rate)
: Buffer(*std::max_element(dilations.begin(), dilations.end()), expected_sample_rate)
{
this->_verify_params(channels, dilations, batchnorm, params.size());
this->_verify_weights(channels, dilations, batchnorm, weights.size());
this->_blocks.resize(dilations.size());
std::vector<float>::iterator it = params.begin();
std::vector<float>::iterator it = weights.begin();
for (size_t i = 0; i < dilations.size(); i++)
this->_blocks[i].set_params_(i == 0 ? 1 : channels, channels, dilations[i], batchnorm, activation, it);
this->_blocks[i].set_weights_(i == 0 ? 1 : channels, channels, dilations[i], batchnorm, activation, it);
this->_block_vals.resize(this->_blocks.size() + 1);
for (auto& matrix : this->_block_vals)
matrix.setZero();
std::fill(this->_input_buffer.begin(), this->_input_buffer.end(), 0.0f);
this->_head = _Head(channels, it);
if (it != params.end())
throw std::runtime_error("Didn't touch all the params when initializing ConvNet");
if (it != weights.end())
throw std::runtime_error("Didn't touch all the weights when initializing ConvNet");

_prewarm_samples = 1;
for (size_t i = 0; i < dilations.size(); i++)
Expand All @@ -136,8 +134,8 @@ void nam::convnet::ConvNet::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const
output[s] = this->_head_output(s);
}

void nam::convnet::ConvNet::_verify_params(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_params)
void nam::convnet::ConvNet::_verify_weights(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_weights)
{
// TODO
}
Expand Down
14 changes: 7 additions & 7 deletions NAM/convnet.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class BatchNorm
{
public:
BatchNorm(){};
BatchNorm(const int dim, std::vector<float>::iterator& params);
BatchNorm(const int dim, std::vector<float>::iterator& weights);
void process_(Eigen::MatrixXf& input, const long i_start, const long i_end) const;

private:
Expand All @@ -40,8 +40,8 @@ class ConvNetBlock
{
public:
ConvNetBlock(){};
void set_params_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm,
const std::string activation, std::vector<float>::iterator& params);
void set_weights_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm,
const std::string activation, std::vector<float>::iterator& weights);
void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end) const;
long get_out_channels() const;
Conv1D conv;
Expand All @@ -56,7 +56,7 @@ class _Head
{
public:
_Head(){};
_Head(const int channels, std::vector<float>::iterator& params);
_Head(const int channels, std::vector<float>::iterator& weights);
void process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start, const long i_end) const;

private:
Expand All @@ -68,16 +68,16 @@ class ConvNet : public Buffer
{
public:
ConvNet(const int channels, const std::vector<int>& dilations, const bool batchnorm, const std::string activation,
std::vector<float>& params, const double expected_sample_rate = -1.0);
std::vector<float>& weights, const double expected_sample_rate = -1.0);
~ConvNet() = default;

protected:
std::vector<ConvNetBlock> _blocks;
std::vector<Eigen::MatrixXf> _block_vals;
Eigen::VectorXf _head_output;
_Head _head;
void _verify_params(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_params);
void _verify_weights(const int channels, const std::vector<int>& dilations, const bool batchnorm,
const size_t actual_weights);
void _update_buffers_(NAM_SAMPLE* input, const int num_frames) override;
void _rewind_buffers_() override;

Expand Down
51 changes: 17 additions & 34 deletions NAM/dsp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
#include <unordered_set>

#include "dsp.h"
#include "json.hpp"
#include "util.h"

#define tanh_impl_ std::tanh
// #define tanh_impl_ fast_tanh_
Expand Down Expand Up @@ -62,21 +60,6 @@ void nam::DSP::SetLoudness(const double loudness)

void nam::DSP::finalize_(const int num_frames) {}

void nam::DSP::_get_params_(const std::unordered_map<std::string, double>& input_params)
{
this->_stale_params = false;
for (auto it = input_params.begin(); it != input_params.end(); ++it)
{
const std::string key = util::lowercase(it->first);
const double value = it->second;
if (this->_params.find(key) == this->_params.end()) // Not contained
this->_stale_params = true;
else if (this->_params[key] != value) // Contained but new value
this->_stale_params = true;
this->_params[key] = value;
}
}

// Buffer =====================================================================

nam::Buffer::Buffer(const int receptive_field, const double expected_sample_rate)
Expand Down Expand Up @@ -153,20 +136,20 @@ void nam::Buffer::finalize_(const int num_frames)

// Linear =====================================================================

nam::Linear::Linear(const int receptive_field, const bool _bias, const std::vector<float>& params,
nam::Linear::Linear(const int receptive_field, const bool _bias, const std::vector<float>& weights,
const double expected_sample_rate)
: nam::Buffer(receptive_field, expected_sample_rate)
{
if ((int)params.size() != (receptive_field + (_bias ? 1 : 0)))
if ((int)weights.size() != (receptive_field + (_bias ? 1 : 0)))
throw std::runtime_error(
"Params vector does not match expected size based "
"on architecture parameters");

this->_weight.resize(this->_receptive_field);
// Pass in in reverse order so that dot products work out of the box.
for (int i = 0; i < this->_receptive_field; i++)
this->_weight(i) = params[receptive_field - 1 - i];
this->_bias = _bias ? params[receptive_field] : (float)0.0;
this->_weight(i) = weights[receptive_field - 1 - i];
this->_bias = _bias ? weights[receptive_field] : (float)0.0;
}

void nam::Linear::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames)
Expand All @@ -184,7 +167,7 @@ void nam::Linear::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_f

// NN modules =================================================================

void nam::Conv1D::set_params_(std::vector<float>::iterator& params)
void nam::Conv1D::set_weights_(std::vector<float>::iterator& weights)
{
if (this->_weight.size() > 0)
{
Expand All @@ -194,10 +177,10 @@ void nam::Conv1D::set_params_(std::vector<float>::iterator& params)
for (auto i = 0; i < out_channels; i++)
for (auto j = 0; j < in_channels; j++)
for (size_t k = 0; k < this->_weight.size(); k++)
this->_weight[k](i, j) = *(params++);
this->_weight[k](i, j) = *(weights++);
}
for (long i = 0; i < this->_bias.size(); i++)
this->_bias(i) = *(params++);
this->_bias(i) = *(weights++);
}

void nam::Conv1D::set_size_(const int in_channels, const int out_channels, const int kernel_size, const bool do_bias,
Expand All @@ -214,11 +197,11 @@ void nam::Conv1D::set_size_(const int in_channels, const int out_channels, const
this->_dilation = _dilation;
}

void nam::Conv1D::set_size_and_params_(const int in_channels, const int out_channels, const int kernel_size,
const int _dilation, const bool do_bias, std::vector<float>::iterator& params)
void nam::Conv1D::set_size_and_weights_(const int in_channels, const int out_channels, const int kernel_size,
const int _dilation, const bool do_bias, std::vector<float>::iterator& weights)
{
this->set_size_(in_channels, out_channels, kernel_size, do_bias, _dilation);
this->set_params_(params);
this->set_weights_(weights);
}

void nam::Conv1D::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long ncols,
Expand All @@ -237,12 +220,12 @@ void nam::Conv1D::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output
output.middleCols(j_start, ncols).colwise() += this->_bias;
}

long nam::Conv1D::get_num_params() const
long nam::Conv1D::get_num_weights() const
{
long num_params = this->_bias.size();
long num_weights = this->_bias.size();
for (size_t i = 0; i < this->_weight.size(); i++)
num_params += this->_weight[i].size();
return num_params;
num_weights += this->_weight[i].size();
return num_weights;
}

nam::Conv1x1::Conv1x1(const int in_channels, const int out_channels, const bool _bias)
Expand All @@ -253,14 +236,14 @@ nam::Conv1x1::Conv1x1(const int in_channels, const int out_channels, const bool
this->_bias.resize(out_channels);
}

void nam::Conv1x1::set_params_(std::vector<float>::iterator& params)
void nam::Conv1x1::set_weights_(std::vector<float>::iterator& weights)
{
for (int i = 0; i < this->_weight.rows(); i++)
for (int j = 0; j < this->_weight.cols(); j++)
this->_weight(i, j) = *(params++);
this->_weight(i, j) = *(weights++);
if (this->_do_bias)
for (int i = 0; i < this->_bias.size(); i++)
this->_bias(i) = *(params++);
this->_bias(i) = *(weights++);
}

Eigen::MatrixXf nam::Conv1x1::process(const Eigen::MatrixXf& input) const
Expand Down
41 changes: 9 additions & 32 deletions NAM/dsp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,6 @@ enum EArchitectures
kNumModels
};

// Class for providing params from the plugin to the DSP module
// For now, we'll work with doubles. Later, we'll add other types.
class DSPParam
{
public:
const char* name;
const double val;
};
// And the params shall be provided as a std::vector<DSPParam>.

class DSP
{
public:
Expand All @@ -65,9 +55,6 @@ class DSP
// Anything to take care of before next buffer comes in.
// For example:
// * Move the buffer index forward
// * Does NOT say that params aren't stale; that's the job of the routine
// that actually uses them, which varies depends on the particulars of the
// DSP subclass implementation.
virtual void finalize_(const int num_frames);
// Expected sample rate, in Hz.
// TODO throw if it doesn't know.
Expand All @@ -88,18 +75,8 @@ class DSP
double mLoudness = 0.0;
// What sample rate does the model expect?
double mExpectedSampleRate;
// Parameters (aka "knobs")
std::unordered_map<std::string, double> _params;
// If the params have changed since the last buffer was processed:
bool _stale_params = true;
// How many samples should be processed during "pre-warming"
int _prewarm_samples = 0;

// Methods

// Copy the parameters to the DSP module.
// If anything has changed, then set this->_stale_params to true.
// (TODO use "listener" approach)
void _get_params_(const std::unordered_map<std::string, double>& input_params);
};

// Class where an input buffer is kept so that long-time effects can be
Expand Down Expand Up @@ -132,7 +109,7 @@ class Buffer : public DSP
class Linear : public Buffer
{
public:
Linear(const int receptive_field, const bool _bias, const std::vector<float>& params,
Linear(const int receptive_field, const bool _bias, const std::vector<float>& weights,
const double expected_sample_rate = -1.0);
void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) override;

Expand All @@ -147,19 +124,19 @@ class Conv1D
{
public:
Conv1D() { this->_dilation = 1; };
void set_params_(std::vector<float>::iterator& params);
void set_weights_(std::vector<float>::iterator& weights);
void set_size_(const int in_channels, const int out_channels, const int kernel_size, const bool do_bias,
const int _dilation);
void set_size_and_params_(const int in_channels, const int out_channels, const int kernel_size, const int _dilation,
const bool do_bias, std::vector<float>::iterator& params);
void set_size_and_weights_(const int in_channels, const int out_channels, const int kernel_size, const int _dilation,
const bool do_bias, std::vector<float>::iterator& weights);
// Process from input to output
// Rightmost indices of input go from i_start to i_end,
// Indices on output for from j_start (to j_start + i_end - i_start)
void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end,
const long j_start) const;
long get_in_channels() const { return this->_weight.size() > 0 ? this->_weight[0].cols() : 0; };
long get_kernel_size() const { return this->_weight.size(); };
long get_num_params() const;
long get_num_weights() const;
long get_out_channels() const { return this->_weight.size() > 0 ? this->_weight[0].rows() : 0; };
int get_dilation() const { return this->_dilation; };

Expand All @@ -176,7 +153,7 @@ class Conv1x1
{
public:
Conv1x1(const int in_channels, const int out_channels, const bool _bias);
void set_params_(std::vector<float>::iterator& params);
void set_weights_(std::vector<float>::iterator& weights);
// :param input: (N,Cin) or (Cin,)
// :return: (N,Cout) or (Cout,), respectively
Eigen::MatrixXf process(const Eigen::MatrixXf& input) const;
Expand All @@ -203,7 +180,7 @@ class Conv1x1
// * "WaveNet"
// :param config:
// :param metadata:
// :param params: The model parameters ("weights")
// :param weights: The model weights
// :param expected_sample_rate: Most NAM models implicitly assume that data will be provided to them at some sample
// rate. This captures it for other components interfacing with the model to understand its needs. Use -1.0 for "I
// don't know".
Expand All @@ -213,7 +190,7 @@ struct dspData
std::string architecture;
nlohmann::json config;
nlohmann::json metadata;
std::vector<float> params;
std::vector<float> weights;
double expected_sample_rate;
};

Expand Down
Loading