Skip to content

Commit

Permalink
using one bias per output channels intead of one per input channel pe…
Browse files Browse the repository at this point in the history 
…r output channels. (wip)
  • Loading branch information
@JulioJerez
JulioJerez committed Oct 28, 2023
1 parent cbedbed commit 5c8ded0
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 56 deletions.
12 changes: 5 additions & 7 deletions newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,16 +517,14 @@ static void MnistTrainingSet()
if (trainingLabels && trainingDigits)
{
ndBrain brain;

ndInt32 neuronsPerLayers = 64;
ndFixSizeArray<ndBrainLayer*, 16> layers;

#ifdef D_USE_CONVOLUTIONAL_LAYERS
ndInt32 height = 28;
ndInt32 width = trainingDigits->GetColumns() / height;
ndAssert((height * width) == trainingDigits->GetColumns());

layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 10));
layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 16));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv0 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv0->GetOutputWidth(), conv0->GetOutputHeight(), conv0->GetOutputChannels()));
Expand All @@ -538,18 +536,18 @@ static void MnistTrainingSet()
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv1->GetOutputWidth(), conv1->GetOutputHeight(), conv1->GetOutputChannels()));
const ndBrainLayerConvolutionalMaxPooling* const pooling1 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

layers.PushBack(new ndBrainLayerConvolutional(pooling1->GetOutputWidth(), pooling1->GetOutputHeight(), pooling1->GetOutputChannels(), 3, 3));
layers.PushBack(new ndBrainLayerConvolutional(pooling1->GetOutputWidth(), pooling1->GetOutputHeight(), pooling1->GetOutputChannels(), 3, 32));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv2 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv2->GetOutputWidth(), conv2->GetOutputHeight(), conv2->GetOutputChannels()));

//const ndBrainLayerConvolutionalMaxPooling* const pooling2 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns()));
layers.PushBack(new ndBrainLayerCategoricalSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize()));

#else


ndInt32 neuronsPerLayers = 64;
layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), neuronsPerLayers));
layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));

Expand Down Expand Up @@ -613,7 +611,7 @@ void ndTestDeedBrian()
// //xxx.PushBack(ndGaussianRandom(0.0f, 0.1f));
// xxx.PushBack(1.0f);
//}
//xxx.GaussianNormalize();/
//xxx.GaussianNormalize();

//ThreeLayersTwoInputsTwoOutputs();
//MnistTrainingSet();
Expand Down
31 changes: 10 additions & 21 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,8 @@ ndBrainLayerConvolutional::ndBrainLayerConvolutional(ndInt32 inputWidth, ndInt32
m_outputWidth = m_inputWidth - m_kernelSize + 1;
m_outputHeight = m_inputHeight - m_kernelSize + 1;

ndInt32 paddedKernelSize = ((m_kernelSize * m_kernelSize) + 8) & -8;

m_bias.SetCount(m_numberOfKernels * m_inputDepth);
m_kernels.SetCount(m_numberOfKernels * m_inputDepth * paddedKernelSize);
m_bias.SetCount(m_numberOfKernels);
m_kernels.SetCount(m_numberOfKernels * m_inputDepth * m_kernelSize * m_kernelSize);

m_bias.Set(ndBrainFloat(0.0f));
m_kernels.Set(ndBrainFloat(0.0f));
Expand Down Expand Up @@ -135,12 +133,7 @@ void ndBrainLayerConvolutional::InitGaussianBias(ndBrainFloat variance)

void ndBrainLayerConvolutional::InitGaussianWeights(ndBrainFloat variance)
{
const ndInt32 paddedKernelSize = ((m_kernelSize * m_kernelSize) + 8) & -8;
for (ndInt32 i = 0; i < m_kernels.GetCount(); i += paddedKernelSize)
{
ndBrainMemVector kernel (&m_kernels[i], m_kernelSize * m_kernelSize);
kernel.InitGaussianWeights(variance);
}
m_kernels.InitGaussianWeights(variance);
}

void ndBrainLayerConvolutional::InitWeights(ndBrainFloat weighVariance, ndBrainFloat biasVariance)
Expand Down Expand Up @@ -253,41 +246,37 @@ void ndBrainLayerConvolutional::MakePrediction(const ndBrainVector& input, ndBra
//output.Add(m_bias);
ndAssert(input.GetCount() == GetInputSize());

const ndInt32 paddedKernelSize = ((m_kernelSize * m_kernelSize) + 8) & -8;
for (ndInt32 i = 0; i < m_numberOfKernels; ++i)
{
ndInt32 outStart = i * m_outputWidth * m_outputHeight;
ndInt32 kernelStart = i * paddedKernelSize * m_inputDepth;
ndInt32 kernelStart = i * m_inputDepth * m_kernelSize * m_kernelSize;

ndBrainMemVector out(&output[outStart], m_outputWidth * m_outputHeight);
const ndBrainMemVector kernels(&m_kernels[kernelStart], paddedKernelSize * m_inputDepth);
//const ndBrainMemVector bias(&m_bias[i * m_inputDepth], m_numberOfKernels * m_inputDepth);
const ndBrainMemVector bias(&m_bias[i * m_inputDepth], m_inputDepth);
PredictionOutputChannel(input, bias, kernels, out);
const ndBrainMemVector kernels(&m_kernels[kernelStart], m_inputDepth * m_kernelSize * m_kernelSize);
PredictionOutputChannel(input, kernels, m_bias[i], out);
}
}

void ndBrainLayerConvolutional::PredictionOutputChannel(const ndBrainVector& input, const ndBrainVector& bias, const ndBrainVector& kernels, ndBrainVector& output) const
void ndBrainLayerConvolutional::PredictionOutputChannel(const ndBrainVector& input, const ndBrainVector& kernels, ndBrainFloat bias, ndBrainVector& output) const
{
ndInt32 inputOffset = 0;
ndInt32 outputOffset = 0;
const ndInt32 inputSize = m_inputWidth * m_kernelSize;
const ndInt32 kernelSize = m_kernelSize * m_kernelSize;
const ndInt32 paddedKernelSize = ((m_kernelSize * m_kernelSize) + 8) & -8;
for (ndInt32 j = 0; j < m_outputHeight; ++j)
{
for (ndInt32 i = 0; i < m_outputWidth; ++i)
{
ndBrainFloat value = ndBrainFloat(0.0f);
ndBrainFloat value = bias;
const ndBrainMemVector in(&input[inputOffset + i], inputSize);

ndInt32 kernelOffset = 0;
for (ndInt32 k = 0; k < m_inputDepth; ++k)
{
const ndBrainMemVector filter(&kernels[kernelOffset], kernelSize);
value += bias[k] + CrossCorrelation(in, filter);
value += CrossCorrelation(in, filter);
}
kernelOffset += paddedKernelSize;
kernelOffset += kernelSize;
output[outputOffset + i] = value;
}
inputOffset += m_inputWidth;
Expand Down
2 changes: 1 addition & 1 deletion newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ class ndBrainLayerConvolutional : public ndBrainLayer
void InitGaussianWeights(ndBrainFloat variance);

ndBrainFloat CrossCorrelation(const ndBrainVector& input, const ndBrainVector& kernels) const;
void PredictionOutputChannel(const ndBrainVector& input, const ndBrainVector& bias, const ndBrainVector& kernels, ndBrainVector& output) const;
void PredictionOutputChannel(const ndBrainVector& input, const ndBrainVector& kernels, ndBrainFloat bias, ndBrainVector& output) const;

ndBrainVector m_bias;
ndBrainVector m_kernels;
Expand Down
35 changes: 10 additions & 25 deletions newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ void ndBrainLayerLinear::AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v
for (ndInt32 i = m_bias.GetCount() - 1; i >= 0; --i)
{
ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(bias_V[i])) + epsilon);
//biasGradients[j] = uHat[j] * bias_den;
m_bias[i] = bias_U[i] * bias_den;
}

Expand All @@ -181,23 +180,11 @@ void ndBrainLayerLinear::AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v
for (ndInt32 j = m_weights[i].GetCount() - 1; j >= 0; --j)
{
ndBrainFloat weight_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(weight_V[i][j])) + epsilon);
//weightGradients[j][k] = uHat[j][k] * bias_den;
m_weights[i][j] = weight_U[i][j] * weight_den;
}
}
}

void ndBrainLayerLinear::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const
{
m_weights.Mul(input, output);
output.Add(m_bias);
}

void ndBrainLayerLinear::InputDerivative(const ndBrainVector&, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
{
m_weights.TransposeMul(outputDerivative, inputDerivative);
}

void ndBrainLayerLinear::Save(const ndBrainSave* const loadSave) const
{
char buffer[1024];
Expand Down Expand Up @@ -268,18 +255,16 @@ ndBrainLayer* ndBrainLayerLinear::Load(const ndBrainLoad* const loadSave)
}


//void ndBrainLayerLinear::CalculateParamGradients(const ndBrainVector& input, const ndBrainVector& output, const ndBrainVector& outputDerivative,
// ndBrainVector& inputGradient, ndBrainVector& biasGradient, ndBrainMatrix& weightGradient)
//{
// ndAssert(biasGradient.GetCount() == outputDerivative.GetCount());
// biasGradient.Set(outputDerivative);
// for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0 ; --i)
// {
// ndBrainFloat value = outputDerivative[i];
// weightGradient[i].ScaleSet(input, value);
// }
// InputDerivative(output, outputDerivative, inputGradient);
//}
void ndBrainLayerLinear::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const
{
m_weights.Mul(input, output);
output.Add(m_bias);
}

void ndBrainLayerLinear::InputDerivative(const ndBrainVector&, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
{
m_weights.TransposeMul(outputDerivative, inputDerivative);
}

void ndBrainLayerLinear::CalculateParamGradients(
const ndBrainVector& input, const ndBrainVector& output,
Expand Down
5 changes: 3 additions & 2 deletions newton-4.00/sdk/dBrain/ndBrainTrainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,13 @@ void ndBrainTrainer::BackPropagate(const ndBrainVector& input, ndBrainLoss& loss
prefixScan.PushBack(sizeAcc);

const ndBrainFloat* const memBuffer = ndAlloca(ndBrainFloat, sizeAcc + 8);
ndBrainMemVector in0(memBuffer, input.GetCount());

ndBrainMemVector in0(memBuffer, input.GetCount());
in0.Set(input);

for (ndInt32 i = 0; i < layersCount; ++i)
{
ndBrainMemVector in(memBuffer + prefixScan[i + 0], layers[i]->GetInputSize());
const ndBrainMemVector in(memBuffer + prefixScan[i + 0], layers[i]->GetInputSize());
ndBrainMemVector out(memBuffer + prefixScan[i + 1], layers[i]->GetOutputSize());
layers[i]->MakePrediction(in, out);
}
Expand Down

0 comments on commit 5c8ded0

Please sign in to comment.