Skip to content

Commit

Permalink
setting up a full CNN (wip)
Browse files Browse the repository at this point in the history
  • Loading branch information
Julio Jerez committed Oct 27, 2023
1 parent 8e8f4e9 commit c6ab89b
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 34 deletions.
20 changes: 17 additions & 3 deletions newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,22 +518,37 @@ static void MnistTrainingSet()
{
ndBrain brain;

ndInt32 neuronsPerLayers = 64;
ndFixSizeArray<ndBrainLayer*, 16> layers;

#ifdef D_USE_CONVOLUTIONAL_LAYERS
ndInt32 height = 28;
ndInt32 width = trainingDigits->GetColumns() / height;
ndAssert((height * width) == trainingDigits->GetColumns());

layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 10));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv0 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv0->GetOutputWidth(), conv0->GetOutputHeighh(), conv0->GetOutputChannels()));
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv0->GetOutputWidth(), conv0->GetOutputHeight(), conv0->GetOutputChannels()));
const ndBrainLayerConvolutionalMaxPooling* const pooling0 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

layers.PushBack(new ndBrainLayerConvolutional(pooling0->GetOutputWidth(), pooling0->GetOutputHeight(), pooling0->GetOutputChannels(), 3, 16));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv1 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv1->GetOutputWidth(), conv1->GetOutputHeight(), conv1->GetOutputChannels()));
const ndBrainLayerConvolutionalMaxPooling* const pooling1 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

layers.PushBack(new ndBrainLayerConvolutional(pooling1->GetOutputWidth(), pooling1->GetOutputHeight(), pooling1->GetOutputChannels(), 3, 32));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv2 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv2->GetOutputWidth(), conv2->GetOutputHeight(), conv2->GetOutputChannels()));


layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns()));
layers.PushBack(new ndBrainLayerCategoricalSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize()));

#else
ndInt32 neuronsPerLayers = 64;


layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), neuronsPerLayers));
layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
Expand All @@ -548,7 +563,6 @@ static void MnistTrainingSet()
layers.PushBack(new ndBrainLayerCategoricalSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
#endif


for (ndInt32 i = 0; i < layers.GetCount(); ++i)
{
brain.AddLayer(layers[i]);
Expand Down
38 changes: 19 additions & 19 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ ndInt32 ndBrainLayerConvolutional::GetOutputWidth() const
return m_outputWidth;
}

ndInt32 ndBrainLayerConvolutional::GetOutputHeighh() const
ndInt32 ndBrainLayerConvolutional::GetOutputHeight() const
{
return m_outputHeight;
}
Expand Down Expand Up @@ -235,23 +235,6 @@ ndBrainLayer* ndBrainLayerConvolutional::Load(const ndBrainLoad* const loadSave)
return nullptr;
}

void ndBrainLayerConvolutional::CalculateParamGradients(
const ndBrainVector& input, const ndBrainVector& output,
const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const gradientOut) const
{
ndAssert(0);
//ndAssert(!strcmp(GetLabelId(), gradientOut->GetLabelId()));
//ndBrainLayerLinear* const gradients = (ndBrainLayerLinear*)gradientOut;
//ndAssert(gradients->m_bias.GetCount() == outputDerivative.GetCount());
//gradients->m_bias.Set(outputDerivative);
//for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0; --i)
//{
// ndBrainFloat value = outputDerivative[i];
// gradients->m_weights[i].ScaleSet(input, value);
//}
//InputDerivative(output, outputDerivative, inputGradient);
}

void ndBrainLayerConvolutional::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const
{
//m_weights.Mul(input, output);
Expand Down Expand Up @@ -316,4 +299,21 @@ ndBrainFloat ndBrainLayerConvolutional::CrossCorrelation(const ndBrainVector& in
cacheStart += m_kernelSize;
}
return kernels.Dot(inputCache);
}
}

void ndBrainLayerConvolutional::CalculateParamGradients(
const ndBrainVector& input, const ndBrainVector& output,
const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const gradientOut) const
{
ndAssert(0);
//ndAssert(!strcmp(GetLabelId(), gradientOut->GetLabelId()));
//ndBrainLayerLinear* const gradients = (ndBrainLayerLinear*)gradientOut;
//ndAssert(gradients->m_bias.GetCount() == outputDerivative.GetCount());
//gradients->m_bias.Set(outputDerivative);
//for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0; --i)
//{
// ndBrainFloat value = outputDerivative[i];
// gradients->m_weights[i].ScaleSet(input, value);
//}
//InputDerivative(output, outputDerivative, inputGradient);
}
2 changes: 1 addition & 1 deletion newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class ndBrainLayerConvolutional : public ndBrainLayer
virtual ndBrainLayer* Clone() const;

ndInt32 GetOutputWidth() const;
ndInt32 GetOutputHeighh() const;
ndInt32 GetOutputHeight() const;
ndInt32 GetOutputChannels() const;

virtual bool HasParameters() const;
Expand Down
49 changes: 39 additions & 10 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutionalMaxPooling.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,36 @@ ndBrainLayerConvolutionalMaxPooling::ndBrainLayerConvolutionalMaxPooling(const n
m_index.SetCount(src.m_index.GetCount());
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetInputWidth() const
{
return m_width;
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetInputHeight() const
{
return m_height;
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetInputChannels() const
{
return m_channels;
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetOutputWidth() const
{
return (m_width + 1) / 2;
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetOutputHeight() const
{
return (m_height + 1) / 2;
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetOutputChannels() const
{
return m_channels;
}

ndInt32 ndBrainLayerConvolutionalMaxPooling::GetInputSize() const
{
return m_width * m_height * m_channels;
Expand Down Expand Up @@ -124,14 +154,13 @@ void ndBrainLayerConvolutionalMaxPooling::MakePrediction(const ndBrainVector& in

void ndBrainLayerConvolutionalMaxPooling::InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
{
ndAssert(0);
//ndAssert(output.GetCount() == outputDerivative.GetCount());
//ndAssert(output.GetCount() == inputDerivative.GetCount());
//
//for (ndInt32 i = output.GetCount() - 1; i >= 0; --i)
//{
// inputDerivative[i] = (output[i] > ndBrainFloat(0.0f)) ? ndBrainFloat(1.0f) : ndBrainFloat(0.0f);
// ndAssert(ndCheckFloat(inputDerivative[i]));
//}
//inputDerivative.Mul(outputDerivative);
ndAssert(output.GetCount() == outputDerivative.GetCount());
ndAssert(m_index.GetCount() == outputDerivative.GetCount());

inputDerivative.Set(ndBrainFloat(0.0f));
for (ndInt32 i = m_index.GetCount() - 1; i >= 0; --i)
{
ndInt32 index = m_index[i];
inputDerivative[index] = outputDerivative[i];
}
}
8 changes: 8 additions & 0 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutionalMaxPooling.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ class ndBrainLayerConvolutionalMaxPooling : public ndBrainLayerActivation
ndBrainLayer* Clone() const;
static ndBrainLayer* Load(const ndBrainLoad* const loadSave);

ndInt32 GetInputWidth() const;
ndInt32 GetInputHeight() const;
ndInt32 GetInputChannels() const;

ndInt32 GetOutputWidth() const;
ndInt32 GetOutputHeight() const;
ndInt32 GetOutputChannels() const;

virtual ndInt32 GetInputSize() const;

const char* GetLabelId() const;
Expand Down
1 change: 0 additions & 1 deletion newton-4.00/sdk/dBrain/ndBrainTrainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,6 @@ void ndBrainTrainer::BackPropagate(const ndBrainVector& input, ndBrainLoss& loss
gradientIn.SetSize(layer->GetInputSize());
const ndBrainMemVector in(memBuffer + prefixScan[i + 0], layer->GetInputSize());
const ndBrainMemVector out(memBuffer + prefixScan[i + 1], layer->GetOutputSize());
//layer->CalculateParamGradients(in, out, gradientOut, gradientIn, m_data[i]->m_gradBias, m_data[i]->m_gradWeight);
layer->CalculateParamGradients(in, out, gradientOut, gradientIn, m_data[i]->m_gradient);
gradientIn.Swap(gradientOut);
}
Expand Down

0 comments on commit c6ab89b

Please sign in to comment.