Skip to content

Commit

Permalink
start testing CNN with Mnist data set. (wip)
Browse files Browse the repository at this point in the history
  • Loading branch information
Julio Jerez committed Nov 7, 2023
1 parent ccccbff commit 1a9c637
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 39 deletions.
26 changes: 13 additions & 13 deletions newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ static void MnistTrainingSet()
//optimizer.SetRegularizer(ndReal(5.0e-5f)); // test data score 98.22%

bool firstPass = true;
for (ndInt32 i = 0; i < 4000000; ++i)
for (ndInt32 passes = 0; passes < 4000000; ++passes)
{
ndInt32 priorityStart = ndMin(priorityIndexArray.GetCount(), 2);
for (ndInt32 j = 0; j < priorityStart; ++j)
Expand All @@ -378,7 +378,7 @@ static void MnistTrainingSet()
ndBrainThreadPool::ParallelExecute(BackPropagateBash);
optimizer.Update(this, m_trainers, ndReal(1.0e-3f));

if ((i % 1024) == 0)
if ((passes % 1024) == 0)
{
ndInt32 failCount[D_MAX_THREADS_COUNT];
ndFixSizeArray<ndFixSizeArray<ndUnsigned32, 1024>, D_MAX_THREADS_COUNT> failPriorities;
Expand Down Expand Up @@ -490,7 +490,7 @@ static void MnistTrainingSet()
bestBrain.CopyFrom(m_brain);
ndExpandTraceMessage("success rate: %f%% ", (ndFloat32)(trainingDigits->GetCount() - actualTraining) * 100.0f / (ndFloat32)trainingDigits->GetCount());
ndExpandTraceMessage("failed count: %d ", actualTraining);
ndExpandTraceMessage("steps: %d", i);
ndExpandTraceMessage("steps: %d", passes);
ndExpandTraceMessage("\n");
}
else if (fails <= minTestFail)
Expand All @@ -499,7 +499,7 @@ static void MnistTrainingSet()
bestBrain.CopyFrom(m_brain);
ndExpandTraceMessage("success rate: %f%% ", (ndFloat32)(trainingDigits->GetCount() - actualTraining) * 100.0f / (ndFloat32)trainingDigits->GetCount());
ndExpandTraceMessage("failed count: %d ", actualTraining);
ndExpandTraceMessage("steps: %d", i);
ndExpandTraceMessage("steps: %d", passes);
ndExpandTraceMessage(" %d\n", minTestFail);
}
}
Expand Down Expand Up @@ -527,23 +527,23 @@ static void MnistTrainingSet()
const ndBrainLayerConvolutional* conv;
const ndBrainLayerConvolutionalMaxPooling* pooling;

//layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 16));
layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 3));
layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 16));
//layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 3));
layers.PushBack(new ndBrainLayerTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
conv = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
pooling = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

//layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 16));
layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 3));
layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 32));
//layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 3));
layers.PushBack(new ndBrainLayerTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
conv = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
pooling = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

//layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 32));
layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 2));
layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 32));
//layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 2));
layers.PushBack(new ndBrainLayerTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
conv = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
pooling = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);
Expand Down
87 changes: 69 additions & 18 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,17 @@ ndBrainLayerConvolutional::ndBrainLayerConvolutional(ndInt32 inputWidth, ndInt32
//Debug(m_inputWidth, m_inputHeight, m_inputDepth, m_kernelSize, m_numberOfKernels);
//Debug(7, 7, 2, 3, 3);
//Debug(4, 4, 2, 2, 3);
Debug(3, 3, 1, 2, 1);
//Debug(3, 3, 1, 2, 1);
}
}

ndBrainLayerConvolutional::ndBrainLayerConvolutional(const ndBrainLayerConvolutional& src)
:ndBrainLayer(src)
,m_bias(src.m_bias)
,m_kernels(src.m_kernels)
,m_paddedGradientOut()
,m_inputOffsets(src.m_inputOffsets)
,m_inputGradOffsets(src.m_inputGradOffsets)
,m_inputWidth(src.m_inputWidth)
,m_inputHeight(src.m_inputHeight)
,m_inputDepth(src.m_inputDepth)
Expand Down Expand Up @@ -135,9 +137,8 @@ bool ndBrainLayerConvolutional::HasParameters() const

void ndBrainLayerConvolutional::InitWeightsXavierMethod()
{
//ndBrainFloat weighVariance = ndBrainFloat(ndSqrt(ndFloat32(6.0f) / ndFloat32(GetInputSize() + GetOutputSize())));
//InitWeights(weighVariance, ndBrainFloat(0.0f));
ndAssert(0);
ndBrainFloat weighVariance = ndBrainFloat(ndSqrt(ndFloat32(6.0f) / ndFloat32(GetInputSize() + GetOutputSize())));
InitWeights(weighVariance, ndBrainFloat(0.0f));
}

void ndBrainLayerConvolutional::InitGaussianBias(ndBrainFloat variance)
Expand Down Expand Up @@ -165,6 +166,45 @@ void ndBrainLayerConvolutional::Set(const ndBrainLayer& src)
m_kernels.Set(convSrc.m_kernels);
}

void ndBrainLayerConvolutional::Clear()
{
m_bias.Set(ndBrainFloat(0.0f));
m_kernels.Set(ndBrainFloat(0.0f));
}

void ndBrainLayerConvolutional::FlushToZero()
{
m_bias.FlushToZero();
m_kernels.FlushToZero();
}

void ndBrainLayerConvolutional::Scale(ndBrainFloat scale)
{
m_bias.Scale(scale);
m_kernels.Scale(scale);
}

void ndBrainLayerConvolutional::Add(const ndBrainLayer& src)
{
const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src;
m_bias.Add(linearSrc.m_bias);
m_kernels.Add(linearSrc.m_kernels);
}

void ndBrainLayerConvolutional::Mul(const ndBrainLayer& src)
{
const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src;
m_bias.Mul(linearSrc.m_bias);
m_kernels.Mul(linearSrc.m_kernels);
}

void ndBrainLayerConvolutional::ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale)
{
const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src;
m_bias.ScaleAdd(linearSrc.m_bias, scale);
m_kernels.ScaleAdd(linearSrc.m_kernels, scale);
}

void ndBrainLayerConvolutional::Blend(const ndBrainLayer& src, ndBrainFloat blend)
{
//const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src;
Expand All @@ -173,12 +213,34 @@ void ndBrainLayerConvolutional::Blend(const ndBrainLayer& src, ndBrainFloat blen
ndAssert(0);
}

void ndBrainLayerConvolutional::InputDerivative(const ndBrainVector&, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
void ndBrainLayerConvolutional::AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon)
{
//m_weights.TransposeMul(outputDerivative, inputDerivative);
ndAssert(0);
const ndBrainLayerConvolutional& linear_U = (ndBrainLayerConvolutional&)u;
const ndBrainLayerConvolutional& linear_V = (ndBrainLayerConvolutional&)v;

const ndBrainVector& bias_U = linear_U.m_bias;
const ndBrainVector& bias_V = linear_V.m_bias;
for (ndInt32 i = m_bias.GetCount() - 1; i >= 0; --i)
{
ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(bias_V[i])) + epsilon);
m_bias[i] = bias_U[i] * bias_den;
}

const ndBrainVector& kernels_U = linear_U.m_kernels;
const ndBrainVector& kernels_V = linear_V.m_kernels;
for (ndInt32 j = m_kernels.GetCount() - 1; j >= 0; --j)
{
ndBrainFloat weight_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(kernels_V[j])) + epsilon);
m_kernels[j] = kernels_U[j] * weight_den;
}
}

//void ndBrainLayerConvolutional::InputDerivative(const ndBrainVector&, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
//{
// //m_weights.TransposeMul(outputDerivative, inputDerivative);
// ndAssert(0);
//}

void ndBrainLayerConvolutional::Save(const ndBrainSave* const loadSave) const
{
//char buffer[1024];
Expand Down Expand Up @@ -848,23 +910,12 @@ void ndBrainLayerConvolutional::CalculateParamGradients(
ndInt32 gradInputOffset = 0;
const ndInt32 gradInputWidth = m_inputWidth + m_kernelSize - 1;
const ndInt32 gradInputSize = gradInputWidth * m_kernelSize;
//const ndInt32 inputSize = m_inputWidth * m_kernelSize;
const ndInt32 kernelSize = m_kernelSize * m_kernelSize;
for (ndInt32 y = 0; y < m_inputHeight; ++y)
{
for (ndInt32 x = 0; x < m_inputWidth; ++x)
{
//ndBrainFloat value = ndBrainFloat(0.0f);
//const ndBrainMemVector in(&input[inputOffset + i], inputSize);
const ndBrainMemVector in(&m_paddedGradientOut[gradInputOffset + x], gradInputSize);

//ndInt32 kernelOffset = 0;
//for (ndInt32 k = 0; k < m_inputDepth; ++k)
//{
// const ndBrainMemVector filter(&kernels[kernelOffset], kernelSize);
// value += CrossCorrelation(in, filter);
//}
//kernelOffset += kernelSize;
output[outputOffset + x] += CrossCorrelation(in);
}
outputOffset += m_inputWidth;
Expand Down
15 changes: 8 additions & 7 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ class ndBrainLayerConvolutional : public ndBrainLayer

virtual void InitWeightsXavierMethod();
virtual void InitWeights(ndBrainFloat weighVariance, ndBrainFloat biasVariance);
virtual void AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon);

virtual void MakePrediction(const ndBrainVector& input, ndBrainVector& output) const;
virtual void InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const;
//virtual void InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const;

virtual void CalculateParamGradients(
const ndBrainVector& input, const ndBrainVector& output,
Expand All @@ -58,14 +59,14 @@ class ndBrainLayerConvolutional : public ndBrainLayer
virtual void Save(const ndBrainSave* const loadSave) const;
static ndBrainLayer* Load(const ndBrainLoad* const loadSave);

//virtual void Clear();
//virtual void FlushToZero();
//virtual void Scale(ndBrainFloat scale);
virtual void Clear();
virtual void FlushToZero();
virtual void Scale(ndBrainFloat scale);
virtual void Set(const ndBrainLayer& src);
//virtual void Add(const ndBrainLayer& src);
//virtual void Mul(const ndBrainLayer& src);
virtual void Add(const ndBrainLayer& src);
virtual void Mul(const ndBrainLayer& src);
//virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend);
//virtual void ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale);
virtual void ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale);

private:
void InitGaussianBias(ndBrainFloat variance);
Expand Down
1 change: 0 additions & 1 deletion newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,6 @@ void ndBrainLayerLinear::FlushToZero()

void ndBrainLayerLinear::Scale(ndBrainFloat scale)
{
//const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src;
m_bias.Scale(scale);
m_weights.Scale(scale);
}
Expand Down

0 comments on commit 1a9c637

Please sign in to comment.