Skip to content

Commit

Permalink
print symbolic convolution operation, (wip)
Browse files Browse the repository at this point in the history
  • Loading branch information
@JulioJerez
JulioJerez committed Oct 30, 2023
1 parent ed85cac commit 6c161b9
Show file tree
Hide file tree
Showing 9 changed files with 215 additions and 96 deletions.
33 changes: 18 additions & 15 deletions newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -524,26 +524,29 @@ static void MnistTrainingSet()
ndInt32 width = trainingDigits->GetColumns() / height;
ndAssert((height * width) == trainingDigits->GetColumns());

const ndBrainLayerConvolutional* conv;
const ndBrainLayerConvolutionalMaxPooling* pooling;

//layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 16));
layers.PushBack(new ndBrainLayerConvolutional(width, height, 1, 5, 1));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv0 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv0->GetOutputWidth(), conv0->GetOutputHeight(), conv0->GetOutputChannels()));
const ndBrainLayerConvolutionalMaxPooling* const pooling0 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

//layers.PushBack(new ndBrainLayerConvolutional(pooling0->GetOutputWidth(), pooling0->GetOutputHeight(), pooling0->GetOutputChannels(), 3, 16));
layers.PushBack(new ndBrainLayerConvolutional(pooling0->GetOutputWidth(), pooling0->GetOutputHeight(), pooling0->GetOutputChannels(), 3, 2));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv1 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv1->GetOutputWidth(), conv1->GetOutputHeight(), conv1->GetOutputChannels()));
const ndBrainLayerConvolutionalMaxPooling* const pooling1 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);
conv = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
pooling = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

//layers.PushBack(new ndBrainLayerConvolutional(pooling1->GetOutputWidth(), pooling1->GetOutputHeight(), pooling1->GetOutputChannels(), 3, 32));
layers.PushBack(new ndBrainLayerConvolutional(pooling1->GetOutputWidth(), pooling1->GetOutputHeight(), pooling1->GetOutputChannels(), 3, 3));
//layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 16));
layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 1));
layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
const ndBrainLayerConvolutional* const conv2 = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv2->GetOutputWidth(), conv2->GetOutputHeight(), conv2->GetOutputChannels()));
//const ndBrainLayerConvolutionalMaxPooling* const pooling2 = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);
conv = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
pooling = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

////layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 32));
//layers.PushBack(new ndBrainLayerConvolutional(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 1));
//layers.PushBack(new ndBrainLayerReluActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
//conv = (ndBrainLayerConvolutional*)(layers[layers.GetCount() - 2]);
//layers.PushBack(new ndBrainLayerConvolutionalMaxPooling(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
//pooling = (ndBrainLayerConvolutionalMaxPooling*)(layers[layers.GetCount() - 1]);

layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns()));
layers.PushBack(new ndBrainLayerCategoricalSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize()));
Expand Down
1 change: 1 addition & 0 deletions newton-4.00/sdk/dBrain/ndBrainLayerCategoricalSoftmaxActivation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ void ndBrainLayerCategoricalSoftmaxActivation::InputDerivative(const ndBrainVect
// check that the ouputDerivative is a one hat encoding
ndAssert(output.GetCount() == inputDerivative.GetCount());
ndAssert(output.GetCount() == outputDerivative.GetCount());

ndInt32 index = 0;
for (ndInt32 i = 0; i < outputDerivative.GetCount(); ++i)
{
Expand Down
106 changes: 105 additions & 1 deletion newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ ndBrainLayerConvolutional::ndBrainLayerConvolutional(ndInt32 inputWidth, ndInt32
}
offset += m_inputWidth;
}

//if (inputWidth == 5)
if (inputWidth == 12)
Debug(inputWidth, m_inputHeight, inputDepth, kernelSize, numberOfKernels);
}

ndBrainLayerConvolutional::ndBrainLayerConvolutional(const ndBrainLayerConvolutional& src)
Expand Down Expand Up @@ -144,7 +148,6 @@ void ndBrainLayerConvolutional::InitWeights(ndBrainFloat weighVariance, ndBrainF
InitGaussianWeights(weighVariance);
}

//void ndBrainLayerConvolutional::CopyFrom(const ndBrainLayer& src)
void ndBrainLayerConvolutional::Set(const ndBrainLayer& src)
{
const ndBrainLayerConvolutional& convSrc = (ndBrainLayerConvolutional&)src;
Expand Down Expand Up @@ -288,6 +291,106 @@ void ndBrainLayerConvolutional::PredictionOutputChannel(const ndBrainVector& inp
}
}

void ndBrainLayerConvolutional::Debug(ndInt32 width, ndInt32 height, ndInt32 channels, ndInt32 filterSize, ndInt32 filterCount)
{
// print inputs
for (ndInt32 channel = 0; channel < channels; ++channel)
{
for (ndInt32 y = 0; y < height; ++y)
{
for (ndInt32 x = 0; x < width; ++x)
{
ndTrace(("x(%d,%d,%d) ", channel, y, x));
}
ndTrace(("\n"));
}
ndTrace(("\n"));
}

// print kilters
ndTrace(("\n"));
for (ndInt32 filter = 0; filter < filterCount; ++filter)
{
for (ndInt32 channel = 0; channel < channels; ++channel)
{
for (ndInt32 y = 0; y < filterSize; ++y)
{
for (ndInt32 x = 0; x < filterSize; ++x)
{
ndTrace(("w(%d,%d,%d,%d) ", filter, channel, y, x));
}
ndTrace(("\n"));
}
ndTrace(("\n"));
}
ndTrace(("b(%d)\n", filter));
ndTrace(("\n"));
}

// print outputs
ndTrace(("\n"));
for (ndInt32 filter = 0; filter < filterCount; ++filter)
{
for (ndInt32 y = 0; y < (height - filterSize + 1); ++y)
{
for (ndInt32 x = 0; x < (width - filterSize + 1); ++x)
{
ndTrace(("y(%d,%d,%d) ", filter, y, x));
}
ndTrace(("\n"));
}
ndTrace(("\n"));
//ndTrace(("b(%d)\n\n", filter));
}


// print convolutions
ndTrace(("\n"));
for (ndInt32 filter = 0; filter < filterCount; ++filter)
{
for (ndInt32 y0 = 0; y0 < (height - filterSize + 1); ++y0)
{
for (ndInt32 x0 = 0; x0 < (width - filterSize + 1); ++x0)
{
ndTrace(("y(%d,%d,%d)=\n", filter, y0, x0));
for (ndInt32 channel = 0; channel < channels; ++channel)
{
for (ndInt32 y = 0; y < filterSize; ++y)
{
ndTrace((" "));
for (ndInt32 x = 0; x < filterSize; ++x)
{
ndTrace(("x(%d,%d,%d)*w(%d,%d,%d,%d) + ", channel, y0 + y, x0 + x, filter, channel, y, x));
}
ndTrace(("\n"));
}
}
ndTrace((" b(%d)\n", filter));
ndTrace(("\n"));
}
}
ndTrace(("\n"));
}

// print weight gradients
for (ndInt32 filter = 0; filter < filterCount; ++filter)
{
for (ndInt32 channel = 0; channel < channels; ++channel)
{
for (ndInt32 y0 = 0; y0 < filterSize; ++y0)
{
for (ndInt32 x0 = 0; x0 < filterSize; ++x0)
{
ndTrace(("dL/dw(%d,%d,%d,%d)=\n", filter, channel, y0, x0));


ndTrace(("\n"));
}
}
}
}
}

void ndBrainLayerConvolutional::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const
{
//m_weights.Mul(input, output);
Expand All @@ -313,6 +416,7 @@ void ndBrainLayerConvolutional::CalculateParamGradients(
ndBrainLayerConvolutional* const gradients = (ndBrainLayerConvolutional*)gradientOut;

ndAssert(gradients->m_bias.GetCount() == m_numberOfKernels);
ndAssert(output.GetCount() == outputDerivative.GetCount());

//gradients->m_bias.Set(outputDerivative);
const ndInt32 size = m_outputWidth * m_outputHeight;
Expand Down
12 changes: 10 additions & 2 deletions newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,24 @@ class ndBrainLayerConvolutional : public ndBrainLayer
virtual void Save(const ndBrainSave* const loadSave) const;
static ndBrainLayer* Load(const ndBrainLoad* const loadSave);

//virtual void Clear();
//virtual void FlushToZero();
//virtual void Scale(ndBrainFloat scale);
virtual void Set(const ndBrainLayer& src);
//virtual void Add(const ndBrainLayer& src);
//virtual void Mul(const ndBrainLayer& src);
//virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend);
//virtual void ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale);

void Set(const ndBrainLayer& src);

private:
void InitGaussianBias(ndBrainFloat variance);
void InitGaussianWeights(ndBrainFloat variance);

ndBrainFloat CrossCorrelation(const ndBrainVector& input, const ndBrainVector& kernels) const;
void PredictionOutputChannel(const ndBrainVector& input, const ndBrainVector& kernels, ndBrainFloat bias, ndBrainVector& output) const;

void Debug(ndInt32 width, ndInt32 height, ndInt32 channels, ndInt32 filterSize, ndInt32 filterCount);

ndBrainVector m_bias;
ndBrainVector m_kernels;
ndFixSizeArray<ndInt32, 128> m_inputOffsets;
Expand Down
1 change: 1 addition & 0 deletions newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ void ndBrainLayerLinear::CalculateParamGradients(
ndAssert(!strcmp(GetLabelId(), gradientOut->GetLabelId()));
ndBrainLayerLinear* const gradients = (ndBrainLayerLinear*)gradientOut;
ndAssert(gradients->m_bias.GetCount() == outputDerivative.GetCount());
ndAssert(output.GetCount() == outputDerivative.GetCount());

gradients->m_bias.Set(outputDerivative);
for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0; --i)
Expand Down
24 changes: 12 additions & 12 deletions newton-4.00/sdk/dBrain/ndBrainLayerReluActivation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,18 @@ const char* ndBrainLayerReluActivation::GetLabelId() const
return "ndBrainLayerReluActivation";
}

ndBrainLayer* ndBrainLayerReluActivation::Load(const ndBrainLoad* const loadSave)
{
char buffer[1024];
loadSave->ReadString(buffer);

loadSave->ReadString(buffer);
ndInt32 inputs = loadSave->ReadInt();
ndBrainLayerReluActivation* const layer = new ndBrainLayerReluActivation(inputs);
loadSave->ReadString(buffer);
return layer;
}

void ndBrainLayerReluActivation::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const
{
ndAssert(input.GetCount() == output.GetCount());
Expand All @@ -65,15 +77,3 @@ void ndBrainLayerReluActivation::InputDerivative(const ndBrainVector& output, co
}
inputDerivative.Mul(outputDerivative);
}

ndBrainLayer* ndBrainLayerReluActivation::Load(const ndBrainLoad* const loadSave)
{
char buffer[1024];
loadSave->ReadString(buffer);

loadSave->ReadString(buffer);
ndInt32 inputs = loadSave->ReadInt();
ndBrainLayerReluActivation* const layer = new ndBrainLayerReluActivation(inputs);
loadSave->ReadString(buffer);
return layer;
}
36 changes: 18 additions & 18 deletions newton-4.00/sdk/dBrain/ndBrainLayerSigmoidActivation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,16 @@ const char* ndBrainLayerSigmoidActivation::GetLabelId() const
return "ndBrainLayerSigmoidActivation";
}

void ndBrainLayerSigmoidActivation::InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
ndBrainLayer* ndBrainLayerSigmoidActivation::Load(const ndBrainLoad* const loadSave)
{
ndAssert(output.GetCount() == outputDerivative.GetCount());
ndAssert(output.GetCount() == inputDerivative.GetCount());

inputDerivative.Set(ndBrainFloat(1.0f));
inputDerivative.Sub(output);
inputDerivative.Mul(output);
char buffer[1024];
loadSave->ReadString(buffer);

inputDerivative.Mul(outputDerivative);
inputDerivative.FlushToZero();
loadSave->ReadString(buffer);
ndInt32 inputs = loadSave->ReadInt();
ndBrainLayerSigmoidActivation* const layer = new ndBrainLayerSigmoidActivation(inputs);
loadSave->ReadString(buffer);
return layer;
}

void ndBrainLayerSigmoidActivation::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const
Expand Down Expand Up @@ -82,14 +81,15 @@ void ndBrainLayerSigmoidActivation::MakePrediction(const ndBrainVector& input, n
}
}

ndBrainLayer* ndBrainLayerSigmoidActivation::Load(const ndBrainLoad* const loadSave)
void ndBrainLayerSigmoidActivation::InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const
{
char buffer[1024];
loadSave->ReadString(buffer);
ndAssert(output.GetCount() == outputDerivative.GetCount());
ndAssert(output.GetCount() == inputDerivative.GetCount());

loadSave->ReadString(buffer);
ndInt32 inputs = loadSave->ReadInt();
ndBrainLayerSigmoidActivation* const layer = new ndBrainLayerSigmoidActivation(inputs);
loadSave->ReadString(buffer);
return layer;
}
inputDerivative.Set(ndBrainFloat(1.0f));
inputDerivative.Sub(output);
inputDerivative.Mul(output);

inputDerivative.Mul(outputDerivative);
inputDerivative.FlushToZero();
}
Loading

0 comments on commit 6c161b9

Please sign in to comment.