kmath_tensors_nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "@file:Repository(\"https://repo.kotlin.link\")\n",
    "@file:DependsOn(\"space.kscience:kmath-tensors-jvm:0.3.0-dev-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import space.kscience.kmath.tensors.core.*\n",
    "import space.kscience.kmath.operations.invoke\n",
    "import kotlin.math.sqrt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "interface Layer {\n",
    "    fun forward(input: DoubleTensor): DoubleTensor\n",
    "    fun backward(input: DoubleTensor, outputError: DoubleTensor): DoubleTensor\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "interface Activation : Layer {\n",
    "    \n",
    "    val activation: (DoubleTensor) -> DoubleTensor\n",
    "    val activationDer: (DoubleTensor) -> DoubleTensor\n",
    "    \n",
    "    override fun forward(input: DoubleTensor): DoubleTensor {\n",
    "        return activation(input)\n",
    "    }\n",
    "\n",
    "    override fun backward(input: DoubleTensor, outputError: DoubleTensor): DoubleTensor {\n",
    "        return DoubleTensorAlgebra { outputError * activationDer(input) }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReLU : Activation {\n",
    "    fun relu(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {\n",
    "        x.map { if (it > 0) it else 0.0 }\n",
    "    }\n",
    "\n",
    "    fun reluDer(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {\n",
    "        x.map { if (it > 0) 1.0 else 0.0 }\n",
    "    }\n",
    "\n",
    "    override val activation = ::relu\n",
    "    override val activationDer = ::reluDer\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sigmoid : Activation {\n",
    "    fun sigmoid(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {\n",
    "        1.0 / (1.0 + (-x).exp())\n",
    "    }\n",
    "\n",
    "    fun sigmoidDer(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {\n",
    "        sigmoid(x) * (1.0 - sigmoid(x))\n",
    "    }\n",
    "\n",
    "    override val activation = ::sigmoid\n",
    "    override val activationDer = ::sigmoidDer\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dense(\n",
    "    private val inputUnits: Int,\n",
    "    private val outputUnits: Int,\n",
    "    private val learningRate: Double = 0.1\n",
    ") : Layer {\n",
    "\n",
    "    private val weights: DoubleTensor = DoubleTensorAlgebra {\n",
    "        randomNormal(\n",
    "            intArrayOf(inputUnits, outputUnits)\n",
    "        ) * kotlin.math.sqrt(2.0 / (inputUnits + outputUnits))\n",
    "    }\n",
    "\n",
    "    private val bias: DoubleTensor = DoubleTensorAlgebra { zeros(intArrayOf(outputUnits)) }\n",
    "\n",
    "    override fun forward(input: DoubleTensor): DoubleTensor {\n",
    "        return BroadcastDoubleTensorAlgebra { (input dot weights) + bias }\n",
    "    }\n",
    "\n",
    "    override fun backward(input: DoubleTensor, outputError: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {\n",
    "        val gradInput = outputError dot weights.transpose()\n",
    "\n",
    "        val gradW = input.transpose() dot outputError\n",
    "        val gradBias = outputError.mean(dim = 0, keepDim = false) * input.shape[0].toDouble()\n",
    "\n",
    "        weights -= learningRate * gradW\n",
    "        bias -= learningRate * gradBias\n",
    "\n",
    "        gradInput\n",
    "    }\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork(private val layers: List<Layer>) {\n",
    "    \n",
    "    private fun softMaxLossGrad(yPred: DoubleTensor, yTrue: DoubleTensor): DoubleTensor = BroadcastDoubleTensorAlgebra {\n",
    "\n",
    "        val onesForAnswers = yPred.zeroesLike()\n",
    "        yTrue.toDoubleArray().forEachIndexed { index, labelDouble ->\n",
    "            val label = labelDouble.toInt()\n",
    "            onesForAnswers[intArrayOf(index, label)] = 1.0\n",
    "        }\n",
    "\n",
    "        val softmaxValue =  yPred.exp() / yPred.exp().sum(dim = 1, keepDim = true)\n",
    "\n",
    "        (-onesForAnswers + softmaxValue) / (yPred.shape[0].toDouble())\n",
    "    }\n",
    "\n",
    "\n",
    "    private fun forward(x: DoubleTensor): List<DoubleTensor> {\n",
    "        var input = x\n",
    "        \n",
    "        val outputs = mutableListOf<DoubleTensor>() \n",
    "        \n",
    "        layers.forEach { layer ->\n",
    "                val output = layer.forward(input)\n",
    "                outputs.add(output)\n",
    "                input = output\n",
    "            }\n",
    "\n",
    "        return outputs \n",
    "    }\n",
    "\n",
    "    private fun train(xTrain: DoubleTensor, yTrain: DoubleTensor) {\n",
    "        val layerInputs = mutableListOf<DoubleTensor>(xTrain)\n",
    "        \n",
    "        layerInputs.addAll(forward(xTrain))\n",
    "        \n",
    "        var lossGrad = softMaxLossGrad(layerInputs.last(), yTrain)\n",
    "\n",
    "        layers.zip(layerInputs).reversed().forEach { (layer, input) ->\n",
    "            lossGrad = layer.backward(input, lossGrad)\n",
    "        }\n",
    "    }\n",
    "    \n",
    "    fun accuracy(yPred: DoubleTensor, yTrue: DoubleTensor): Double {\n",
    "        check(yPred.shape contentEquals yTrue.shape)\n",
    "        val n = yPred.shape[0]\n",
    "        var correctCnt = 0\n",
    "        for (i in 0 until n) {\n",
    "            if (yPred[intArrayOf(i, 0)] == yTrue[intArrayOf(i, 0)]) {\n",
    "                correctCnt += 1\n",
    "            }\n",
    "        }\n",
    "        return correctCnt.toDouble() / n.toDouble()\n",
    "    }\n",
    "\n",
    "    fun fit(xTrain: DoubleTensor, yTrain: DoubleTensor, batchSize: Int, epochs: Int) = DoubleTensorAlgebra {\n",
    "        fun iterBatch(x: DoubleTensor, y: DoubleTensor): Sequence<Pair<DoubleTensor, DoubleTensor>> = sequence {\n",
    "            val n = x.shape[0]\n",
    "            val shuffledIndices = (0 until n).shuffled()\n",
    "            for (i in 0 until n step batchSize) {\n",
    "                val excerptIndices = shuffledIndices.drop(i).take(batchSize).toIntArray()\n",
    "                val batch = x.rowsByIndices(excerptIndices) to y.rowsByIndices(excerptIndices)\n",
    "                yield(batch)\n",
    "            }\n",
    "        }\n",
    "\n",
    "        for (epoch in 0 until epochs) {\n",
    "            println(\"Epoch ${epoch + 1}/$epochs\")\n",
    "            for ((xBatch, yBatch) in iterBatch(xTrain, yTrain)) {\n",
    "                train(xBatch, yBatch)\n",
    "            }\n",
    "            println(\"Accuracy:${accuracy(yTrain, predict(xTrain).argMax(1, true))}\")\n",
    "        }\n",
    "    }\n",
    "\n",
    "    fun predict(x: DoubleTensor): DoubleTensor {\n",
    "        return forward(x).last()\n",
    "    }\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "val features = 5\n",
    "val sampleSize = 250\n",
    "val trainSize = 180"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take features from normal distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "val x = DoubleTensorAlgebra { randomNormal(intArrayOf(sampleSize, features)) * 2.5 }\n",
    "\n",
    "BroadcastDoubleTensorAlgebra {\n",
    "    x += fromArray(\n",
    "        intArrayOf(5),\n",
    "        doubleArrayOf(0.0, -1.0, -2.5, -3.0, 5.5) // rows means\n",
    "    )\n",
    "}        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define class like `'1'` if the sum of features > 0 and `'0'` otherwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "val y = DoubleTensorAlgebra { \n",
    "    fromArray(\n",
    "        intArrayOf(sampleSize, 1),\n",
    "        DoubleArray(sampleSize) { i ->\n",
    "            if (x[i].sum() > 0.0) {\n",
    "                1.0\n",
    "            } else {\n",
    "                0.0\n",
    "            }\n",
    "        }\n",
    "    )\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split train and test (validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "val trainIndices = (0 until trainSize).toList().toIntArray()\n",
    "val testIndices = (trainSize until sampleSize).toList().toIntArray()\n",
    "\n",
    "val xTrain = DoubleTensorAlgebra { x.rowsByIndices(trainIndices) }\n",
    "val yTrain = DoubleTensorAlgebra { y.rowsByIndices(trainIndices) }\n",
    "\n",
    "val xTest = DoubleTensorAlgebra { x.rowsByIndices(testIndices) }\n",
    "val yTest = DoubleTensorAlgebra { y.rowsByIndices(testIndices) }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "val layers = mutableListOf(\n",
    "    Dense(features, 64),\n",
    "    ReLU(),\n",
    "    Dense(64, 16),\n",
    "    ReLU(),\n",
    "    Dense(16, 2),\n",
    "    Sigmoid()\n",
    ")\n",
    "\n",
    "val model = NeuralNetwork(layers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit with it with train data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "Accuracy:0.6888888888888889\n",
      "Epoch 2/10\n",
      "Accuracy:0.9333333333333333\n",
      "Epoch 3/10\n",
      "Accuracy:0.95\n",
      "Epoch 4/10\n",
      "Accuracy:0.9611111111111111\n",
      "Epoch 5/10\n",
      "Accuracy:0.9777777777777777\n",
      "Epoch 6/10\n",
      "Accuracy:0.9833333333333333\n",
      "Epoch 7/10\n",
      "Accuracy:0.9722222222222222\n",
      "Epoch 8/10\n",
      "Accuracy:0.9888888888888889\n",
      "Epoch 9/10\n",
      "Accuracy:0.95\n",
      "Epoch 10/10\n",
      "Accuracy:0.9833333333333333\n"
     ]
    }
   ],
   "source": [
    "model.fit(xTrain, yTrain, batchSize = 20, epochs = 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check out accuracy on validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "val prediction = model.predict(xTest) // logits\n",
    "\n",
    "val predictionLabels = DoubleTensorAlgebra { prediction.argMax(1, true) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9428571428571428"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.accuracy(yTest, predictionLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Kotlin",
   "language": "kotlin",
   "name": "kotlin"
  },
  "language_info": {
   "codemirror_mode": "text/x-kotlin",
   "file_extension": ".kt",
   "mimetype": "text/x-kotlin",
   "name": "kotlin",
   "nbconvert_exporter": "",
   "pygments_lexer": "kotlin",
   "version": "1.5.20-dev-3998"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}