auto_mpg_nn_KotlinDL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Kotlin neural network for Auto-Mpg Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "   <div id=\"Oe5NOD\"></div>\n",
       "   <script type=\"text/javascript\" data-lets-plot-script=\"library\">\n",
       "       if(!window.letsPlotCallQueue) {\n",
       "           window.letsPlotCallQueue = [];\n",
       "       }; \n",
       "       window.letsPlotCall = function(f) {\n",
       "           window.letsPlotCallQueue.push(f);\n",
       "       };\n",
       "       (function() {\n",
       "           var script = document.createElement(\"script\");\n",
       "           script.type = \"text/javascript\";\n",
       "           script.src = \"https://dl.bintray.com/jetbrains/lets-plot/lets-plot-2.0.1.min.js\";\n",
       "           script.onload = function() {\n",
       "               window.letsPlotCall = function(f) {f();};\n",
       "               window.letsPlotCallQueue.forEach(function(f) {f();});\n",
       "               window.letsPlotCallQueue = [];\n",
       "               \n",
       "               \n",
       "           };\n",
       "           script.onerror = function(event) {\n",
       "               window.letsPlotCall = function(f) {};\n",
       "               window.letsPlotCallQueue = [];\n",
       "               var div = document.createElement(\"div\");\n",
       "               div.style.color = 'darkred';\n",
       "               div.textContent = 'Error loading Lets-Plot JS';\n",
       "               document.getElementById(\"Oe5NOD\").appendChild(div);\n",
       "           };\n",
       "           var e = document.getElementById(\"Oe5NOD\");\n",
       "           e.appendChild(script);\n",
       "       })();\n",
       "   </script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "// dependencies for KotlinDL\n",
    "@file:Repository(\"https://kotlin.bintray.com/kotlin-datascience\")\n",
    "@file:DependsOn(\"org.jetbrains.kotlin-deeplearning:api:[0.1.1]\")\n",
    "// pandas-inspired library to work with data frames\n",
    "%use krangl\n",
    "// library for plotting\n",
    "%use lets-plot\n",
    "// these two libraries are already in kotlin jupyter kernel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Prepoccesing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Raw auto-mpg dataset is not provided in any standard format. There are some tools for parsing it :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "// path to file with dataset\n",
    "val path = \"data/auto-mpg.data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "fun parseDataString(string: String): List<Double>? {\n",
    "    val splitString = string.split('\\t')[0].split(Regex(\"( |\\t|\\n)+\"))\n",
    "    \n",
    "    // ignore raws with any null values\n",
    "    if (\"?\" in splitString) {\n",
    "        return null\n",
    "    }\n",
    "\n",
    "    return splitString.map { it -> it.toString().toDouble() }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import java.io.File\n",
    "\n",
    "val dataList = mutableListOf<Double>()\n",
    "\n",
    "File(path).forEachLine { \n",
    "    val stringData = parseDataString(it)\n",
    "    stringData?.let {\n",
    "        it.forEach { dataList.add(it) }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create keras DataFrame :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "val colNames = listOf(\n",
    "    \"mpg\", \n",
    "    \"cylinders\", \n",
    "    \"displacement\", \n",
    "    \"horsepower\", \"weight\", \n",
    "    \"acceleration\", \n",
    "    \"model year\", \n",
    "    \"origin\"\n",
    ")\n",
    "\n",
    "val df: DataFrame = dataFrameOf(colNames)(dataList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html><body><table><tr><th style=\"text-align:left\">mpg</th><th style=\"text-align:left\">cylinders</th><th style=\"text-align:left\">displacement</th><th style=\"text-align:left\">horsepower</th><th style=\"text-align:left\">weight</th><th style=\"text-align:left\">acceleration</th><th style=\"text-align:left\">model year</th><th style=\"text-align:left\">origin</th></tr><tr><td style=\"text-align:left\" title=\"18.0\">18.0</td><td style=\"text-align:left\" title=\"8.0\">8.0</td><td style=\"text-align:left\" title=\"307.0\">307.0</td><td style=\"text-align:left\" title=\"130.0\">130.0</td><td style=\"text-align:left\" title=\"3504.0\">3504.0</td><td style=\"text-align:left\" title=\"12.0\">12.0</td><td style=\"text-align:left\" title=\"70.0\">70.0</td><td style=\"text-align:left\" title=\"1.0\">1.0</td></tr><tr><td style=\"text-align:left\" title=\"15.0\">15.0</td><td style=\"text-align:left\" title=\"8.0\">8.0</td><td style=\"text-align:left\" title=\"350.0\">350.0</td><td style=\"text-align:left\" title=\"165.0\">165.0</td><td style=\"text-align:left\" title=\"3693.0\">3693.0</td><td style=\"text-align:left\" title=\"11.5\">11.5</td><td style=\"text-align:left\" title=\"70.0\">70.0</td><td style=\"text-align:left\" title=\"1.0\">1.0</td></tr><tr><td style=\"text-align:left\" title=\"18.0\">18.0</td><td style=\"text-align:left\" title=\"8.0\">8.0</td><td style=\"text-align:left\" title=\"318.0\">318.0</td><td style=\"text-align:left\" title=\"150.0\">150.0</td><td style=\"text-align:left\" title=\"3436.0\">3436.0</td><td style=\"text-align:left\" title=\"11.0\">11.0</td><td style=\"text-align:left\" title=\"70.0\">70.0</td><td style=\"text-align:left\" title=\"1.0\">1.0</td></tr><tr><td style=\"text-align:left\" title=\"16.0\">16.0</td><td style=\"text-align:left\" title=\"8.0\">8.0</td><td style=\"text-align:left\" title=\"304.0\">304.0</td><td style=\"text-align:left\" title=\"150.0\">150.0</td><td style=\"text-align:left\" title=\"3433.0\">3433.0</td><td style=\"text-align:left\" title=\"12.0\">12.0</td><td style=\"text-align:left\" title=\"70.0\">70.0</td><td style=\"text-align:left\" title=\"1.0\">1.0</td></tr><tr><td style=\"text-align:left\" title=\"17.0\">17.0</td><td style=\"text-align:left\" title=\"8.0\">8.0</td><td style=\"text-align:left\" title=\"302.0\">302.0</td><td style=\"text-align:left\" title=\"140.0\">140.0</td><td style=\"text-align:left\" title=\"3449.0\">3449.0</td><td style=\"text-align:left\" title=\"10.5\">10.5</td><td style=\"text-align:left\" title=\"70.0\">70.0</td><td style=\"text-align:left\" title=\"1.0\">1.0</td></tr></table></body></html>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rearrange raws in random order :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "val dfSample = df.sampleN(df.nrow)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalize feature columns:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html><body><table><tr><th style=\"text-align:left\">tmp_col_57495247-e86c-4614-b8fc-a40b2c827e9d</th><th style=\"text-align:left\">tmp_col_3fa703d1-4f6e-41c6-b230-faa718f3dffc</th><th style=\"text-align:left\">tmp_col_6762b213-cc7f-4faf-9e03-400e3768b0c7</th><th style=\"text-align:left\">tmp_col_e7d7274b-db03-4ece-bc93-f25faa1eca61</th><th style=\"text-align:left\">tmp_col_89a471b6-4d62-4c98-8eb4-977d3f19c982</th><th style=\"text-align:left\">tmp_col_e4e67842-c973-478f-a5bd-286e5a9053dc</th><th style=\"text-align:left\">tmp_col_f737e930-e3f6-4e99-862b-2393c2bcfede</th></tr><tr><td style=\"text-align:left\" title=\"1.483947024779763\">1.483947024779763</td><td style=\"text-align:left\" title=\"2.483847544975299\">2.483847544975299</td><td style=\"text-align:left\" title=\"3.0053200647098817\">3.0053200647098817</td><td style=\"text-align:left\" title=\"1.6225225584708765\">1.6225225584708765</td><td style=\"text-align:left\" title=\"-2.374051701637827\">-2.374051701637827</td><td style=\"text-align:left\" title=\"-1.6253153340187592\">-1.6253153340187592</td><td style=\"text-align:left\" title=\"-0.7166410451853089\">-0.7166410451853089</td></tr><tr><td style=\"text-align:left\" title=\"0.30996673474208963\">0.30996673474208963</td><td style=\"text-align:left\" title=\"0.6084368954910576\">0.6084368954910576</td><td style=\"text-align:left\" title=\"-0.24632900724702103\">-0.24632900724702103</td><td style=\"text-align:left\" title=\"0.25393272679329487\">0.25393272679329487</td><td style=\"text-align:left\" title=\"0.819743146646186\">0.819743146646186</td><td style=\"text-align:left\" title=\"0.005547151310642964\">0.005547151310642964</td><td style=\"text-align:left\" title=\"-0.7166410451853089\">-0.7166410451853089</td></tr><tr><td style=\"text-align:left\" title=\"0.30996673474208963\">0.30996673474208963</td><td style=\"text-align:left\" title=\"0.5318895220427213\">0.5318895220427213</td><td style=\"text-align:left\" title=\"-0.8446324364870911\">-0.8446324364870911</td><td style=\"text-align:left\" title=\"0.21267463539905254\">0.21267463539905254</td><td style=\"text-align:left\" title=\"1.4367262423374154\">1.4367262423374154</td><td style=\"text-align:left\" title=\"-0.26626326291092406\">-0.26626326291092406</td><td style=\"text-align:left\" title=\"-0.7166410451853089\">-0.7166410451853089</td></tr><tr><td style=\"text-align:left\" title=\"1.483947024779763\">1.483947024779763</td><td style=\"text-align:left\" title=\"1.4983001118279682\">1.4983001118279682</td><td style=\"text-align:left\" title=\"1.1323701992627055\">1.1323701992627055</td><td style=\"text-align:left\" title=\"1.9796997496838886\">1.9796997496838886</td><td style=\"text-align:left\" title=\"-0.7408611542198661\">-0.7408611542198661</td><td style=\"text-align:left\" title=\"-0.26626326291092406\">-0.26626326291092406</td><td style=\"text-align:left\" title=\"-0.7166410451853089\">-0.7166410451853089</td></tr><tr><td style=\"text-align:left\" title=\"-0.8640135552955837\">-0.8640135552955837</td><td style=\"text-align:left\" title=\"-0.520636862871904\">-0.520636862871904</td><td style=\"text-align:left\" title=\"-0.8446324364870911\">-0.8446324364870911</td><td style=\"text-align:left\" title=\"-0.6714273230489975\">-0.6714273230489975</td><td style=\"text-align:left\" title=\"1.255260625957642\">1.255260625957642</td><td style=\"text-align:left\" title=\"-1.3535049197971922\">-1.3535049197971922</td><td style=\"text-align:left\" title=\"-0.7166410451853089\">-0.7166410451853089</td></tr></table></body></html>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fun normalizeCol(col: DoubleCol): DataCol {\n",
    "    return (col - col.mean()!!)/(col.sd()!!)\n",
    "}\n",
    "\n",
    "val dfNormed = dataFrameOf(\n",
    "    columns = dfSample.cols.drop(1).map {\n",
    "        normalizeCol (it as DoubleCol)\n",
    "    } .toTypedArray()\n",
    ")\n",
    "\n",
    "dfNormed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's put the data (features and targets) in the form `Array<FloatArray>`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "val x = dfNormed.rows.map {it.values.map {(it as Double).toFloat()}.toFloatArray() }.toTypedArray()\n",
    "val y = dfSample.select(\"mpg\").rows.map {it.values.map {(it as Double).toFloat()}.toFloatArray() }.toTypedArray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create KotlinDL `Dataset` :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import org.jetbrains.kotlinx.dl.datasets.Dataset\n",
    "val dataset = Dataset.create({x}, {y})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the dataset into a training set and a test set in a ratio of 9:1 :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "val (train, test) = dataset.split(splitRatio = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "352"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.xSize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.xSize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create neural network model with 3 hidden dense layers : "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import org.jetbrains.kotlinx.dl.api.core.Sequential\n",
    "import org.jetbrains.kotlinx.dl.api.core.layer.*\n",
    "import org.jetbrains.kotlinx.dl.api.core.activation.Activations\n",
    "\n",
    "val model = Sequential.of(\n",
    "    Input(7),\n",
    "    Dense(64), //by default activation function set to ReLU\n",
    "    Dense(64),\n",
    "    Dense(1, activation=Activations.Linear)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compile it ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import org.jetbrains.kotlinx.dl.api.core.loss.Losses\n",
    "import org.jetbrains.kotlinx.dl.api.core.metric.Metrics\n",
    "import org.jetbrains.kotlinx.dl.api.core.optimizer.RMSProp\n",
    "\n",
    "model.compile(\n",
    "    optimizer = RMSProp(),\n",
    "    loss = Losses.MAE,\n",
    "    metric = Metrics.MAE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[dense_1(Dense)               [None, 64]                512, dense_2(Dense)               [None, 64]                4160, dense_3(Dense)               [None, 1]                 65]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... and fit with train data :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "org.jetbrains.kotlinx.dl.api.core.history.TrainingHistory@10f6f870"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(\n",
    "    dataset = train,\n",
    "    epochs = 100,\n",
    "    batchSize = 32,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the test set performance :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MAE: 1.6794047355651855"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val mae = model.evaluate(\n",
    "    dataset = test,\n",
    "    batchSize = 32\n",
    ").metrics[Metrics.MAE]\n",
    "\n",
    "\"MAE: $mae\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a look at the error distribution. Draw histogram with lets-plot :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "   <div id=\"p9knCh\"></div>\n",
       "   <script type=\"text/javascript\" data-lets-plot-script=\"plot\">\n",
       "       (function() {\n",
       "           var plotSpec={\n",
       "'mapping':{\n",
       "'x':\"Absoulte error\"\n",
       "},\n",
       "'data':{\n",
       "},\n",
       "'ggsize':{\n",
       "'width':500,\n",
       "'height':250\n",
       "},\n",
       "'kind':\"plot\",\n",
       "'scales':[],\n",
       "'layers':[{\n",
       "'mapping':{\n",
       "},\n",
       "'stat':\"bin\",\n",
       "'data':{\n",
       "'Absoulte error':[-6.524182891845703,-6.024182891845703,-5.524182891845703,-5.024182891845703,-4.524182891845703,-4.024182891845703,-3.5241828918457028,-3.0241828918457028,-2.5241828918457028,-2.0241828918457028,-1.5241828918457028,-1.0241828918457028,-0.5241828918457028,-0.02418289184570277,0.47581710815429723,0.9758171081542972,1.4758171081542972,1.9758171081542972,2.4758171081542972,2.9758171081542972,3.4758171081542972,3.9758171081542972,4.475817108154297,4.975817108154297,5.475817108154297,5.975817108154297,6.475817108154297,6.975817108154297],\n",
       "'..count..':[1.0,0.0,1.0,0.0,0.0,2.0,0.0,0.0,2.0,0.0,4.0,3.0,1.0,4.0,9.0,1.0,3.0,2.0,0.0,2.0,1.0,1.0,1.0,0.0,0.0,0.0,2.0,0.0]\n",
       "},\n",
       "'position':\"stack\",\n",
       "'binwidth':0.5,\n",
       "'geom':\"histogram\"\n",
       "}]\n",
       "};\n",
       "           var plotContainer = document.getElementById(\"p9knCh\");\n",
       "           window.letsPlotCall(function() {{\n",
       "               LetsPlot.buildPlotFromProcessedSpecs(plotSpec, -1, -1, plotContainer);\n",
       "           }});\n",
       "       })();    \n",
       "   </script>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val testSize = test.xSize()\n",
    "val xTest = x.takeLast(testSize)\n",
    "val yTest = y.takeLast(testSize)\n",
    "\n",
    "val errList = mutableListOf<Float>()\n",
    "\n",
    "xTest.zip(yTest).forEach { (x, y) ->\n",
    "    val pred = model.predictSoftly((x))[0]\n",
    "    errList.add(pred - y[0])\n",
    "}\n",
    "\n",
    "val err = mapOf(\"Absoulte error\" to errList)\n",
    "\n",
    "val hist = lets_plot(err) { x = \"Absoulte error\" } + ggsize(500, 250)\n",
    "hist + geom_histogram(binWidth=0.5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Kotlin",
   "language": "kotlin",
   "name": "kotlin"
  },
  "language_info": {
   "codemirror_mode": "text/x-kotlin",
   "file_extension": ".kt",
   "mimetype": "text/x-kotlin",
   "name": "kotlin",
   "nbconvert_exporter": "",
   "pygments_lexer": "kotlin",
   "version": "1.5.20-dev-3998"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}