6.5.ram-training-on-bach.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training RAM on BACH\n",
    "\n",
    "## Requirements\n",
    "\n",
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "# 2019041500 - use this tf nightly version\n",
    "import tensorflow as tf\n",
    "import logging\n",
    "tf.get_logger().setLevel(logging.ERROR)\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from model.ram import RecurrentAttentionModel\n",
    "\n",
    "from data.bach_loader import minibatcher\n",
    "from data.bach_loader import load_bach_images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), X_test, _ = load_bach_images(\"/tf/BACH\")\n",
    "\n",
    "X_train = X_train.reshape(-1, 1536, 2048, 3)\n",
    "X_test = X_test.reshape(-1, 1536, 2048, 3)\n",
    "    \n",
    "X_train = (X_train/255).astype(np.float32)\n",
    "X_test = (X_test/255).astype(np.float32)\n",
    "\n",
    "y_train = tf.keras.utils.to_categorical(y_train)\n",
    "\n",
    "print(X_train.shape, y_train.shape, np.max(X_train), np.min(X_train), X_train.dtype)\n",
    "print(X_test.shape, np.max(X_test), np.min(X_test), X_test.dtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trainings\n",
    "### Hyperparameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 10\n",
    "\n",
    "learning_rate = 0.0001\n",
    "std = 0.2\n",
    "\n",
    "ram = RecurrentAttentionModel(time_steps=8,\n",
    "                              n_glimpses=3, \n",
    "                              glimpse_size=256,\n",
    "                              num_classes=4,\n",
    "                              max_gradient_norm=1.0,\n",
    "                              input_channels=3,\n",
    "                              std=std)\n",
    "adam_opt = tf.keras.optimizers.Adam(learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trainingsloop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "history = []\n",
    "for timestep in tqdm(range(500)):\n",
    "    losses = []\n",
    "    rewards = []\n",
    "    classification_losses = []\n",
    "    \n",
    "    # training steps\n",
    "    batcher = minibatcher(X_train, y_train, batch_size, True)\n",
    "    for X, y in batcher:\n",
    "        with tf.GradientTape() as tape:\n",
    "            # calculate losses\n",
    "            logits = ram(X)\n",
    "            loss, classification_loss, reward, _ = ram.hybrid_loss(logits, y)\n",
    "            \n",
    "            # append to list for output\n",
    "            losses.append(loss.numpy())\n",
    "            classification_losses.append(classification_loss.numpy())\n",
    "            rewards.append(reward.numpy())\n",
    "            \n",
    "            # calculate gradient and do gradient descent\n",
    "            gradients = tape.gradient(loss, ram.trainable_variables)\n",
    "            adam_opt.apply_gradients(zip(gradients, ram.trainable_variables))\n",
    "        \n",
    "    print(\"step:\", timestep, \"loss:\", np.mean(losses), \"classification_loss:\", np.mean(classification_losses), \n",
    "          \"reward:\", np.mean(rewards))\n",
    "    history.append([(np.mean(losses), np.mean(classification_losses), np.mean(rewards))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = np.array(history).reshape(-1, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.title(\"hybrid loss\")\n",
    "plt.plot(history.T[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.title(\"classification loss\")\n",
    "plt.plot(history.T[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.title(\"reward\")\n",
    "plt.plot(history.T[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_path_of(number, batch):\n",
    "    from visualization.model import plot_prediction_path_3d\n",
    "    imgs = X_train[batch*batch_size:batch*batch_size + batch_size]\n",
    "    labels = y_train[batch*batch_size:batch*batch_size + batch_size]\n",
    "    logits = ram(imgs)\n",
    "    _, prediction, location = ram.predict(logits, labels)\n",
    "    labels = np.argmax(labels, 1)\n",
    "    for i, (y, y_hat) in enumerate(zip(list(prediction.numpy()), list(labels))):\n",
    "        if y == y_hat & y == number:\n",
    "            loc = location[i].numpy()\n",
    "            img = imgs[i]\n",
    "            print(\"right\")\n",
    "            plot_prediction_path_3d(img, loc, 3, 64)\n",
    "        if y != y_hat & y == number:\n",
    "            print(\"wrong\")\n",
    "            loc = location[i].numpy()\n",
    "            img = imgs[i]\n",
    "            plot_prediction_path_3d(img, loc, 3, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_path_of(0, 1)\n",
    "plot_path_of(0, 2)\n",
    "plot_path_of(0, 3)\n",
    "plot_path_of(0, 4)\n",
    "plot_path_of(0, 5)\n",
    "plot_path_of(0, 6)\n",
    "plot_path_of(0, 7)\n",
    "plot_path_of(0, 8)\n",
    "plot_path_of(0, 9)\n",
    "plot_path_of(0, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_path_of(1, 1)\n",
    "plot_path_of(1, 2)\n",
    "plot_path_of(1, 3)\n",
    "plot_path_of(1, 4)\n",
    "plot_path_of(1, 5)\n",
    "plot_path_of(1, 6)\n",
    "plot_path_of(1, 7)\n",
    "plot_path_of(1, 8)\n",
    "plot_path_of(1, 9)\n",
    "plot_path_of(1, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_path_of(2, 11)\n",
    "plot_path_of(2, 12)\n",
    "plot_path_of(2, 13)\n",
    "plot_path_of(2, 14)\n",
    "plot_path_of(2, 15)\n",
    "plot_path_of(2, 16)\n",
    "plot_path_of(2, 17)\n",
    "plot_path_of(2, 18)\n",
    "plot_path_of(2, 19)\n",
    "plot_path_of(2, 20)\n",
    "plot_path_of(2, 21)\n",
    "plot_path_of(2, 22)\n",
    "plot_path_of(2, 23)\n",
    "plot_path_of(2, 24)\n",
    "plot_path_of(2, 25)\n",
    "plot_path_of(2, 26)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_path_of(3, 21)\n",
    "plot_path_of(3, 22)\n",
    "plot_path_of(3, 23)\n",
    "plot_path_of(3, 24)\n",
    "plot_path_of(3, 25)\n",
    "plot_path_of(3, 26)\n",
    "plot_path_of(3, 27)\n",
    "plot_path_of(3, 28)\n",
    "plot_path_of(3, 29)\n",
    "plot_path_of(3, 30)\n",
    "plot_path_of(3, 31)\n",
    "plot_path_of(3, 32)\n",
    "plot_path_of(3, 39)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "notify_time": "0",
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "176px",
    "left": "909px",
    "right": "50px",
    "top": "121px",
    "width": "321px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}