notebook/14_keras_learning_rate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas\n",
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load dataset\n",
    "dataframe = pandas.read_csv(\"ionosphere.csv\", header=None)\n",
    "dataset = dataframe.values\n",
    "\n",
    "# split into input (X) and output (Y) variables\n",
    "X = dataset[:,0:34].astype(float)\n",
    "Y = dataset[:,34]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.     ,  0.     ,  0.99539, -0.05889,  0.85243,  0.02306,\n",
       "         0.83398, -0.37708,  1.     ,  0.0376 ,  0.85243, -0.17755,\n",
       "         0.59755, -0.44945,  0.60536, -0.38223,  0.84356, -0.38542,\n",
       "         0.58212, -0.32192,  0.56971, -0.29674,  0.36946, -0.47357,\n",
       "         0.56811, -0.51171,  0.41078, -0.46168,  0.21266, -0.3409 ,\n",
       "         0.42267, -0.54487,  0.18641, -0.453  ],\n",
       "       [ 1.     ,  0.     ,  1.     , -0.18829,  0.93035, -0.36156,\n",
       "        -0.10868, -0.93597,  1.     , -0.04549,  0.50874, -0.67743,\n",
       "         0.34432, -0.69707, -0.51685, -0.97515,  0.05499, -0.62237,\n",
       "         0.33109, -1.     , -0.13151, -0.453  , -0.18056, -0.35734,\n",
       "        -0.20332, -0.26569, -0.20468, -0.18401, -0.1904 , -0.11593,\n",
       "        -0.16626, -0.06288, -0.13738, -0.02447],\n",
       "       [ 1.     ,  0.     ,  1.     , -0.03365,  1.     ,  0.00485,\n",
       "         1.     , -0.12062,  0.88965,  0.01198,  0.73082,  0.05346,\n",
       "         0.85443,  0.00827,  0.54591,  0.00299,  0.83775, -0.13644,\n",
       "         0.75535, -0.0854 ,  0.70887, -0.27502,  0.43385, -0.12062,\n",
       "         0.57528, -0.4022 ,  0.58984, -0.22145,  0.431  , -0.17365,\n",
       "         0.60436, -0.2418 ,  0.56045, -0.38238]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['g', 'b', 'g'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# encode class values as integers\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(Y)\n",
    "Y = encoder.transform(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create model\n",
    "model = Sequential()\n",
    "model.add(Dense(34, input_dim=34, init='normal', activation='relu'))\n",
    "model.add(Dense(1, init='normal', activation='sigmoid'))\n",
    "\n",
    "# compile model\n",
    "epochs = 50\n",
    "learning_rate = 0.1\n",
    "decay_rate = learning_rate / epochs\n",
    "momentum = 0.8\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 235 samples, validate on 116 samples\n",
      "Epoch 1/50\n",
      "235/235 [==============================] - 0s - loss: 0.6895 - acc: 0.5617 - val_loss: 0.6670 - val_acc: 0.8707\n",
      "Epoch 2/50\n",
      "235/235 [==============================] - 0s - loss: 0.6584 - acc: 0.7447 - val_loss: 0.5859 - val_acc: 0.9052\n",
      "Epoch 3/50\n",
      "235/235 [==============================] - 0s - loss: 0.5850 - acc: 0.8043 - val_loss: 0.4708 - val_acc: 0.8966\n",
      "Epoch 4/50\n",
      "235/235 [==============================] - 0s - loss: 0.4876 - acc: 0.8298 - val_loss: 0.4661 - val_acc: 0.9310\n",
      "Epoch 5/50\n",
      "235/235 [==============================] - 0s - loss: 0.4001 - acc: 0.8596 - val_loss: 0.3011 - val_acc: 0.9483\n",
      "Epoch 6/50\n",
      "235/235 [==============================] - 0s - loss: 0.3266 - acc: 0.8809 - val_loss: 0.4153 - val_acc: 0.8707\n",
      "Epoch 7/50\n",
      "235/235 [==============================] - 0s - loss: 0.2825 - acc: 0.8979 - val_loss: 0.2238 - val_acc: 0.9569\n",
      "Epoch 8/50\n",
      "235/235 [==============================] - 0s - loss: 0.2436 - acc: 0.9149 - val_loss: 0.1475 - val_acc: 0.9569\n",
      "Epoch 9/50\n",
      "235/235 [==============================] - 0s - loss: 0.2378 - acc: 0.9149 - val_loss: 0.2304 - val_acc: 0.9397\n",
      "Epoch 10/50\n",
      "235/235 [==============================] - 0s - loss: 0.2001 - acc: 0.9277 - val_loss: 0.2435 - val_acc: 0.9224\n",
      "Epoch 11/50\n",
      "235/235 [==============================] - 0s - loss: 0.1885 - acc: 0.9319 - val_loss: 0.1655 - val_acc: 0.9655\n",
      "Epoch 12/50\n",
      "235/235 [==============================] - 0s - loss: 0.1673 - acc: 0.9447 - val_loss: 0.1130 - val_acc: 0.9655\n",
      "Epoch 13/50\n",
      "235/235 [==============================] - 0s - loss: 0.1772 - acc: 0.9319 - val_loss: 0.1101 - val_acc: 0.9655\n",
      "Epoch 14/50\n",
      "235/235 [==============================] - 0s - loss: 0.1592 - acc: 0.9489 - val_loss: 0.1646 - val_acc: 0.9483\n",
      "Epoch 15/50\n",
      "235/235 [==============================] - 0s - loss: 0.1386 - acc: 0.9617 - val_loss: 0.0975 - val_acc: 0.9741\n",
      "Epoch 16/50\n",
      "235/235 [==============================] - 0s - loss: 0.1418 - acc: 0.9404 - val_loss: 0.1602 - val_acc: 0.9483\n",
      "Epoch 17/50\n",
      "235/235 [==============================] - 0s - loss: 0.1374 - acc: 0.9489 - val_loss: 0.1437 - val_acc: 0.9655\n",
      "Epoch 18/50\n",
      "235/235 [==============================] - 0s - loss: 0.1292 - acc: 0.9574 - val_loss: 0.1229 - val_acc: 0.9741\n",
      "Epoch 19/50\n",
      "235/235 [==============================] - 0s - loss: 0.1217 - acc: 0.9617 - val_loss: 0.0912 - val_acc: 0.9914\n",
      "Epoch 20/50\n",
      "235/235 [==============================] - 0s - loss: 0.1150 - acc: 0.9702 - val_loss: 0.1151 - val_acc: 0.9828\n",
      "Epoch 21/50\n",
      "235/235 [==============================] - 0s - loss: 0.1090 - acc: 0.9617 - val_loss: 0.1010 - val_acc: 0.9914\n",
      "Epoch 22/50\n",
      "235/235 [==============================] - 0s - loss: 0.1043 - acc: 0.9660 - val_loss: 0.1058 - val_acc: 0.9828\n",
      "Epoch 23/50\n",
      "235/235 [==============================] - 0s - loss: 0.1036 - acc: 0.9702 - val_loss: 0.1109 - val_acc: 0.9828\n",
      "Epoch 24/50\n",
      "235/235 [==============================] - 0s - loss: 0.0974 - acc: 0.9745 - val_loss: 0.0824 - val_acc: 0.9914\n",
      "Epoch 25/50\n",
      "235/235 [==============================] - 0s - loss: 0.1032 - acc: 0.9702 - val_loss: 0.1000 - val_acc: 0.9914\n",
      "Epoch 26/50\n",
      "235/235 [==============================] - 0s - loss: 0.0927 - acc: 0.9745 - val_loss: 0.0941 - val_acc: 0.9914\n",
      "Epoch 27/50\n",
      "235/235 [==============================] - 0s - loss: 0.0907 - acc: 0.9787 - val_loss: 0.0950 - val_acc: 0.9914\n",
      "Epoch 28/50\n",
      "235/235 [==============================] - 0s - loss: 0.0879 - acc: 0.9830 - val_loss: 0.0908 - val_acc: 0.9914\n",
      "Epoch 29/50\n",
      "235/235 [==============================] - 0s - loss: 0.0812 - acc: 0.9830 - val_loss: 0.0921 - val_acc: 0.9914\n",
      "Epoch 30/50\n",
      "235/235 [==============================] - 0s - loss: 0.0820 - acc: 0.9830 - val_loss: 0.0962 - val_acc: 0.9828\n",
      "Epoch 31/50\n",
      "235/235 [==============================] - 0s - loss: 0.0798 - acc: 0.9830 - val_loss: 0.0901 - val_acc: 0.9914\n",
      "Epoch 32/50\n",
      "235/235 [==============================] - 0s - loss: 0.0777 - acc: 0.9830 - val_loss: 0.0918 - val_acc: 0.9914\n",
      "Epoch 33/50\n",
      "235/235 [==============================] - 0s - loss: 0.0728 - acc: 0.9830 - val_loss: 0.0942 - val_acc: 0.9828\n",
      "Epoch 34/50\n",
      "235/235 [==============================] - 0s - loss: 0.0761 - acc: 0.9830 - val_loss: 0.0993 - val_acc: 0.9828\n",
      "Epoch 35/50\n",
      "235/235 [==============================] - 0s - loss: 0.0722 - acc: 0.9830 - val_loss: 0.0830 - val_acc: 0.9914\n",
      "Epoch 36/50\n",
      "235/235 [==============================] - 0s - loss: 0.0707 - acc: 0.9830 - val_loss: 0.0866 - val_acc: 0.9914\n",
      "Epoch 37/50\n",
      "235/235 [==============================] - 0s - loss: 0.0677 - acc: 0.9830 - val_loss: 0.0930 - val_acc: 0.9914\n",
      "Epoch 38/50\n",
      "235/235 [==============================] - 0s - loss: 0.0682 - acc: 0.9830 - val_loss: 0.0793 - val_acc: 0.9914\n",
      "Epoch 39/50\n",
      "235/235 [==============================] - 0s - loss: 0.0714 - acc: 0.9830 - val_loss: 0.0913 - val_acc: 0.9828\n",
      "Epoch 40/50\n",
      "235/235 [==============================] - 0s - loss: 0.0671 - acc: 0.9830 - val_loss: 0.0797 - val_acc: 0.9914\n",
      "Epoch 41/50\n",
      "235/235 [==============================] - 0s - loss: 0.0631 - acc: 0.9830 - val_loss: 0.0885 - val_acc: 0.9914\n",
      "Epoch 42/50\n",
      "235/235 [==============================] - 0s - loss: 0.0643 - acc: 0.9830 - val_loss: 0.0883 - val_acc: 0.9914\n",
      "Epoch 43/50\n",
      "235/235 [==============================] - 0s - loss: 0.0635 - acc: 0.9830 - val_loss: 0.0840 - val_acc: 0.9914\n",
      "Epoch 44/50\n",
      "235/235 [==============================] - 0s - loss: 0.0629 - acc: 0.9830 - val_loss: 0.0741 - val_acc: 0.9914\n",
      "Epoch 45/50\n",
      "235/235 [==============================] - 0s - loss: 0.0622 - acc: 0.9830 - val_loss: 0.1029 - val_acc: 0.9741\n",
      "Epoch 46/50\n",
      "235/235 [==============================] - 0s - loss: 0.0681 - acc: 0.9787 - val_loss: 0.0725 - val_acc: 0.9914\n",
      "Epoch 47/50\n",
      "235/235 [==============================] - 0s - loss: 0.0660 - acc: 0.9872 - val_loss: 0.0686 - val_acc: 0.9914\n",
      "Epoch 48/50\n",
      "235/235 [==============================] - 0s - loss: 0.0626 - acc: 0.9830 - val_loss: 0.0895 - val_acc: 0.9828\n",
      "Epoch 49/50\n",
      "235/235 [==============================] - 0s - loss: 0.0566 - acc: 0.9830 - val_loss: 0.0837 - val_acc: 0.9914\n",
      "Epoch 50/50\n",
      "235/235 [==============================] - 0s - loss: 0.0576 - acc: 0.9830 - val_loss: 0.0712 - val_acc: 0.9914\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1069b6090>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit the model\n",
    "model.fit(X, Y, validation_split=0.33, nb_epoch=epochs, batch_size=28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import numpy\n",
    "import math\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.callbacks import LearningRateScheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# learning rate schedule\n",
    "def step_decay(epoch):\n",
    "    initial_lrate = 0.1\n",
    "    drop = 0.5\n",
    "    epochs_drop = 10.0\n",
    "    lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop))\n",
    "    return lrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# load dataset\n",
    "dataframe = pandas.read_csv(\"ionosphere.csv\", header=None)\n",
    "dataset = dataframe.values\n",
    "\n",
    "# split into input (X) and output (Y) variables\n",
    "X = dataset[:,0:34].astype(float)\n",
    "Y = dataset[:,34]\n",
    "\n",
    "# encode class values as integers\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(Y)\n",
    "Y = encoder.transform(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create model\n",
    "model = Sequential()\n",
    "model.add(Dense(34, input_dim=34, init='normal', activation='relu'))\n",
    "model.add(Dense(1, init='normal', activation='sigmoid'))\n",
    "\n",
    "# compile model\n",
    "sgd = SGD(lr=0.0, momentum=0.9, decay=0.0, nesterov=False)\n",
    "model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "\n",
    "# learning schedule callback\n",
    "lrate = LearningRateScheduler(step_decay)\n",
    "callbacks_list = [lrate]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 235 samples, validate on 116 samples\n",
      "Epoch 1/50\n",
      "235/235 [==============================] - 0s - loss: 0.6891 - acc: 0.4894 - val_loss: 0.6179 - val_acc: 0.9397\n",
      "Epoch 2/50\n",
      "235/235 [==============================] - 0s - loss: 0.6312 - acc: 0.7021 - val_loss: 0.4865 - val_acc: 0.7586\n",
      "Epoch 3/50\n",
      "235/235 [==============================] - 0s - loss: 0.5214 - acc: 0.8043 - val_loss: 0.4297 - val_acc: 0.9310\n",
      "Epoch 4/50\n",
      "235/235 [==============================] - 0s - loss: 0.3792 - acc: 0.8681 - val_loss: 0.2425 - val_acc: 0.9483\n",
      "Epoch 5/50\n",
      "235/235 [==============================] - 0s - loss: 0.2703 - acc: 0.8979 - val_loss: 0.5098 - val_acc: 0.7069\n",
      "Epoch 6/50\n",
      "235/235 [==============================] - 0s - loss: 0.3106 - acc: 0.8553 - val_loss: 0.4310 - val_acc: 0.8103\n",
      "Epoch 7/50\n",
      "235/235 [==============================] - 0s - loss: 0.2246 - acc: 0.9191 - val_loss: 0.1191 - val_acc: 0.9655\n",
      "Epoch 8/50\n",
      "235/235 [==============================] - 0s - loss: 0.1789 - acc: 0.9277 - val_loss: 0.1304 - val_acc: 0.9741\n",
      "Epoch 9/50\n",
      "235/235 [==============================] - 0s - loss: 0.1823 - acc: 0.9149 - val_loss: 0.1746 - val_acc: 0.9655\n",
      "Epoch 10/50\n",
      "235/235 [==============================] - 0s - loss: 0.1338 - acc: 0.9617 - val_loss: 0.0990 - val_acc: 0.9914\n",
      "Epoch 11/50\n",
      "235/235 [==============================] - 0s - loss: 0.1255 - acc: 0.9617 - val_loss: 0.1199 - val_acc: 0.9741\n",
      "Epoch 12/50\n",
      "235/235 [==============================] - 0s - loss: 0.1112 - acc: 0.9702 - val_loss: 0.0870 - val_acc: 0.9914\n",
      "Epoch 13/50\n",
      "235/235 [==============================] - 0s - loss: 0.1063 - acc: 0.9702 - val_loss: 0.1086 - val_acc: 0.9914\n",
      "Epoch 14/50\n",
      "235/235 [==============================] - 0s - loss: 0.1068 - acc: 0.9660 - val_loss: 0.0927 - val_acc: 0.9914\n",
      "Epoch 15/50\n",
      "235/235 [==============================] - 0s - loss: 0.0971 - acc: 0.9830 - val_loss: 0.0839 - val_acc: 0.9914\n",
      "Epoch 16/50\n",
      "235/235 [==============================] - 0s - loss: 0.0941 - acc: 0.9787 - val_loss: 0.0929 - val_acc: 0.9914\n",
      "Epoch 17/50\n",
      "235/235 [==============================] - 0s - loss: 0.0834 - acc: 0.9830 - val_loss: 0.0797 - val_acc: 0.9914\n",
      "Epoch 18/50\n",
      "235/235 [==============================] - 0s - loss: 0.0813 - acc: 0.9830 - val_loss: 0.0771 - val_acc: 0.9914\n",
      "Epoch 19/50\n",
      "235/235 [==============================] - 0s - loss: 0.0772 - acc: 0.9830 - val_loss: 0.0874 - val_acc: 0.9914\n",
      "Epoch 20/50\n",
      "235/235 [==============================] - 0s - loss: 0.0733 - acc: 0.9787 - val_loss: 0.0770 - val_acc: 0.9914\n",
      "Epoch 21/50\n",
      "235/235 [==============================] - 0s - loss: 0.0713 - acc: 0.9830 - val_loss: 0.0754 - val_acc: 0.9914\n",
      "Epoch 22/50\n",
      "235/235 [==============================] - 0s - loss: 0.0689 - acc: 0.9830 - val_loss: 0.0776 - val_acc: 0.9914\n",
      "Epoch 23/50\n",
      "235/235 [==============================] - 0s - loss: 0.0680 - acc: 0.9830 - val_loss: 0.0768 - val_acc: 0.9914\n",
      "Epoch 24/50\n",
      "235/235 [==============================] - 0s - loss: 0.0667 - acc: 0.9830 - val_loss: 0.0812 - val_acc: 0.9914\n",
      "Epoch 25/50\n",
      "235/235 [==============================] - 0s - loss: 0.0668 - acc: 0.9830 - val_loss: 0.0729 - val_acc: 0.9914\n",
      "Epoch 26/50\n",
      "235/235 [==============================] - 0s - loss: 0.0667 - acc: 0.9830 - val_loss: 0.0757 - val_acc: 0.9914\n",
      "Epoch 27/50\n",
      "235/235 [==============================] - 0s - loss: 0.0633 - acc: 0.9830 - val_loss: 0.0687 - val_acc: 0.9914\n",
      "Epoch 28/50\n",
      "235/235 [==============================] - 0s - loss: 0.0610 - acc: 0.9830 - val_loss: 0.0786 - val_acc: 0.9914\n",
      "Epoch 29/50\n",
      "235/235 [==============================] - 0s - loss: 0.0618 - acc: 0.9830 - val_loss: 0.0774 - val_acc: 0.9914\n",
      "Epoch 30/50\n",
      "235/235 [==============================] - 0s - loss: 0.0589 - acc: 0.9830 - val_loss: 0.0688 - val_acc: 0.9914\n",
      "Epoch 31/50\n",
      "235/235 [==============================] - 0s - loss: 0.0589 - acc: 0.9830 - val_loss: 0.0660 - val_acc: 0.9914\n",
      "Epoch 32/50\n",
      "235/235 [==============================] - 0s - loss: 0.0579 - acc: 0.9830 - val_loss: 0.0718 - val_acc: 0.9914\n",
      "Epoch 33/50\n",
      "235/235 [==============================] - 0s - loss: 0.0577 - acc: 0.9830 - val_loss: 0.0807 - val_acc: 0.9914\n",
      "Epoch 34/50\n",
      "235/235 [==============================] - 0s - loss: 0.0581 - acc: 0.9830 - val_loss: 0.0692 - val_acc: 0.9914\n",
      "Epoch 35/50\n",
      "235/235 [==============================] - 0s - loss: 0.0561 - acc: 0.9830 - val_loss: 0.0688 - val_acc: 0.9914\n",
      "Epoch 36/50\n",
      "235/235 [==============================] - 0s - loss: 0.0557 - acc: 0.9830 - val_loss: 0.0694 - val_acc: 0.9914\n",
      "Epoch 37/50\n",
      "235/235 [==============================] - 0s - loss: 0.0557 - acc: 0.9830 - val_loss: 0.0700 - val_acc: 0.9914\n",
      "Epoch 38/50\n",
      "235/235 [==============================] - 0s - loss: 0.0549 - acc: 0.9830 - val_loss: 0.0694 - val_acc: 0.9914\n",
      "Epoch 39/50\n",
      "235/235 [==============================] - 0s - loss: 0.0546 - acc: 0.9830 - val_loss: 0.0707 - val_acc: 0.9914\n",
      "Epoch 40/50\n",
      "235/235 [==============================] - 0s - loss: 0.0543 - acc: 0.9830 - val_loss: 0.0670 - val_acc: 0.9914\n",
      "Epoch 41/50\n",
      "235/235 [==============================] - 0s - loss: 0.0532 - acc: 0.9830 - val_loss: 0.0711 - val_acc: 0.9914\n",
      "Epoch 42/50\n",
      "235/235 [==============================] - 0s - loss: 0.0532 - acc: 0.9830 - val_loss: 0.0718 - val_acc: 0.9914\n",
      "Epoch 43/50\n",
      "235/235 [==============================] - 0s - loss: 0.0534 - acc: 0.9830 - val_loss: 0.0724 - val_acc: 0.9914\n",
      "Epoch 44/50\n",
      "235/235 [==============================] - 0s - loss: 0.0527 - acc: 0.9830 - val_loss: 0.0686 - val_acc: 0.9914\n",
      "Epoch 45/50\n",
      "235/235 [==============================] - 0s - loss: 0.0525 - acc: 0.9830 - val_loss: 0.0681 - val_acc: 0.9914\n",
      "Epoch 46/50\n",
      "235/235 [==============================] - 0s - loss: 0.0524 - acc: 0.9830 - val_loss: 0.0665 - val_acc: 0.9914\n",
      "Epoch 47/50\n",
      "235/235 [==============================] - 0s - loss: 0.0523 - acc: 0.9830 - val_loss: 0.0656 - val_acc: 0.9914\n",
      "Epoch 48/50\n",
      "235/235 [==============================] - 0s - loss: 0.0523 - acc: 0.9830 - val_loss: 0.0684 - val_acc: 0.9914\n",
      "Epoch 49/50\n",
      "235/235 [==============================] - 0s - loss: 0.0517 - acc: 0.9830 - val_loss: 0.0677 - val_acc: 0.9914\n",
      "Epoch 50/50\n",
      "235/235 [==============================] - 0s - loss: 0.0514 - acc: 0.9830 - val_loss: 0.0685 - val_acc: 0.9914\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x119faef50>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit the model\n",
    "model.fit(X, Y, validation_split=0.33, nb_epoch=50, batch_size=28, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}