combined-v.01-light-underfit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import time\n",
    "import nltk\n",
    "import numpy\n",
    "import pandas\n",
    "import torch.nn\n",
    "import statistics\n",
    "import torch.utils\n",
    "import sklearn.metrics\n",
    "import sklearn.model_selection\n",
    "import sklearn.feature_extraction.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dataframe = pandas.read_csv('onion-or-not.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_vector = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in input_dataframe.index:\n",
    "    tokenized_vector[i] = nltk.word_tokenize(input_dataframe.loc[i][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = nltk.PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = set(nltk.corpus.stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tokens in tokenized_vector:\n",
    "    for counter, token in enumerate(tokenized_vector[tokens]):\n",
    "        if stemmer.stem(tokenized_vector[tokens][counter]) not in stopwords:\n",
    "            tokenized_vector[tokens][counter] = stemmer.stem(tokenized_vector\n",
    "                                                             [tokens][counter])\n",
    "        else:\n",
    "            tokenized_vector[tokens].remove(tokenized_vector[tokens][counter])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_tokenized_vector = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in tokenized_vector:\n",
    "    preprocessed_tokenized_vector.append(' '.join(tokenized_vector[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vectorizer.fit_transform(preprocessed_tokenized_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_idf_df = pandas.DataFrame(X.todense(),\n",
    "                             columns=vectorizer.get_feature_names(),\n",
    "                             dtype=numpy.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_data = pandas.concat([tf_idf_df, input_dataframe.iloc[:, 1:]],\n",
    "                                  axis=1, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total size of dataframe:  1012.99 MB\n"
     ]
    }
   ],
   "source": [
    "print('Total size of dataframe: ',\n",
    "      round(sys.getsizeof(preprocessed_data) / 2**20, 2), 'MB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "del tf_idf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "del preprocessed_tokenized_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "del tokenized_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "del input_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = '/home/andreas/Documents/data_mining/project/onion/input/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = pandas.DataFrame(preprocessed_data.columns[:-1].tolist(),\n",
    "                        columns=['tokens'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = preprocessed_data.iloc[:, :-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = preprocessed_data.iloc[:, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "del preprocessed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_fit, x_test, y_fit, y_test = \\\n",
    "    sklearn.model_selection.train_test_split(X, Y,\n",
    "                                             test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_val, y_train, y_val = \\\n",
    "    sklearn.model_selection.train_test_split(x_fit,\n",
    "                                             y_fit,\n",
    "                                             test_size=0.10,\n",
    "                                             random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = torch.from_numpy(x_train.to_numpy()).float()\n",
    "y_train = torch.from_numpy(y_train.to_numpy()).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = torch.utils.data.TensorDataset(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_val = torch.from_numpy(x_val.to_numpy()).float()\n",
    "y_val = torch.from_numpy(y_val.to_numpy()).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset = torch.utils.data.TensorDataset(x_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = torch.from_numpy(x_test.to_numpy()).float()\n",
    "y_test = torch.from_numpy(y_test.to_numpy()).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = torch.utils.data.TensorDataset(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(train_dataset)\n",
    "val_loader = torch.utils.data.DataLoader(val_dataset)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "del val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.from_numpy(X.to_numpy()).float()\n",
    "Y = torch.from_numpy(Y.to_numpy()).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(X.shape[1], 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 16),\n",
    "    torch.nn.Dropout(0.1),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(16, 1),\n",
    "    torch.nn.Sigmoid()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(),\n",
    "                             lr=learning_rate,\n",
    "                             weight_decay=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_device():\n",
    "    device = None\n",
    "    if torch.cuda.is_available():\n",
    "        device = torch.device('cuda')\n",
    "    else:\n",
    "        device = torch.device('cpu')\n",
    "    return device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = get_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=22125, out_features=64, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=64, out_features=16, bias=True)\n",
       "  (3): Dropout(p=0.1, inplace=False)\n",
       "  (4): ReLU()\n",
       "  (5): Linear(in_features=16, out_features=1, bias=True)\n",
       "  (6): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stopping = False\n",
    "# initialize early stopping prevention limit\n",
    "prevent = 5\n",
    "# initialize early stopping prevention limit\n",
    "consecutive = False\n",
    "# initialize early stopping message\n",
    "message = ' '\n",
    "# initialize epoch counter\n",
    "epoch = 0\n",
    "# number of epochs to train the model\n",
    "epochs = 500\n",
    "# initialize variables\n",
    "prev_mean_valid_loss = numpy.Inf\n",
    "start = 0\n",
    "# initialize error lists\n",
    "train_loss = []\n",
    "valid_loss = []\n",
    "history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  0  (in seconds)\n",
      "Epoch:  2 \t Time: + 52.68909525871277 \t Training        loss:  0.6697393896347946 \t Validation loss:  0.6709876214464505\n",
      "Epoch:  4 \t Time: + 54.70188903808594 \t Training        loss:  0.6693080399396979 \t Validation loss:  0.6708298934996129\n",
      "Epoch:  6 \t Time: + 52.84192514419556 \t Training        loss:  0.6691505167734476 \t Validation loss:  0.6707720929273853\n",
      "Epoch:  8 \t Time: + 53.11253118515015 \t Training        loss:  0.6690712356466202 \t Validation loss:  0.67074298289501\n",
      "Epoch:  10 \t Time: + 53.518542528152466 \t Training        loss:  0.6690236478032153 \t Validation loss:  0.6707255162199338\n",
      "Epoch:  12 \t Time: + 55.48626351356506 \t Training        loss:  0.6689919226604357 \t Validation loss:  0.6707138774737164\n",
      "Epoch:  14 \t Time: + 54.901580572128296 \t Training        loss:  0.6689692605545702 \t Validation loss:  0.6707055640835611\n",
      "Epoch:  16 \t Time: + 53.245259046554565 \t Training        loss:  0.6689522648041631 \t Validation loss:  0.6706993290409445\n",
      "Epoch:  18 \t Time: + 54.66584229469299 \t Training        loss:  0.6689390453896569 \t Validation loss:  0.6706944795633539\n",
      "Epoch:  20 \t Time: + 55.63969802856445 \t Training        loss:  0.6689284700447763 \t Validation loss:  0.6706905999812814\n",
      "Epoch:  22 \t Time: + 53.864062786102295 \t Training        loss:  0.6689198171099027 \t Validation loss:  0.6706874257777676\n",
      "Epoch:  24 \t Time: + 52.00634050369263 \t Training        loss:  0.6689126068007185 \t Validation loss:  0.6706847806081727\n",
      "Epoch:  26 \t Time: + 55.281638383865356 \t Training        loss:  0.6689065058639756 \t Validation loss:  0.6706825423877464\n",
      "Epoch:  28 \t Time: + 53.3165225982666 \t Training        loss:  0.6689012772261413 \t Validation loss:  0.6706806239130951\n",
      "Epoch:  30 \t Time: + 52.95462679862976 \t Training        loss:  0.6688967452687 \t Validation loss:  0.6706789612350641\n",
      "Epoch:  32 \t Time: + 52.86055111885071 \t Training        loss:  0.6688927799310351 \t Validation loss:  0.6706775063917869\n",
      "Epoch:  34 \t Time: + 53.547465562820435 \t Training        loss:  0.6688892807150848 \t Validation loss:  0.6706762227065423\n",
      "Epoch:  36 \t Time: + 53.26141428947449 \t Training        loss:  0.6688861705377105 \t Validation loss:  0.6706750816529916\n",
      "Epoch:  38 \t Time: + 52.5032639503479 \t Training        loss:  0.6688833878844337 \t Validation loss:  0.6706740607103409\n",
      "Epoch:  40 \t Time: + 52.84576916694641 \t Training        loss:  0.6688808832743469 \t Validation loss:  0.6706731418619554\n",
      "Epoch:  42 \t Time: + 52.656673192977905 \t Training        loss:  0.6688786170746844 \t Validation loss:  0.67067230931429\n",
      "Epoch:  44 \t Time: + 52.84081792831421 \t Training        loss:  0.6688765569291298 \t Validation loss:  0.6706715536064873\n",
      "Epoch:  46 \t Time: + 52.74382829666138 \t Training        loss:  0.6688746760947148 \t Validation loss:  0.6706708636124065\n",
      "Epoch:  48 \t Time: + 54.48318362236023 \t Training        loss:  0.6688729517629424 \t Validation loss:  0.6706702300602639\n",
      "Epoch:  50 \t Time: + 57.5406289100647 \t Training        loss:  0.6688713652119225 \t Validation loss:  0.6706696482075585\n",
      "Epoch:  52 \t Time: + 57.22856903076172 \t Training        loss:  0.6688699011757956 \t Validation loss:  0.6706691101365364\n",
      "Epoch:  54 \t Time: + 59.18875789642334 \t Training        loss:  0.6688685453995538 \t Validation loss:  0.6706686119226272\n",
      "Epoch:  56 \t Time: + 57.157264947891235 \t Training        loss:  0.6688672865550092 \t Validation loss:  0.670668150201913\n",
      "Epoch:  58 \t Time: + 57.169820070266724 \t Training        loss:  0.6688661142969294 \t Validation loss:  0.6706677203240066\n",
      "Epoch:  60 \t Time: + 54.3676974773407 \t Training        loss:  0.6688650203199421 \t Validation loss:  0.6706673191046273\n",
      "Epoch:  62 \t Time: + 54.332828998565674 \t Training        loss:  0.668863996944369 \t Validation loss:  0.6706669437703693\n",
      "Epoch:  64 \t Time: + 59.91272187232971 \t Training        loss:  0.6688630370365709 \t Validation loss:  0.6706665911013261\n",
      "Epoch:  66 \t Time: + 52.10487246513367 \t Training        loss:  0.668862135468453 \t Validation loss:  0.670666260575305\n",
      "Epoch:  68 \t Time: + 51.71306610107422 \t Training        loss:  0.6688612872839363 \t Validation loss:  0.670665948745472\n",
      "Epoch:  70 \t Time: + 51.82513189315796 \t Training        loss:  0.6688604873828261 \t Validation loss:  0.6706656554596765\n",
      "Epoch:  72 \t Time: + 51.94565296173096 \t Training        loss:  0.668859731985923 \t Validation loss:  0.6706653784675363\n",
      "Epoch:  74 \t Time: + 52.19538331031799 \t Training        loss:  0.6688590172292651 \t Validation loss:  0.6706651164479441\n",
      "Epoch:  76 \t Time: + 51.54919075965881 \t Training        loss:  0.6688583404035867 \t Validation loss:  0.670664868218857\n",
      "Epoch:  78 \t Time: + 51.820186614990234 \t Training        loss:  0.6688576981859725 \t Validation loss:  0.6706646327194665\n",
      "Epoch:  80 \t Time: + 51.91531229019165 \t Training        loss:  0.6688570881799599 \t Validation loss:  0.6706644089950455\n",
      "Epoch:  82 \t Time: + 51.75338578224182 \t Training        loss:  0.6688565078778858 \t Validation loss:  0.670664196184011\n",
      "Epoch:  84 \t Time: + 51.93813729286194 \t Training        loss:  0.668855955177882 \t Validation loss:  0.6706639935068353\n",
      "Epoch:  86 \t Time: + 52.026893615722656 \t Training        loss:  0.6688554277652439 \t Validation loss:  0.6706638002565048\n",
      "Epoch:  88 \t Time: + 51.76008224487305 \t Training        loss:  0.668854924664612 \t Validation loss:  0.6706636157902804\n",
      "Epoch:  90 \t Time: + 52.13298487663269 \t Training        loss:  0.6688544439452258 \t Validation loss:  0.6706634395225548\n",
      "Epoch:  92 \t Time: + 51.62832975387573 \t Training        loss:  0.6688539840352196 \t Validation loss:  0.6706632709186434\n",
      "Epoch:  94 \t Time: + 52.142101764678955 \t Training        loss:  0.6688535438097538 \t Validation loss:  0.6706631094893665\n",
      "Epoch:  96 \t Time: + 51.85021352767944 \t Training        loss:  0.6688531218505935 \t Validation loss:  0.6706629547863094\n",
      "Epoch:  98 \t Time: + 51.804935455322266 \t Training        loss:  0.6688527173638104 \t Validation loss:  0.670662806397663\n",
      "Epoch:  100 \t Time: + 51.69692516326904 \t Training        loss:  0.6688523289378411 \t Validation loss:  0.6706626639445623\n",
      "Epoch:  102 \t Time: + 51.93511462211609 \t Training        loss:  0.6688519557092012 \t Validation loss:  0.6706625270778577\n",
      "Epoch:  104 \t Time: + 52.16494417190552 \t Training        loss:  0.6688515968505799 \t Validation loss:  0.6706623954752572\n",
      "Epoch:  106 \t Time: + 51.89978528022766 \t Training        loss:  0.6688512515248034 \t Validation loss:  0.6706622688387924\n",
      "Epoch:  108 \t Time: + 51.76830697059631 \t Training        loss:  0.668850918964921 \t Validation loss:  0.6706621468925672\n",
      "Epoch:  110 \t Time: + 52.04307174682617 \t Training        loss:  0.6688505985444175 \t Validation loss:  0.6706620293807501\n",
      "Epoch:  112 \t Time: + 51.87680101394653 \t Training        loss:  0.6688502893205633 \t Validation loss:  0.6706619160657837\n",
      "Epoch:  114 \t Time: + 53.61300754547119 \t Training        loss:  0.6688499911099459 \t Validation loss:  0.670661806726781\n",
      "Epoch:  116 \t Time: + 51.88495492935181 \t Training        loss:  0.6688497032251776 \t Validation loss:  0.6706617011580888\n",
      "Epoch:  118 \t Time: + 51.862263917922974 \t Training        loss:  0.6688494251251258 \t Validation loss:  0.6706615991679962\n",
      "Epoch:  120 \t Time: + 56.75907564163208 \t Training        loss:  0.6688491562911206 \t Validation loss:  0.6706615005775735\n",
      "Epoch:  122 \t Time: + 60.36628818511963 \t Training        loss:  0.668848896301473 \t Validation loss:  0.6706614052196235\n",
      "Epoch:  124 \t Time: + 62.275147676467896 \t Training        loss:  0.66884864467132 \t Validation loss:  0.6706613129377366\n",
      "Epoch:  126 \t Time: + 60.571754693984985 \t Training        loss:  0.668848400940305 \t Validation loss:  0.670661223042211\n",
      "Epoch:  128 \t Time: + 68.63215684890747 \t Training        loss:  0.6688481649602935 \t Validation loss:  0.6706611364906551\n",
      "Epoch:  130 \t Time: + 64.99816155433655 \t Training        loss:  0.6688479361956389 \t Validation loss:  0.6706610526022239\n",
      "Epoch:  132 \t Time: + 75.42485761642456 \t Training        loss:  0.6688477142686996 \t Validation loss:  0.6706609712558663\n",
      "Epoch:  134 \t Time: + 71.52716827392578 \t Training        loss:  0.6688474990486802 \t Validation loss:  0.6706608919589279\n",
      "Epoch:  136 \t Time: + 69.2496542930603 \t Training        loss:  0.6688472901597705 \t Validation loss:  0.6706608149942522\n",
      "Epoch:  138 \t Time: + 66.72904586791992 \t Training        loss:  0.6688470871684685 \t Validation loss:  0.6706607406282866\n",
      "Epoch:  140 \t Time: + 66.11111044883728 \t Training        loss:  0.6688468901714477 \t Validation loss:  0.670660668387063\n",
      "Epoch:  142 \t Time: + 66.03466439247131 \t Training        loss:  0.6688466987059678 \t Validation loss:  0.6706605981808034\n",
      "Epoch:  144 \t Time: + 65.87058353424072 \t Training        loss:  0.6688465125495453 \t Validation loss:  0.6706605299247176\n",
      "Epoch:  146 \t Time: + 65.51572155952454 \t Training        loss:  0.6688463315001533 \t Validation loss:  0.6706604631909678\n",
      "Epoch:  148 \t Time: + 65.38549852371216 \t Training        loss:  0.6688461552809926 \t Validation loss:  0.6706603986038281\n",
      "Epoch:  150 \t Time: + 61.04422211647034 \t Training        loss:  0.6688459836678005 \t Validation loss:  0.6706603357390121\n",
      "Epoch:  152 \t Time: + 65.66370606422424 \t Training        loss:  0.6688458166394656 \t Validation loss:  0.6706602741945644\n",
      "Epoch:  154 \t Time: + 66.36811780929565 \t Training        loss:  0.6688456539773866 \t Validation loss:  0.6706602145783055\n",
      "Epoch:  156 \t Time: + 66.80877327919006 \t Training        loss:  0.6688454955065787 \t Validation loss:  0.6706601558398572\n",
      "Epoch:  158 \t Time: + 66.47004008293152 \t Training        loss:  0.6688453410038875 \t Validation loss:  0.6706600989097449\n",
      "Epoch:  160 \t Time: + 63.943257093429565 \t Training        loss:  0.668845190267826 \t Validation loss:  0.670660043720156\n",
      "Epoch:  162 \t Time: + 53.9894061088562 \t Training        loss:  0.6688450434253885 \t Validation loss:  0.6706599898932729\n",
      "Epoch:  164 \t Time: + 52.28543424606323 \t Training        loss:  0.6688448999722529 \t Validation loss:  0.6706599373792407\n",
      "Epoch:  166 \t Time: + 51.840739488601685 \t Training        loss:  0.6688447602144642 \t Validation loss:  0.6706598861306068\n",
      "Epoch:  168 \t Time: + 52.141035318374634 \t Training        loss:  0.6688446237531648 \t Validation loss:  0.6706598361021785\n",
      "Epoch:  170 \t Time: + 52.475345611572266 \t Training        loss:  0.6688444904400199 \t Validation loss:  0.6706597872508897\n",
      "Epoch:  172 \t Time: + 54.551597356796265 \t Training        loss:  0.6688443602380262 \t Validation loss:  0.6706597392405419\n",
      "Epoch:  174 \t Time: + 55.36105704307556 \t Training        loss:  0.6688442330311694 \t Validation loss:  0.6706596926256234\n",
      "Epoch:  176 \t Time: + 56.43198490142822 \t Training        loss:  0.6688441087172171 \t Validation loss:  0.6706596470701348\n",
      "Epoch:  178 \t Time: + 57.66703391075134 \t Training        loss:  0.6688439872452878 \t Validation loss:  0.6706596025383651\n",
      "Epoch:  180 \t Time: + 55.728859663009644 \t Training        loss:  0.6688438683947949 \t Validation loss:  0.6706595589961902\n",
      "Epoch:  182 \t Time: + 55.31055474281311 \t Training        loss:  0.6688437521819739 \t Validation loss:  0.6706595164109863\n",
      "Epoch:  184 \t Time: + 56.51261854171753 \t Training        loss:  0.6688436384699434 \t Validation loss:  0.6706594747515476\n",
      "Epoch:  186 \t Time: + 55.17784929275513 \t Training        loss:  0.66884352722802 \t Validation loss:  0.6706594339880109\n",
      "Epoch:  188 \t Time: + 56.213178873062134 \t Training        loss:  0.6688434183700899 \t Validation loss:  0.6706593940917834\n",
      "Epoch:  190 \t Time: + 56.9613311290741 \t Training        loss:  0.668843311835412 \t Validation loss:  0.6706593550354765\n",
      "Epoch:  192 \t Time: + 54.13565635681152 \t Training        loss:  0.6688432074879169 \t Validation loss:  0.6706593167928427\n",
      "Epoch:  194 \t Time: + 56.9600350856781 \t Training        loss:  0.6688431053324269 \t Validation loss:  0.6706592793387168\n",
      "Epoch:  196 \t Time: + 55.27466440200806 \t Training        loss:  0.6688430052451672 \t Validation loss:  0.6706592426489608\n",
      "Epoch:  198 \t Time: + 52.13642644882202 \t Training        loss:  0.6688429071499546 \t Validation loss:  0.6706592064440318\n",
      "Epoch:  200 \t Time: + 50.613646268844604 \t Training        loss:  0.6688428110669424 \t Validation loss:  0.6706591712170177\n",
      "Epoch:  202 \t Time: + 50.677175998687744 \t Training        loss:  0.6688427168186792 \t Validation loss:  0.6706591366875683\n",
      "Epoch:  204 \t Time: + 50.50028085708618 \t Training        loss:  0.6688426244790395 \t Validation loss:  0.6706591028351669\n",
      "Epoch:  206 \t Time: + 50.80591320991516 \t Training        loss:  0.6688425339132558 \t Validation loss:  0.6706590696400938\n",
      "Epoch:  208 \t Time: + 50.565518617630005 \t Training        loss:  0.6688424451178663 \t Validation loss:  0.6706590370833874\n",
      "Epoch:  210 \t Time: + 50.47713112831116 \t Training        loss:  0.66884235793883 \t Validation loss:  0.6706590049050788\n",
      "Epoch:  212 \t Time: + 50.68085694313049 \t Training        loss:  0.6688422724209255 \t Validation loss:  0.6706589735733576\n",
      "Epoch:  214 \t Time: + 50.62151575088501 \t Training        loss:  0.6688421884850158 \t Validation loss:  0.6706589425900643\n",
      "Epoch:  216 \t Time: + 51.475759744644165 \t Training        loss:  0.6688421061621561 \t Validation loss:  0.6706589124155511\n",
      "Epoch:  218 \t Time: + 52.568363189697266 \t Training        loss:  0.6688420253312423 \t Validation loss:  0.6706588827946985\n",
      "Epoch:  220 \t Time: + 50.90149641036987 \t Training        loss:  0.6688419459264487 \t Validation loss:  0.6706588537124071\n",
      "Epoch:  222 \t Time: + 50.42025947570801 \t Training        loss:  0.6688418680018272 \t Validation loss:  0.6706588249254573\n",
      "Epoch:  224 \t Time: + 50.7119414806366 \t Training        loss:  0.6688417914200646 \t Validation loss:  0.6706587968791821\n",
      "Epoch:  226 \t Time: + 50.61536431312561 \t Training        loss:  0.6688417161671135 \t Validation loss:  0.6706587691046847\n",
      "Epoch:  228 \t Time: + 50.53164887428284 \t Training        loss:  0.6688416422976317 \t Validation loss:  0.6706587420401052\n",
      "Epoch:  230 \t Time: + 50.8755578994751 \t Training        loss:  0.6688415696483847 \t Validation loss:  0.6706587154462141\n",
      "Epoch:  232 \t Time: + 50.59445810317993 \t Training        loss:  0.668841498309089 \t Validation loss:  0.6706586893108384\n",
      "Epoch:  234 \t Time: + 50.811264753341675 \t Training        loss:  0.6688414282285929 \t Validation loss:  0.6706586636222213\n",
      "Epoch:  236 \t Time: + 50.75258994102478 \t Training        loss:  0.6688413593035365 \t Validation loss:  0.6706586383690044\n",
      "Epoch:  238 \t Time: + 50.65432667732239 \t Training        loss:  0.6688412915542915 \t Validation loss:  0.6706586135402113\n",
      "Epoch:  240 \t Time: + 50.77795743942261 \t Training        loss:  0.6688412249008876 \t Validation loss:  0.6706585891252315\n",
      "Epoch:  242 \t Time: + 51.008094787597656 \t Training        loss:  0.6688411593160487 \t Validation loss:  0.670658565113805\n",
      "Epoch:  244 \t Time: + 50.74777388572693 \t Training        loss:  0.668841094836831 \t Validation loss:  0.6706585412879622\n",
      "Epoch:  246 \t Time: + 50.73659896850586 \t Training        loss:  0.6688410313539472 \t Validation loss:  0.6706585180558863\n",
      "Epoch:  248 \t Time: + 50.729511976242065 \t Training        loss:  0.6688409689188964 \t Validation loss:  0.6706584951985214\n",
      "Epoch:  250 \t Time: + 50.50116229057312 \t Training        loss:  0.668840907514478 \t Validation loss:  0.6706584727068742\n",
      "Epoch:  252 \t Time: + 51.328906774520874 \t Training        loss:  0.6688408471235697 \t Validation loss:  0.6706584505722374\n",
      "Epoch:  254 \t Time: + 50.76886796951294 \t Training        loss:  0.6688407876289156 \t Validation loss:  0.6706584287861774\n",
      "Epoch:  256 \t Time: + 50.744752168655396 \t Training        loss:  0.6688407290828224 \t Validation loss:  0.6706584073405247\n",
      "Epoch:  258 \t Time: + 50.72329354286194 \t Training        loss:  0.6688406713984009 \t Validation loss:  0.6706583862273627\n",
      "Epoch:  260 \t Time: + 50.70840835571289 \t Training        loss:  0.6688406146896503 \t Validation loss:  0.6706583654390186\n",
      "Epoch:  262 \t Time: + 52.33890652656555 \t Training        loss:  0.6688405588077685 \t Validation loss:  0.6706583447743009\n",
      "Epoch:  264 \t Time: + 53.13838338851929 \t Training        loss:  0.6688405037068285 \t Validation loss:  0.6706583244226848\n",
      "Epoch:  266 \t Time: + 61.1167950630188 \t Training        loss:  0.6688404495115872 \t Validation loss:  0.6706583045679475\n",
      "Epoch:  268 \t Time: + 65.84563660621643 \t Training        loss:  0.6688403961499692 \t Validation loss:  0.6706582850095496\n",
      "Epoch:  270 \t Time: + 71.05097460746765 \t Training        loss:  0.6688403435625412 \t Validation loss:  0.6706582655528935\n",
      "Epoch:  272 \t Time: + 72.27917838096619 \t Training        loss:  0.6688402917464422 \t Validation loss:  0.6706582465689945\n",
      "Epoch:  274 \t Time: + 69.7716429233551 \t Training        loss:  0.6688402406640893 \t Validation loss:  0.6706582278622328\n",
      "Epoch:  276 \t Time: + 74.54334998130798 \t Training        loss:  0.6688401903095826 \t Validation loss:  0.6706582094265835\n",
      "Epoch:  278 \t Time: + 74.63495683670044 \t Training        loss:  0.6688401407092341 \t Validation loss:  0.6706581912561953\n",
      "Epoch:  280 \t Time: + 72.69165992736816 \t Training        loss:  0.6688400918044926 \t Validation loss:  0.6706581733453841\n",
      "Epoch:  282 \t Time: + 73.35857105255127 \t Training        loss:  0.6688400435210756 \t Validation loss:  0.670658155688627\n",
      "Epoch:  284 \t Time: + 73.1550304889679 \t Training        loss:  0.6688399959683361 \t Validation loss:  0.6706581382805566\n",
      "Epoch:  286 \t Time: + 73.74925470352173 \t Training        loss:  0.6688399490570246 \t Validation loss:  0.6706581211159557\n",
      "Epoch:  288 \t Time: + 72.81868720054626 \t Training        loss:  0.668839902842471 \t Validation loss:  0.6706581040134907\n",
      "Epoch:  290 \t Time: + 73.35244488716125 \t Training        loss:  0.6688398572784147 \t Validation loss:  0.6706580873219674\n",
      "Epoch:  292 \t Time: + 72.56899428367615 \t Training        loss:  0.6688398123131087 \t Validation loss:  0.6706580706852484\n",
      "Epoch:  294 \t Time: + 73.277348279953 \t Training        loss:  0.6688397679811633 \t Validation loss:  0.6706580544475435\n",
      "Epoch:  296 \t Time: + 73.30071520805359 \t Training        loss:  0.6688397241317166 \t Validation loss:  0.6706580381980306\n",
      "Epoch:  298 \t Time: + 72.48299980163574 \t Training        loss:  0.6688396810082835 \t Validation loss:  0.6706580223963164\n",
      "Epoch:  300 \t Time: + 73.13335108757019 \t Training        loss:  0.6688396384422671 \t Validation loss:  0.6706580068052919\n",
      "Epoch:  302 \t Time: + 73.6969199180603 \t Training        loss:  0.6688395964025028 \t Validation loss:  0.670657991420771\n",
      "Epoch:  304 \t Time: + 72.61254072189331 \t Training        loss:  0.6688395548973879 \t Validation loss:  0.6706579760716934\n",
      "Epoch:  306 \t Time: + 72.3744797706604 \t Training        loss:  0.668839513972362 \t Validation loss:  0.6706579610891505\n",
      "Epoch:  308 \t Time: + 73.08683800697327 \t Training        loss:  0.6688394735487384 \t Validation loss:  0.670657946301186\n",
      "Epoch:  310 \t Time: + 73.54159283638 \t Training        loss:  0.668839433681248 \t Validation loss:  0.6706579317040341\n",
      "Epoch:  312 \t Time: + 72.90625810623169 \t Training        loss:  0.6688393942203023 \t Validation loss:  0.6706579172940251\n",
      "Epoch:  314 \t Time: + 72.93286633491516 \t Training        loss:  0.6688393553357117 \t Validation loss:  0.6706579030675831\n",
      "Epoch:  316 \t Time: + 74.56576681137085 \t Training        loss:  0.6688393170004651 \t Validation loss:  0.6706578890212226\n",
      "Epoch:  318 \t Time: + 73.59442377090454 \t Training        loss:  0.6688392791553716 \t Validation loss:  0.670657875151546\n",
      "Epoch:  320 \t Time: + 73.74363827705383 \t Training        loss:  0.6688392417706135 \t Validation loss:  0.6706578614552402\n",
      "Epoch:  322 \t Time: + 73.03297901153564 \t Training        loss:  0.6688392048440808 \t Validation loss:  0.6706578479290749\n",
      "Epoch:  324 \t Time: + 73.69528365135193 \t Training        loss:  0.6688391682892502 \t Validation loss:  0.6706578345698994\n",
      "Epoch:  326 \t Time: + 73.74975848197937 \t Training        loss:  0.6688391322518812 \t Validation loss:  0.6706578212189244\n",
      "Epoch:  328 \t Time: + 73.55457425117493 \t Training        loss:  0.6688390966210356 \t Validation loss:  0.6706578079768555\n",
      "Epoch:  330 \t Time: + 72.868412733078 \t Training        loss:  0.6688390614467973 \t Validation loss:  0.6706577951027086\n",
      "Epoch:  332 \t Time: + 73.2492938041687 \t Training        loss:  0.6688390267429125 \t Validation loss:  0.670657782383672\n",
      "Epoch:  334 \t Time: + 72.3288722038269 \t Training        loss:  0.668838992391401 \t Validation loss:  0.6706577696649734\n",
      "Epoch:  336 \t Time: + 60.191182136535645 \t Training        loss:  0.6688389584932286 \t Validation loss:  0.6706577572487689\n",
      "Epoch:  338 \t Time: + 64.99956464767456 \t Training        loss:  0.6688389249830465 \t Validation loss:  0.6706577449795018\n",
      "Epoch:  340 \t Time: + 73.92126107215881 \t Training        loss:  0.6688388918580335 \t Validation loss:  0.6706577327052752\n",
      "Epoch:  342 \t Time: + 74.3620810508728 \t Training        loss:  0.6688388591475688 \t Validation loss:  0.6706577207230375\n",
      "Epoch:  344 \t Time: + 74.33312463760376 \t Training        loss:  0.6688388267958105 \t Validation loss:  0.6706577088801282\n",
      "Epoch:  346 \t Time: + 73.63861846923828 \t Training        loss:  0.6688387947750472 \t Validation loss:  0.6706576970274164\n",
      "Epoch:  348 \t Time: + 74.0515649318695 \t Training        loss:  0.6688387631709202 \t Validation loss:  0.6706576851650711\n",
      "Epoch:  350 \t Time: + 73.83921718597412 \t Training        loss:  0.668838731917597 \t Validation loss:  0.6706576737283714\n",
      "Epoch:  352 \t Time: + 75.00826597213745 \t Training        loss:  0.668838701040414 \t Validation loss:  0.6706576624216343\n",
      "Epoch:  354 \t Time: + 74.01964950561523 \t Training        loss:  0.6688386705236001 \t Validation loss:  0.6706576512426569\n",
      "Epoch:  356 \t Time: + 68.96589350700378 \t Training        loss:  0.6688386402721688 \t Validation loss:  0.6706576401892859\n",
      "Epoch:  358 \t Time: + 53.449838638305664 \t Training        loss:  0.668838610388002 \t Validation loss:  0.6706576292594163\n",
      "Epoch:  360 \t Time: + 51.13039684295654 \t Training        loss:  0.668838580808992 \t Validation loss:  0.6706576184509897\n",
      "Epoch:  362 \t Time: + 51.11348009109497 \t Training        loss:  0.6688385516255596 \t Validation loss:  0.6706576077619932\n",
      "Epoch:  364 \t Time: + 51.16880464553833 \t Training        loss:  0.6688385227517055 \t Validation loss:  0.6706575971904583\n",
      "Epoch:  366 \t Time: + 51.143134117126465 \t Training        loss:  0.6688384941380122 \t Validation loss:  0.6706575867344592\n",
      "Epoch:  368 \t Time: + 51.13856053352356 \t Training        loss:  0.6688384658703406 \t Validation loss:  0.6706575763921123\n",
      "Epoch:  370 \t Time: + 51.27455687522888 \t Training        loss:  0.6688384378587937 \t Validation loss:  0.6706575661615746\n",
      "Epoch:  372 \t Time: + 51.13489055633545 \t Training        loss:  0.6688384102130318 \t Validation loss:  0.6706575560410426\n",
      "Epoch:  374 \t Time: + 51.06257390975952 \t Training        loss:  0.6688383828744272 \t Validation loss:  0.6706575460287516\n",
      "Epoch:  376 \t Time: + 51.37626600265503 \t Training        loss:  0.6688383557993967 \t Validation loss:  0.6706575361229744\n",
      "Epoch:  378 \t Time: + 51.2440230846405 \t Training        loss:  0.6688383289742567 \t Validation loss:  0.6706575263220201\n",
      "Epoch:  380 \t Time: + 53.92022371292114 \t Training        loss:  0.6688383024972878 \t Validation loss:  0.6706575166242339\n",
      "Epoch:  382 \t Time: + 51.18185758590698 \t Training        loss:  0.6688382762636131 \t Validation loss:  0.6706575070279951\n",
      "Epoch:  384 \t Time: + 51.394381284713745 \t Training        loss:  0.668838250360762 \t Validation loss:  0.6706574975317171\n",
      "Epoch:  386 \t Time: + 51.00628900527954 \t Training        loss:  0.6688382247061554 \t Validation loss:  0.6706574881338463\n",
      "Epoch:  388 \t Time: + 51.91590642929077 \t Training        loss:  0.6688381992973955 \t Validation loss:  0.6706574788328605\n",
      "Epoch:  390 \t Time: + 50.969966650009155 \t Training        loss:  0.6688381741424836 \t Validation loss:  0.6706574696272697\n",
      "Epoch:  392 \t Time: + 51.09643507003784 \t Training        loss:  0.6688381492638334 \t Validation loss:  0.6706574605156134\n",
      "Epoch:  394 \t Time: + 52.775407552719116 \t Training        loss:  0.6688381246400278 \t Validation loss:  0.6706574514964612\n",
      "Epoch:  396 \t Time: + 52.49264168739319 \t Training        loss:  0.6688381002483719 \t Validation loss:  0.6706574425684114\n",
      "Epoch:  398 \t Time: + 53.56213665008545 \t Training        loss:  0.6688380761122491 \t Validation loss:  0.670657433602545\n",
      "Epoch:  400 \t Time: + 54.95208930969238 \t Training        loss:  0.6688380521885591 \t Validation loss:  0.6706574248532454\n",
      "Epoch:  402 \t Time: + 54.02146530151367 \t Training        loss:  0.668838028470278 \t Validation loss:  0.6706574161910035\n",
      "Epoch:  404 \t Time: + 52.99610710144043 \t Training        loss:  0.6688380050190251 \t Validation loss:  0.6706574074888747\n",
      "Epoch:  406 \t Time: + 52.92315101623535 \t Training        loss:  0.6688379817685691 \t Validation loss:  0.670657398997514\n",
      "Epoch:  408 \t Time: + 52.646522998809814 \t Training        loss:  0.6688379587808227 \t Validation loss:  0.670657390340562\n",
      "Epoch:  410 \t Time: + 53.42862129211426 \t Training        loss:  0.6688379360023966 \t Validation loss:  0.6706573818918812\n",
      "Epoch:  412 \t Time: + 52.887612104415894 \t Training        loss:  0.6688379134815648 \t Validation loss:  0.6706573736484382\n",
      "Epoch:  414 \t Time: + 52.85542845726013 \t Training        loss:  0.6688378911491668 \t Validation loss:  0.670657365484642\n",
      "Epoch:  416 \t Time: + 53.282610177993774 \t Training        loss:  0.6688378690507489 \t Validation loss:  0.6706573572773168\n",
      "Epoch:  418 \t Time: + 53.03991508483887 \t Training        loss:  0.6688378471641161 \t Validation loss:  0.6706573490270873\n",
      "Epoch:  420 \t Time: + 52.9353346824646 \t Training        loss:  0.6688378254926464 \t Validation loss:  0.6706573410971612\n",
      "Epoch:  422 \t Time: + 52.97660279273987 \t Training        loss:  0.6688378040449813 \t Validation loss:  0.6706573332424004\n",
      "Epoch:  424 \t Time: + 53.024109840393066 \t Training        loss:  0.6688377827658101 \t Validation loss:  0.6706573254617412\n",
      "Epoch:  426 \t Time: + 52.42870259284973 \t Training        loss:  0.6688377616813561 \t Validation loss:  0.670657317634977\n",
      "Epoch:  428 \t Time: + 53.05111598968506 \t Training        loss:  0.6688377407947612 \t Validation loss:  0.6706573099999661\n",
      "Epoch:  430 \t Time: + 52.734028577804565 \t Training        loss:  0.6688377201251008 \t Validation loss:  0.6706573024359784\n",
      "Epoch:  432 \t Time: + 52.99131655693054 \t Training        loss:  0.6688376996202278 \t Validation loss:  0.6706572949420277\n",
      "Epoch:  434 \t Time: + 52.36419367790222 \t Training        loss:  0.6688376793154116 \t Validation loss:  0.6706572874001796\n",
      "Epoch:  436 \t Time: + 53.51123905181885 \t Training        loss:  0.6688376591930469 \t Validation loss:  0.6706572800439523\n",
      "Epoch:  438 \t Time: + 52.84398794174194 \t Training        loss:  0.6688376392303983 \t Validation loss:  0.6706572727549052\n",
      "Epoch:  440 \t Time: + 53.2389497756958 \t Training        loss:  0.6688376194793735 \t Validation loss:  0.670657265416751\n",
      "Epoch:  442 \t Time: + 53.23050856590271 \t Training        loss:  0.6688375999016469 \t Validation loss:  0.6706572582598546\n",
      "Epoch:  444 \t Time: + 52.81311893463135 \t Training        loss:  0.668837580539161 \t Validation loss:  0.6706572511674346\n",
      "Epoch:  446 \t Time: + 52.877100706100464 \t Training        loss:  0.6688375613230731 \t Validation loss:  0.6706572440248048\n",
      "Epoch:  448 \t Time: + 53.17139530181885 \t Training        loss:  0.6688375422657784 \t Validation loss:  0.6706572370592594\n",
      "Epoch:  450 \t Time: + 52.44000482559204 \t Training        loss:  0.6688375233603358 \t Validation loss:  0.6706572301556298\n",
      "Epoch:  452 \t Time: + 53.351465940475464 \t Training        loss:  0.6688375046453807 \t Validation loss:  0.6706572232007864\n",
      "Epoch:  454 \t Time: + 52.99887132644653 \t Training        loss:  0.6688374860791798 \t Validation loss:  0.6706572164190323\n",
      "Epoch:  456 \t Time: + 53.05819630622864 \t Training        loss:  0.668837467689928 \t Validation loss:  0.6706572096967673\n",
      "Epoch:  458 \t Time: + 53.76188898086548 \t Training        loss:  0.66883744949323 \t Validation loss:  0.6706572030332122\n",
      "Epoch:  460 \t Time: + 53.02574706077576 \t Training        loss:  0.6688374314388151 \t Validation loss:  0.670657196427601\n",
      "Epoch:  462 \t Time: + 52.70064115524292 \t Training        loss:  0.6688374135142756 \t Validation loss:  0.6706571897693039\n",
      "Epoch:  464 \t Time: + 53.179110050201416 \t Training        loss:  0.6688373957538288 \t Validation loss:  0.6706571832778097\n",
      "Epoch:  466 \t Time: + 53.00897669792175 \t Training        loss:  0.6688373781420108 \t Validation loss:  0.6706571768420363\n",
      "Epoch:  468 \t Time: + 54.997456073760986 \t Training        loss:  0.6688373606990705 \t Validation loss:  0.6706571704612696\n",
      "Epoch:  470 \t Time: + 52.92007088661194 \t Training        loss:  0.6688373433984356 \t Validation loss:  0.6706571640268003\n",
      "Epoch:  472 \t Time: + 54.18794822692871 \t Training        loss:  0.6688373261796855 \t Validation loss:  0.6706571577544098\n",
      "Epoch:  474 \t Time: + 51.79512596130371 \t Training        loss:  0.6688373091695112 \t Validation loss:  0.6706571515349508\n",
      "Epoch:  476 \t Time: + 51.9208083152771 \t Training        loss:  0.6688372923015685 \t Validation loss:  0.6706571453677561\n",
      "Epoch:  478 \t Time: + 52.20152473449707 \t Training        loss:  0.6688372755793985 \t Validation loss:  0.6706571392521699\n",
      "Epoch:  480 \t Time: + 58.249515771865845 \t Training        loss:  0.6688372589617416 \t Validation loss:  0.6706571331875468\n",
      "Epoch:  482 \t Time: + 56.905388593673706 \t Training        loss:  0.6688372425224246 \t Validation loss:  0.6706571271732525\n",
      "Epoch:  484 \t Time: + 51.635775566101074 \t Training        loss:  0.6688372261950087 \t Validation loss:  0.6706571212086632\n",
      "Epoch:  486 \t Time: + 57.02062773704529 \t Training        loss:  0.6688372099995371 \t Validation loss:  0.6706571152931651\n",
      "Epoch:  488 \t Time: + 56.6814239025116 \t Training        loss:  0.6688371939734198 \t Validation loss:  0.6706571094261546\n",
      "Epoch:  490 \t Time: + 55.98983860015869 \t Training        loss:  0.668837178041395 \t Validation loss:  0.6706571036070382\n",
      "Epoch:  492 \t Time: + 62.89622235298157 \t Training        loss:  0.6688371622657083 \t Validation loss:  0.6706570978352318\n",
      "Epoch:  494 \t Time: + 69.24791312217712 \t Training        loss:  0.6688371466149894 \t Validation loss:  0.6706570921101604\n",
      "Epoch:  496 \t Time: + 64.9870777130127 \t Training        loss:  0.668837131079322 \t Validation loss:  0.6706570864312591\n",
      "Epoch:  498 \t Time: + 66.64228510856628 \t Training        loss:  0.6688371156635184 \t Validation loss:  0.6706570807979715\n",
      "Epoch:  500 \t Time: + 66.49163365364075 \t Training        loss:  0.6688371003714462 \t Validation loss:  0.67065707520975\n"
     ]
    }
   ],
   "source": [
    "print('Time: ', start, ' (in seconds)')\n",
    "while not early_stopping and epoch < epochs:\n",
    "    if epoch == 0:\n",
    "        start = time.time()\n",
    "\n",
    "    # prep model for training\n",
    "    model.train()\n",
    "    for x_train, y_train in train_loader:\n",
    "        # forward pass\n",
    "        y_hat = model(x_train.to(device))\n",
    "        # calculate the loss\n",
    "        loss = criterion(y_hat.flatten(), y_train.to(device))\n",
    "        # clear the gradients of all optimized variables\n",
    "        optimizer.zero_grad()\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        # perform a single optimization step (parameter update)\n",
    "        optimizer.step()\n",
    "        # update running training loss\n",
    "        train_loss.append(loss.item())\n",
    "    # shut down autograd to begin evaluation\n",
    "    with torch.no_grad():\n",
    "        # prep model for evaluation\n",
    "        model.eval()\n",
    "        for x_val, y_val in val_loader:\n",
    "            # forward pass\n",
    "            y_hat = model(x_val.to(device))\n",
    "            # calculate the loss\n",
    "            loss = criterion(y_hat.flatten(), y_val.to(device))\n",
    "            # update running validation loss\n",
    "            valid_loss.append(loss.item())\n",
    "    # early stopping conditional\n",
    "    if prev_mean_valid_loss <= statistics.mean(valid_loss):\n",
    "        if consecutive is True:\n",
    "            prevent -= 1\n",
    "        consecutive = True\n",
    "        if prevent < 0:\n",
    "            early_stopping = True\n",
    "            message = '\\tPrevious average Validation error was lower than\\\n",
    "                current Validation error'\n",
    "    else:\n",
    "        consecutive = False\n",
    "\n",
    "    # print results after 2 epochs\n",
    "    if epoch % 2 == 1:\n",
    "        end = time.time()\n",
    "        print('Epoch: ', epoch+1, '\\t Time: +', end-start, '\\t Training\\\n",
    "        loss: ', statistics.mean(train_loss), '\\t Validation loss: ',\n",
    "              statistics.mean(valid_loss))\n",
    "        start = time.time()\n",
    "\n",
    "    # update epoch's validation loss variable\n",
    "    prev_mean_valid_loss = statistics.mean(valid_loss)\n",
    "\n",
    "    # early stopping message\n",
    "    if early_stopping is True:\n",
    "        print('\\t\\tStopping at epoch: ', epoch + 1, message)\n",
    "        epoch = epochs - 1\n",
    "    epoch += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "del val_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tTime: 1.954857588 \tTest Loss: 0.666418733954430\n"
     ]
    }
   ],
   "source": [
    "# define test error list\n",
    "test_loss = []\n",
    "# initialize timer\n",
    "start = time.time()\n",
    "# test model\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for x, y in test_loader:\n",
    "        yhat = model(x.to(device))\n",
    "        loss = criterion(yhat.flatten(), y.to(device))\n",
    "        test_loss.append(loss.item())\n",
    "# end time checkpoint\n",
    "end = time.time()\n",
    "# print test results\n",
    "print('\\tTime: {:.10} \\tTest Loss: {:.15f}'.format(end-start,\n",
    "                                                   statistics.mean(test_loss)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check model's prediction on the whole dataset\n",
    "prediction = None\n",
    "with torch.no_grad():\n",
    "    prediction = model(x_test.to(device)).cpu().detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_evaluation = []\n",
    "y_hat = []\n",
    "y_real = []\n",
    "evaluation = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(y_test)):\n",
    "    final_evaluation.append(y_test[i].numpy() - prediction[i].item())\n",
    "    y_real.append(y_test[i].numpy())\n",
    "    y_hat.append(prediction[i].item())\n",
    "    if abs(y_test[i].numpy() - prediction[i].item()) > 0.5:\n",
    "        evaluation.append(False)\n",
    "    else:\n",
    "        evaluation.append(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pandas.DataFrame(list(zip(final_evaluation, evaluation, y_real,\n",
    "                                    y_hat)), columns=['loss_dif',\n",
    "                                                      'evaluation',\n",
    "                                                      'y_real',\n",
    "                                                      'y_hat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss_dif</th>\n",
       "      <th>evaluation</th>\n",
       "      <th>y_real</th>\n",
       "      <th>y_hat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5995</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5996</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5997</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5998</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5999</th>\n",
       "      <td>-0.433266</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      loss_dif  evaluation y_real     y_hat\n",
       "0    -0.433266        True    0.0  0.433266\n",
       "1    -0.433266        True    0.0  0.433266\n",
       "2    -0.433266        True    0.0  0.433266\n",
       "3    -0.433266        True    0.0  0.433266\n",
       "4    -0.433266        True    0.0  0.433266\n",
       "...        ...         ...    ...       ...\n",
       "5995 -0.433266        True    0.0  0.433266\n",
       "5996 -0.433266        True    0.0  0.433266\n",
       "5997 -0.433266        True    0.0  0.433266\n",
       "5998 -0.433266        True    0.0  0.433266\n",
       "5999 -0.433266        True    0.0  0.433266\n",
       "\n",
       "[6000 rows x 4 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "debug = test_df.loc[test_df['evaluation'] == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss_dif</th>\n",
       "      <th>evaluation</th>\n",
       "      <th>y_real</th>\n",
       "      <th>y_hat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5987</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5989</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5991</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5992</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5993</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2202 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      loss_dif  evaluation y_real     y_hat\n",
       "7     0.566734       False    1.0  0.433266\n",
       "9     0.566734       False    1.0  0.433266\n",
       "18    0.566734       False    1.0  0.433266\n",
       "19    0.566734       False    1.0  0.433266\n",
       "29    0.566734       False    1.0  0.433266\n",
       "...        ...         ...    ...       ...\n",
       "5987  0.566734       False    1.0  0.433266\n",
       "5989  0.566734       False    1.0  0.433266\n",
       "5991  0.566734       False    1.0  0.433266\n",
       "5992  0.566734       False    1.0  0.433266\n",
       "5993  0.566734       False    1.0  0.433266\n",
       "\n",
       "[2202 rows x 4 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.63      1.00      0.78      3798\n",
      "         1.0       0.00      0.00      0.00      2202\n",
      "\n",
      "    accuracy                           0.63      6000\n",
      "   macro avg       0.32      0.50      0.39      6000\n",
      "weighted avg       0.40      0.63      0.49      6000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/andreas/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(sklearn.metrics.classification_report(\n",
    "    test_df.y_real.astype(numpy.float16).to_numpy(),\n",
    "    numpy.where(test_df.y_hat.astype(numpy.float16).to_numpy() > 0.5, 1, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check model's prediction on the whole dataset\n",
    "overall = None\n",
    "with torch.no_grad():\n",
    "    overall = model(X.to(device)).cpu().detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_evaluation = []\n",
    "y_hat = []\n",
    "y_real = []\n",
    "evaluation = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(Y)):\n",
    "    final_evaluation.append(Y[i].numpy() - overall[i].item())\n",
    "    y_real.append(Y[i].numpy())\n",
    "    y_hat.append(overall[i].item())\n",
    "    if abs(Y[i].numpy() - overall[i].item()) > 0.5:\n",
    "        evaluation.append(False)\n",
    "    else:\n",
    "        evaluation.append(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_df = pandas.DataFrame(list(zip(final_evaluation, evaluation, y_real,\n",
    "                                       y_hat)), columns=['loss_dif',\n",
    "                                                         'evaluation',\n",
    "                                                         'y_real',\n",
    "                                                         'y_hat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "del final_evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "del y_real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     15000\n",
       "False     9000\n",
       "Name: evaluation, dtype: int64"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_df.evaluation.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.05826577544212341"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_df.loss_dif.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = overall_df.loc[overall_df['evaluation'] == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss_dif</th>\n",
       "      <th>evaluation</th>\n",
       "      <th>y_real</th>\n",
       "      <th>y_hat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23992</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23994</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23995</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23997</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23998</th>\n",
       "      <td>0.566734</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.433266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       loss_dif  evaluation y_real     y_hat\n",
       "0      0.566734       False    1.0  0.433266\n",
       "2      0.566734       False    1.0  0.433266\n",
       "3      0.566734       False    1.0  0.433266\n",
       "4      0.566734       False    1.0  0.433266\n",
       "5      0.566734       False    1.0  0.433266\n",
       "...         ...         ...    ...       ...\n",
       "23992  0.566734       False    1.0  0.433266\n",
       "23994  0.566734       False    1.0  0.433266\n",
       "23995  0.566734       False    1.0  0.433266\n",
       "23997  0.566734       False    1.0  0.433266\n",
       "23998  0.566734       False    1.0  0.433266\n",
       "\n",
       "[9000 rows x 4 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.62      1.00      0.77     15000\n",
      "         1.0       0.00      0.00      0.00      9000\n",
      "\n",
      "    accuracy                           0.62     24000\n",
      "   macro avg       0.31      0.50      0.38     24000\n",
      "weighted avg       0.39      0.62      0.48     24000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/andreas/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(sklearn.metrics.classification_report(\n",
    "    overall_df.y_real.astype(numpy.float16).to_numpy(),\n",
    "    numpy.where(\n",
    "        overall_df.y_hat.astype(numpy.float16).to_numpy() > 0.5, 1, 0)\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "del errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "del overall_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "del valid_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}