AS_7_complex_fat_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>context_type</th>\n",
       "      <th>media_duration</th>\n",
       "      <th>listen_type</th>\n",
       "      <th>user_gender</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_age</th>\n",
       "      <th>is_listened</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5010516</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>454</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4606</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1309460</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>213</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7557</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>824607</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>235</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5034</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5321927</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>467</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13875</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5010515</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>388</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4606</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ts_listen  context_type  media_duration  listen_type  user_gender  \\\n",
       "5010516          1             0             454            0            0   \n",
       "1309460          2             0             213            0            0   \n",
       "824607           2             0             235            0            0   \n",
       "5321927          2             0             467            0            0   \n",
       "5010515          3             0             388            0            0   \n",
       "\n",
       "         user_id  user_age  is_listened  \n",
       "5010516     4606        23            0  \n",
       "1309460     7557        28            1  \n",
       "824607      5034        19            1  \n",
       "5321927    13875        28            0  \n",
       "5010515     4606        23            0  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# спочатку song_listened\n",
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "\n",
    "cols = ['context_type', 'media_duration', 'user_gender', 'user_age']\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8', usecols=cols+['listen_type', 'ts_listen', 'user_id', 'is_listened'])\n",
    "train = train.sort_values(by=['ts_listen'], axis=0)\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "song_listened = pd.read_csv(\"./data/features/C++ Output/song_listened.txt\", \n",
    "                            encoding='utf-8', header=None, names=['song_listened'])\n",
    "\n",
    "song_listened.index = train.index\n",
    "\n",
    "train = pd.concat([train, song_listened], axis=1)\n",
    "train.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2 - song_listened_test\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8', usecols=cols+['ts_listen', 'user_id'])\n",
    "test = test.sort_values(by=['ts_listen'], axis=0)\n",
    "\n",
    "song_listened_test = pd.read_csv(\"./data/features/C++ Output/song_listened_test.txt\", \n",
    "                            encoding='utf-8', header=None, names=['song_listened'])\n",
    "\n",
    "song_listened_test.index = test.index\n",
    "\n",
    "test = pd.concat([test, song_listened_test], axis=1)\n",
    "test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train.sort_index()\n",
    "test = test.sort_index()\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 3 - cumsum\n",
    "path_to_test_cumsum = './data/features/cumsum.test.csv'\n",
    "path_to_train_cumsum = './data/features/cumsum.train.csv'\n",
    "\n",
    "train_cumsum = pd.read_csv(path_to_train_cumsum, encoding='utf-8')\n",
    "train = pd.concat([train, train_cumsum], axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_cumsum = pd.read_csv(path_to_test_cumsum, encoding='utf-8')\n",
    "test = pd.concat([test, test_cumsum], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 4 - listened before\n",
    "path_to_test_prev_is_listened = './data/features/prev_is_listened.test.csv'\n",
    "path_to_train_prev_is_listened = './data/features/prev_is_listened.train.csv'\n",
    "\n",
    "train_prev_is_listened = pd.read_csv(path_to_train_prev_is_listened, encoding='utf-8')\n",
    "train = pd.concat([train, train_prev_is_listened], axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.concat([test, pd.read_csv(path_to_test_prev_is_listened, encoding='utf-8')], axis=1)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 5 - release date\n",
    "path_to_release_date_train = './data/features/release_date.train.txt'\n",
    "path_to_release_date_test = './data/features/release_date.test.txt'\n",
    "newcols = ['release_date_listened_this_decade','release_date_listened_this_year']\n",
    "\n",
    "train = pd.concat([train, pd.read_csv(path_to_release_date_train, encoding='utf-8', usecols=newcols)], axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.concat([test, pd.read_csv(path_to_release_date_test, encoding='utf-8', usecols=newcols)], axis=1)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# more Orest generated features\n",
    "test['is_listened'] = 0\n",
    "\n",
    "df = pd.concat([train, test], ignore_index=True)\n",
    "df = df.sort_values(by=['user_id', 'ts_listen'], axis=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_features = './data/features/C++ Output/features.csv'\n",
    "features = pd.read_csv(path_to_features, encoding='utf-8')\n",
    "df = pd.concat([df, features], axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = df.sort_index()\n",
    "train = df.loc[range(len(train))]\n",
    "test = df.loc[range(len(train), len(train)+len(test))]\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.to_csv('./data/features/fat_train.csv', encoding='utf-8', index=False)\n",
    "test.to_csv('./data/features/fat_test.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/features/fat_train.csv'\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8')\n",
    "\n",
    "listen_type = pd.read_csv('./data/train.csv', encoding='utf-8', usecols=['listen_type'])\n",
    "train = pd.concat([train, listen_type], axis=1)\n",
    "\n",
    "path_to_train_cum_mean = './data/features/cum_mean.train.csv'\n",
    "train_cum_mean = pd.read_csv(path_to_train_cum_mean, encoding='utf-8')\n",
    "train = pd.concat([train, train_cum_mean], axis=1)\n",
    "train.to_csv('./data/features/fat_train.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_test = './data/features/fat_test.csv'\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8')\n",
    "\n",
    "path_to_test_cum_mean = './data/features/cum_mean.test.csv'\n",
    "test_cum_mean = pd.read_csv(path_to_test_cum_mean, encoding='utf-8')\n",
    "\n",
    "test = pd.concat([test, test_cum_mean], axis=1)\n",
    "test.to_csv('./data/features/fat_test.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/features/fat_train.csv'\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8')\n",
    "\n",
    "listen_type = pd.read_csv('./data/train.csv', encoding='utf-8', usecols=['listen_type'])\n",
    "train = pd.concat([train, listen_type], axis=1)\n",
    "\n",
    "path_to_train_cum_mean = './data/features/cum_mean2.train.csv'\n",
    "train_cum_mean = pd.read_csv(path_to_train_cum_mean, encoding='utf-8')\n",
    "train = pd.concat([train, train_cum_mean], axis=1)\n",
    "train.to_csv('./data/features/fat_train.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_test = './data/features/fat_test.csv'\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8')\n",
    "\n",
    "path_to_test_cum_mean = './data/features/cum_mean2.test.csv'\n",
    "test_cum_mean = pd.read_csv(path_to_test_cum_mean, encoding='utf-8')\n",
    "\n",
    "test = pd.concat([test, test_cum_mean], axis=1)\n",
    "test.to_csv('./data/features/fat_test.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context_type</th>\n",
       "      <th>cumsum_is_listened</th>\n",
       "      <th>cumsum_is_listened_1</th>\n",
       "      <th>cumsum_listen_type</th>\n",
       "      <th>cumsum_listen_type_1</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>media_duration</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "      <th>song_listened</th>\n",
       "      <th>...</th>\n",
       "      <th>platform_name_cumulative_mean</th>\n",
       "      <th>artist_id_cumulative_mean</th>\n",
       "      <th>ts_listen_dayofweek_cumulative_mean</th>\n",
       "      <th>ts_listen_hour_cumulative_mean</th>\n",
       "      <th>listen_type.1</th>\n",
       "      <th>release_date_year_cumulative_mean</th>\n",
       "      <th>release_date_decade_cumulative_mean</th>\n",
       "      <th>ExplicitLyrics_cumulative_mean</th>\n",
       "      <th>Isrc_Country_cumulative_mean</th>\n",
       "      <th>Isrc_Label_cumulative_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>223</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.610063</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.757576</td>\n",
       "      <td>0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.686364</td>\n",
       "      <td>0.704000</td>\n",
       "      <td>0.545455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>171</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>149</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.973333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.983051</td>\n",
       "      <td>0.987179</td>\n",
       "      <td>0.990783</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>240</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.460227</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.664671</td>\n",
       "      <td>0.939394</td>\n",
       "      <td>0</td>\n",
       "      <td>0.612500</td>\n",
       "      <td>0.679174</td>\n",
       "      <td>0.637413</td>\n",
       "      <td>0.742574</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>150</td>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.961832</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.934783</td>\n",
       "      <td>0.960317</td>\n",
       "      <td>0.957447</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 52 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   context_type  cumsum_is_listened  cumsum_is_listened_1  cumsum_listen_type  \\\n",
       "0            12                 3.0                   0.0                 5.0   \n",
       "1             0                 1.0                   1.0                 4.0   \n",
       "2             1                 5.0                   5.0                 2.0   \n",
       "3             0                 1.0                   0.0                 5.0   \n",
       "4             0                 5.0                   5.0                 5.0   \n",
       "\n",
       "   cumsum_listen_type_1  is_listened  media_duration  \\\n",
       "0                   0.0            0             223   \n",
       "1                   0.0            1             171   \n",
       "2                   0.0            1             149   \n",
       "3                   0.0            0             240   \n",
       "4                   0.0            1             150   \n",
       "\n",
       "   release_date_listened_this_decade  release_date_listened_this_year  \\\n",
       "0                                 20                               20   \n",
       "1                                  4                                1   \n",
       "2                                 20                               20   \n",
       "3                                 20                                3   \n",
       "4                                 20                                5   \n",
       "\n",
       "   song_listened             ...              platform_name_cumulative_mean  \\\n",
       "0           -1.0             ...                                   0.610063   \n",
       "1           -1.0             ...                                   0.555556   \n",
       "2           -1.0             ...                                   0.973333   \n",
       "3           -1.0             ...                                   0.460227   \n",
       "4            1.0             ...                                   0.961832   \n",
       "\n",
       "   artist_id_cumulative_mean  ts_listen_dayofweek_cumulative_mean  \\\n",
       "0                   0.000000                             0.357143   \n",
       "1                   0.000000                             0.555556   \n",
       "2                   0.000000                             1.000000   \n",
       "3                   0.787879                             0.664671   \n",
       "4                   1.000000                             0.863636   \n",
       "\n",
       "   ts_listen_hour_cumulative_mean  listen_type.1  \\\n",
       "0                        0.757576              0   \n",
       "1                        0.555556              0   \n",
       "2                        1.000000              1   \n",
       "3                        0.939394              0   \n",
       "4                        1.000000              0   \n",
       "\n",
       "   release_date_year_cumulative_mean  release_date_decade_cumulative_mean  \\\n",
       "0                           0.500000                             0.714286   \n",
       "1                           0.000000                             0.666667   \n",
       "2                           0.983051                             0.987179   \n",
       "3                           0.612500                             0.679174   \n",
       "4                           1.000000                             0.934783   \n",
       "\n",
       "   ExplicitLyrics_cumulative_mean  Isrc_Country_cumulative_mean  \\\n",
       "0                        0.686364                      0.704000   \n",
       "1                        0.555556                      0.625000   \n",
       "2                        0.990783                      0.954545   \n",
       "3                        0.637413                      0.742574   \n",
       "4                        0.960317                      0.957447   \n",
       "\n",
       "   Isrc_Label_cumulative_mean  \n",
       "0                    0.545455  \n",
       "1                    0.000000  \n",
       "2                    0.888889  \n",
       "3                    0.000000  \n",
       "4                    1.000000  \n",
       "\n",
       "[5 rows x 52 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import validation as v\n",
    "import pandas as pd\n",
    "import preparation as p\n",
    "\n",
    "path_to_train = './data/features/fat_train.csv'\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8')\n",
    "\n",
    "train = p.minimal_entries_by_user(train, 2)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_index, validation_index = v.split_train_validation_by_last_record_and_flow(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['context_type',\n",
       " 'cumsum_is_listened',\n",
       " 'cumsum_is_listened_1',\n",
       " 'cumsum_listen_type',\n",
       " 'cumsum_listen_type_1',\n",
       " 'is_listened',\n",
       " 'media_duration',\n",
       " 'release_date_listened_this_decade',\n",
       " 'release_date_listened_this_year',\n",
       " 'song_listened',\n",
       " 'ts_listen',\n",
       " 'user_age',\n",
       " 'user_gender',\n",
       " 'user_id',\n",
       " 'was_is_listened_before',\n",
       " 'percent_listened',\n",
       " 'genre_listened',\n",
       " 'album_listened',\n",
       " 'artist_listened',\n",
       " 'media_duration_mean',\n",
       " 'media_duration_std',\n",
       " 'media_duration_max',\n",
       " 'media_duration_min',\n",
       " 'explicit_lyrics_percent',\n",
       " 'country_percent',\n",
       " 'rank_mean',\n",
       " 'rank_std',\n",
       " 'rank_max',\n",
       " 'rank_min',\n",
       " 'bpm_mean',\n",
       " 'bpm_std',\n",
       " 'bpm_max',\n",
       " 'bpm_min',\n",
       " 'gain_mean',\n",
       " 'gain_std',\n",
       " 'gain_max',\n",
       " 'gain_min',\n",
       " 'listen_type',\n",
       " 'listen_type_cumulative_mean',\n",
       " 'genre_id_cumulative_mean',\n",
       " 'album_id_cumulative_mean',\n",
       " 'context_type_cumulative_mean',\n",
       " 'platform_name_cumulative_mean',\n",
       " 'artist_id_cumulative_mean',\n",
       " 'ts_listen_dayofweek_cumulative_mean',\n",
       " 'ts_listen_hour_cumulative_mean',\n",
       " 'listen_type.1',\n",
       " 'release_date_year_cumulative_mean',\n",
       " 'release_date_decade_cumulative_mean',\n",
       " 'ExplicitLyrics_cumulative_mean',\n",
       " 'Isrc_Country_cumulative_mean',\n",
       " 'Isrc_Label_cumulative_mean']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(train.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'application':'binary',\n",
    "    'num_leaves': 31,\n",
    "    'max_depth': 20,\n",
    "    'feature_fraction': 0.7,\n",
    "    'bagging_fraction': 0.7,\n",
    "    'bagging_freq': 1,\n",
    "    'max_bin': 200,\n",
    "    'metric': 'auc',\n",
    "    'verbose': 1\n",
    "}\n",
    "# params = {\n",
    "#     'application': 'binary',\n",
    "#     'num_leaves': 255,\n",
    "#     'feature_fraction': 0.7,\n",
    "#     'sub_row': 0.7,\n",
    "#     'bagging_freq': 1,\n",
    "#     'max_bin': 255,\n",
    "#     'metric': 'auc',\n",
    "#     'verbose': 0,\n",
    "#     'seed': 42\n",
    "# }\n",
    "cols = ['context_type',\n",
    " 'cumsum_is_listened',\n",
    " 'cumsum_is_listened_1',\n",
    " 'cumsum_listen_type',\n",
    " 'cumsum_listen_type_1',\n",
    " 'media_duration',\n",
    " 'release_date_listened_this_decade',\n",
    " 'release_date_listened_this_year',\n",
    " 'song_listened',\n",
    " 'user_age',\n",
    " 'user_gender',\n",
    " 'was_is_listened_before',\n",
    " 'percent_listened',\n",
    " 'genre_listened',\n",
    " 'album_listened',\n",
    " 'artist_listened',\n",
    " 'media_duration_mean',\n",
    " 'media_duration_std',\n",
    " 'media_duration_max',\n",
    " 'media_duration_min',\n",
    " 'explicit_lyrics_percent',\n",
    " 'country_percent',\n",
    " 'rank_mean',\n",
    " 'rank_std',\n",
    " 'rank_max',\n",
    " 'rank_min',\n",
    " 'bpm_mean',\n",
    " 'bpm_std',\n",
    " 'bpm_max',\n",
    " 'bpm_min',\n",
    " 'gain_mean',\n",
    " 'gain_std',\n",
    " 'gain_max',\n",
    " 'gain_min',\n",
    " 'user_id',\n",
    " 'listen_type_cumulative_mean',\n",
    " 'genre_id_cumulative_mean',\n",
    " 'album_id_cumulative_mean',\n",
    " 'context_type_cumulative_mean',\n",
    " 'platform_name_cumulative_mean',\n",
    " 'artist_id_cumulative_mean',\n",
    " 'ts_listen_dayofweek_cumulative_mean',\n",
    " 'ts_listen_hour_cumulative_mean',\n",
    " 'release_date_year_cumulative_mean',\n",
    " 'release_date_decade_cumulative_mean',\n",
    " 'ExplicitLyrics_cumulative_mean',\n",
    " 'Isrc_Country_cumulative_mean',\n",
    " 'Isrc_Label_cumulative_mean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttraining's auc: 0.834472\tvalid_1's auc: 0.686164\n",
      "[2]\ttraining's auc: 0.858495\tvalid_1's auc: 0.70512\n",
      "[3]\ttraining's auc: 0.861777\tvalid_1's auc: 0.707934\n",
      "[4]\ttraining's auc: 0.864183\tvalid_1's auc: 0.708988\n",
      "[5]\ttraining's auc: 0.864431\tvalid_1's auc: 0.709068\n",
      "[6]\ttraining's auc: 0.86506\tvalid_1's auc: 0.711796\n",
      "[7]\ttraining's auc: 0.865747\tvalid_1's auc: 0.711452\n",
      "[8]\ttraining's auc: 0.866279\tvalid_1's auc: 0.712044\n",
      "[9]\ttraining's auc: 0.866348\tvalid_1's auc: 0.711986\n",
      "[10]\ttraining's auc: 0.866475\tvalid_1's auc: 0.713335\n",
      "[11]\ttraining's auc: 0.866748\tvalid_1's auc: 0.713118\n",
      "[12]\ttraining's auc: 0.867074\tvalid_1's auc: 0.713085\n",
      "[13]\ttraining's auc: 0.86714\tvalid_1's auc: 0.713414\n",
      "[14]\ttraining's auc: 0.867379\tvalid_1's auc: 0.7136\n",
      "[15]\ttraining's auc: 0.867644\tvalid_1's auc: 0.713719\n",
      "[16]\ttraining's auc: 0.867735\tvalid_1's auc: 0.713782\n",
      "[17]\ttraining's auc: 0.867816\tvalid_1's auc: 0.713727\n",
      "[18]\ttraining's auc: 0.867945\tvalid_1's auc: 0.714113\n",
      "[19]\ttraining's auc: 0.868157\tvalid_1's auc: 0.714127\n",
      "[20]\ttraining's auc: 0.868321\tvalid_1's auc: 0.714001\n",
      "[21]\ttraining's auc: 0.868491\tvalid_1's auc: 0.714195\n",
      "[22]\ttraining's auc: 0.868603\tvalid_1's auc: 0.714096\n",
      "[23]\ttraining's auc: 0.868701\tvalid_1's auc: 0.714453\n",
      "[24]\ttraining's auc: 0.868787\tvalid_1's auc: 0.714838\n",
      "[25]\ttraining's auc: 0.868925\tvalid_1's auc: 0.71466\n",
      "[26]\ttraining's auc: 0.869017\tvalid_1's auc: 0.714464\n",
      "[27]\ttraining's auc: 0.869129\tvalid_1's auc: 0.714627\n",
      "[28]\ttraining's auc: 0.869242\tvalid_1's auc: 0.71459\n",
      "[29]\ttraining's auc: 0.869307\tvalid_1's auc: 0.714798\n",
      "[30]\ttraining's auc: 0.869405\tvalid_1's auc: 0.714718\n",
      "[31]\ttraining's auc: 0.869476\tvalid_1's auc: 0.714544\n",
      "[32]\ttraining's auc: 0.869582\tvalid_1's auc: 0.714872\n",
      "[33]\ttraining's auc: 0.869661\tvalid_1's auc: 0.714859\n",
      "[34]\ttraining's auc: 0.869737\tvalid_1's auc: 0.715042\n",
      "[35]\ttraining's auc: 0.869865\tvalid_1's auc: 0.71519\n",
      "[36]\ttraining's auc: 0.869925\tvalid_1's auc: 0.715276\n",
      "[37]\ttraining's auc: 0.87001\tvalid_1's auc: 0.7156\n",
      "[38]\ttraining's auc: 0.870118\tvalid_1's auc: 0.71616\n",
      "[39]\ttraining's auc: 0.870174\tvalid_1's auc: 0.71618\n",
      "[40]\ttraining's auc: 0.870308\tvalid_1's auc: 0.716148\n",
      "[41]\ttraining's auc: 0.870372\tvalid_1's auc: 0.716428\n",
      "[42]\ttraining's auc: 0.870467\tvalid_1's auc: 0.716592\n",
      "[43]\ttraining's auc: 0.870517\tvalid_1's auc: 0.716979\n",
      "[44]\ttraining's auc: 0.870549\tvalid_1's auc: 0.717105\n",
      "[45]\ttraining's auc: 0.870657\tvalid_1's auc: 0.717641\n",
      "[46]\ttraining's auc: 0.870703\tvalid_1's auc: 0.71768\n",
      "[47]\ttraining's auc: 0.870773\tvalid_1's auc: 0.717739\n",
      "[48]\ttraining's auc: 0.870807\tvalid_1's auc: 0.717847\n",
      "[49]\ttraining's auc: 0.870907\tvalid_1's auc: 0.71829\n",
      "[50]\ttraining's auc: 0.870946\tvalid_1's auc: 0.7184\n",
      "[51]\ttraining's auc: 0.870999\tvalid_1's auc: 0.718565\n",
      "[52]\ttraining's auc: 0.871048\tvalid_1's auc: 0.718783\n",
      "[53]\ttraining's auc: 0.871116\tvalid_1's auc: 0.71909\n",
      "[54]\ttraining's auc: 0.871164\tvalid_1's auc: 0.719145\n",
      "[55]\ttraining's auc: 0.871213\tvalid_1's auc: 0.719381\n",
      "[56]\ttraining's auc: 0.871272\tvalid_1's auc: 0.719718\n",
      "[57]\ttraining's auc: 0.871337\tvalid_1's auc: 0.720087\n",
      "[58]\ttraining's auc: 0.871454\tvalid_1's auc: 0.72075\n",
      "[59]\ttraining's auc: 0.87153\tvalid_1's auc: 0.720648\n",
      "[60]\ttraining's auc: 0.871603\tvalid_1's auc: 0.720752\n",
      "[61]\ttraining's auc: 0.871645\tvalid_1's auc: 0.721044\n",
      "[62]\ttraining's auc: 0.871693\tvalid_1's auc: 0.721204\n",
      "[63]\ttraining's auc: 0.871718\tvalid_1's auc: 0.721311\n",
      "[64]\ttraining's auc: 0.871837\tvalid_1's auc: 0.72234\n",
      "[65]\ttraining's auc: 0.871892\tvalid_1's auc: 0.722526\n",
      "[66]\ttraining's auc: 0.871937\tvalid_1's auc: 0.722625\n",
      "[67]\ttraining's auc: 0.871974\tvalid_1's auc: 0.722808\n",
      "[68]\ttraining's auc: 0.872006\tvalid_1's auc: 0.722971\n",
      "[69]\ttraining's auc: 0.872049\tvalid_1's auc: 0.72329\n",
      "[70]\ttraining's auc: 0.872191\tvalid_1's auc: 0.72406\n",
      "[71]\ttraining's auc: 0.872255\tvalid_1's auc: 0.724542\n",
      "[72]\ttraining's auc: 0.872295\tvalid_1's auc: 0.725064\n",
      "[73]\ttraining's auc: 0.872333\tvalid_1's auc: 0.725424\n",
      "[74]\ttraining's auc: 0.872368\tvalid_1's auc: 0.725411\n",
      "[75]\ttraining's auc: 0.87241\tvalid_1's auc: 0.725829\n",
      "[76]\ttraining's auc: 0.872485\tvalid_1's auc: 0.726335\n",
      "[77]\ttraining's auc: 0.872501\tvalid_1's auc: 0.726378\n",
      "[78]\ttraining's auc: 0.872524\tvalid_1's auc: 0.726377\n",
      "[79]\ttraining's auc: 0.87255\tvalid_1's auc: 0.726621\n",
      "[80]\ttraining's auc: 0.872578\tvalid_1's auc: 0.726698\n",
      "[81]\ttraining's auc: 0.872638\tvalid_1's auc: 0.726952\n",
      "[82]\ttraining's auc: 0.872667\tvalid_1's auc: 0.727004\n",
      "[83]\ttraining's auc: 0.872682\tvalid_1's auc: 0.726961\n",
      "[84]\ttraining's auc: 0.872705\tvalid_1's auc: 0.726964\n",
      "[85]\ttraining's auc: 0.872721\tvalid_1's auc: 0.727\n",
      "[86]\ttraining's auc: 0.872741\tvalid_1's auc: 0.727067\n",
      "[87]\ttraining's auc: 0.872772\tvalid_1's auc: 0.727234\n",
      "[88]\ttraining's auc: 0.872827\tvalid_1's auc: 0.727567\n",
      "[89]\ttraining's auc: 0.872875\tvalid_1's auc: 0.727917\n",
      "[90]\ttraining's auc: 0.872912\tvalid_1's auc: 0.728181\n",
      "[91]\ttraining's auc: 0.872941\tvalid_1's auc: 0.728213\n",
      "[92]\ttraining's auc: 0.872963\tvalid_1's auc: 0.728273\n",
      "[93]\ttraining's auc: 0.872983\tvalid_1's auc: 0.728276\n",
      "[94]\ttraining's auc: 0.872999\tvalid_1's auc: 0.72833\n",
      "[95]\ttraining's auc: 0.873024\tvalid_1's auc: 0.728501\n",
      "[96]\ttraining's auc: 0.87304\tvalid_1's auc: 0.728567\n",
      "[97]\ttraining's auc: 0.873056\tvalid_1's auc: 0.728581\n",
      "[98]\ttraining's auc: 0.873072\tvalid_1's auc: 0.728607\n",
      "[99]\ttraining's auc: 0.873087\tvalid_1's auc: 0.728655\n",
      "[100]\ttraining's auc: 0.873103\tvalid_1's auc: 0.728687\n",
      "[101]\ttraining's auc: 0.873118\tvalid_1's auc: 0.728698\n",
      "[102]\ttraining's auc: 0.873135\tvalid_1's auc: 0.728692\n",
      "[103]\ttraining's auc: 0.873148\tvalid_1's auc: 0.728743\n",
      "[104]\ttraining's auc: 0.873174\tvalid_1's auc: 0.728838\n",
      "[105]\ttraining's auc: 0.873198\tvalid_1's auc: 0.728919\n",
      "[106]\ttraining's auc: 0.873208\tvalid_1's auc: 0.728945\n",
      "[107]\ttraining's auc: 0.873239\tvalid_1's auc: 0.728979\n",
      "[108]\ttraining's auc: 0.873251\tvalid_1's auc: 0.729089\n",
      "[109]\ttraining's auc: 0.873264\tvalid_1's auc: 0.729164\n",
      "[110]\ttraining's auc: 0.873281\tvalid_1's auc: 0.729311\n",
      "[111]\ttraining's auc: 0.8733\tvalid_1's auc: 0.729298\n",
      "[112]\ttraining's auc: 0.873313\tvalid_1's auc: 0.729269\n",
      "[113]\ttraining's auc: 0.873328\tvalid_1's auc: 0.72928\n",
      "[114]\ttraining's auc: 0.873345\tvalid_1's auc: 0.729284\n",
      "[115]\ttraining's auc: 0.873364\tvalid_1's auc: 0.729306\n",
      "[116]\ttraining's auc: 0.87338\tvalid_1's auc: 0.729307\n",
      "[117]\ttraining's auc: 0.873405\tvalid_1's auc: 0.729516\n",
      "[118]\ttraining's auc: 0.873439\tvalid_1's auc: 0.729616\n",
      "[119]\ttraining's auc: 0.873463\tvalid_1's auc: 0.729622\n",
      "[120]\ttraining's auc: 0.87348\tvalid_1's auc: 0.729737\n",
      "[121]\ttraining's auc: 0.873494\tvalid_1's auc: 0.729845\n",
      "[122]\ttraining's auc: 0.873509\tvalid_1's auc: 0.729863\n",
      "[123]\ttraining's auc: 0.873522\tvalid_1's auc: 0.72992\n",
      "[124]\ttraining's auc: 0.873533\tvalid_1's auc: 0.729984\n",
      "[125]\ttraining's auc: 0.873546\tvalid_1's auc: 0.730064\n",
      "[126]\ttraining's auc: 0.873562\tvalid_1's auc: 0.730097\n",
      "[127]\ttraining's auc: 0.873575\tvalid_1's auc: 0.7301\n",
      "[128]\ttraining's auc: 0.87359\tvalid_1's auc: 0.730184\n",
      "[129]\ttraining's auc: 0.873621\tvalid_1's auc: 0.730242\n",
      "[130]\ttraining's auc: 0.873635\tvalid_1's auc: 0.730267\n",
      "[131]\ttraining's auc: 0.873649\tvalid_1's auc: 0.730273\n",
      "[132]\ttraining's auc: 0.873651\tvalid_1's auc: 0.730271\n",
      "[133]\ttraining's auc: 0.873665\tvalid_1's auc: 0.73039\n",
      "[134]\ttraining's auc: 0.873671\tvalid_1's auc: 0.730376\n",
      "[135]\ttraining's auc: 0.873681\tvalid_1's auc: 0.730397\n",
      "[136]\ttraining's auc: 0.87369\tvalid_1's auc: 0.730409\n",
      "[137]\ttraining's auc: 0.873704\tvalid_1's auc: 0.730445\n",
      "[138]\ttraining's auc: 0.873716\tvalid_1's auc: 0.730469\n",
      "[139]\ttraining's auc: 0.873723\tvalid_1's auc: 0.730473\n",
      "[140]\ttraining's auc: 0.873732\tvalid_1's auc: 0.730451\n",
      "[141]\ttraining's auc: 0.873738\tvalid_1's auc: 0.730468\n",
      "[142]\ttraining's auc: 0.87375\tvalid_1's auc: 0.730508\n",
      "[143]\ttraining's auc: 0.873763\tvalid_1's auc: 0.730525\n",
      "[144]\ttraining's auc: 0.87378\tvalid_1's auc: 0.730541\n",
      "[145]\ttraining's auc: 0.873791\tvalid_1's auc: 0.730613\n",
      "[146]\ttraining's auc: 0.873806\tvalid_1's auc: 0.730682\n",
      "[147]\ttraining's auc: 0.873813\tvalid_1's auc: 0.73067\n",
      "[148]\ttraining's auc: 0.873818\tvalid_1's auc: 0.730674\n",
      "[149]\ttraining's auc: 0.873822\tvalid_1's auc: 0.730668\n",
      "[150]\ttraining's auc: 0.87383\tvalid_1's auc: 0.730682\n",
      "[151]\ttraining's auc: 0.873836\tvalid_1's auc: 0.730682\n",
      "[152]\ttraining's auc: 0.873851\tvalid_1's auc: 0.730712\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[153]\ttraining's auc: 0.873858\tvalid_1's auc: 0.730724\n",
      "[154]\ttraining's auc: 0.873886\tvalid_1's auc: 0.730802\n",
      "[155]\ttraining's auc: 0.87389\tvalid_1's auc: 0.73081\n",
      "[156]\ttraining's auc: 0.873899\tvalid_1's auc: 0.730835\n",
      "[157]\ttraining's auc: 0.873906\tvalid_1's auc: 0.730798\n",
      "[158]\ttraining's auc: 0.873915\tvalid_1's auc: 0.730876\n",
      "[159]\ttraining's auc: 0.873922\tvalid_1's auc: 0.730879\n",
      "[160]\ttraining's auc: 0.873937\tvalid_1's auc: 0.73089\n",
      "[161]\ttraining's auc: 0.873957\tvalid_1's auc: 0.730971\n",
      "[162]\ttraining's auc: 0.873962\tvalid_1's auc: 0.73098\n",
      "[163]\ttraining's auc: 0.873967\tvalid_1's auc: 0.730994\n",
      "[164]\ttraining's auc: 0.873976\tvalid_1's auc: 0.731012\n",
      "[165]\ttraining's auc: 0.873984\tvalid_1's auc: 0.731033\n",
      "[166]\ttraining's auc: 0.873991\tvalid_1's auc: 0.731077\n",
      "[167]\ttraining's auc: 0.874003\tvalid_1's auc: 0.73104\n",
      "[168]\ttraining's auc: 0.874007\tvalid_1's auc: 0.731051\n",
      "[169]\ttraining's auc: 0.874033\tvalid_1's auc: 0.731091\n",
      "[170]\ttraining's auc: 0.874038\tvalid_1's auc: 0.731077\n",
      "[171]\ttraining's auc: 0.874065\tvalid_1's auc: 0.731221\n",
      "[172]\ttraining's auc: 0.874069\tvalid_1's auc: 0.731213\n",
      "[173]\ttraining's auc: 0.874079\tvalid_1's auc: 0.731222\n",
      "[174]\ttraining's auc: 0.874086\tvalid_1's auc: 0.731236\n",
      "[175]\ttraining's auc: 0.874093\tvalid_1's auc: 0.731247\n",
      "[176]\ttraining's auc: 0.874098\tvalid_1's auc: 0.73124\n",
      "[177]\ttraining's auc: 0.874106\tvalid_1's auc: 0.73126\n",
      "[178]\ttraining's auc: 0.87411\tvalid_1's auc: 0.731267\n",
      "[179]\ttraining's auc: 0.874116\tvalid_1's auc: 0.731293\n",
      "[180]\ttraining's auc: 0.874122\tvalid_1's auc: 0.731293\n",
      "[181]\ttraining's auc: 0.874129\tvalid_1's auc: 0.731293\n",
      "[182]\ttraining's auc: 0.874134\tvalid_1's auc: 0.73131\n",
      "[183]\ttraining's auc: 0.874142\tvalid_1's auc: 0.731309\n",
      "[184]\ttraining's auc: 0.874145\tvalid_1's auc: 0.73131\n",
      "[185]\ttraining's auc: 0.874156\tvalid_1's auc: 0.731345\n",
      "[186]\ttraining's auc: 0.874158\tvalid_1's auc: 0.731351\n",
      "[187]\ttraining's auc: 0.874163\tvalid_1's auc: 0.731343\n",
      "[188]\ttraining's auc: 0.874166\tvalid_1's auc: 0.731342\n",
      "[189]\ttraining's auc: 0.87417\tvalid_1's auc: 0.731346\n",
      "[190]\ttraining's auc: 0.874177\tvalid_1's auc: 0.731367\n",
      "[191]\ttraining's auc: 0.874181\tvalid_1's auc: 0.731366\n",
      "[192]\ttraining's auc: 0.87419\tvalid_1's auc: 0.73139\n",
      "[193]\ttraining's auc: 0.874193\tvalid_1's auc: 0.731424\n",
      "[194]\ttraining's auc: 0.874198\tvalid_1's auc: 0.731426\n",
      "[195]\ttraining's auc: 0.874208\tvalid_1's auc: 0.731457\n",
      "[196]\ttraining's auc: 0.874212\tvalid_1's auc: 0.731436\n",
      "[197]\ttraining's auc: 0.874218\tvalid_1's auc: 0.731437\n",
      "[198]\ttraining's auc: 0.874228\tvalid_1's auc: 0.731545\n",
      "[199]\ttraining's auc: 0.874232\tvalid_1's auc: 0.731545\n",
      "[200]\ttraining's auc: 0.874238\tvalid_1's auc: 0.731586\n",
      "[201]\ttraining's auc: 0.874245\tvalid_1's auc: 0.731608\n",
      "[202]\ttraining's auc: 0.874247\tvalid_1's auc: 0.731603\n",
      "[203]\ttraining's auc: 0.874251\tvalid_1's auc: 0.731604\n",
      "[204]\ttraining's auc: 0.874256\tvalid_1's auc: 0.731603\n",
      "[205]\ttraining's auc: 0.874261\tvalid_1's auc: 0.7316\n",
      "[206]\ttraining's auc: 0.874269\tvalid_1's auc: 0.731596\n",
      "[207]\ttraining's auc: 0.874274\tvalid_1's auc: 0.731608\n",
      "[208]\ttraining's auc: 0.874292\tvalid_1's auc: 0.731631\n",
      "[209]\ttraining's auc: 0.874299\tvalid_1's auc: 0.731629\n",
      "[210]\ttraining's auc: 0.874302\tvalid_1's auc: 0.731643\n",
      "[211]\ttraining's auc: 0.874307\tvalid_1's auc: 0.731648\n",
      "[212]\ttraining's auc: 0.874315\tvalid_1's auc: 0.731642\n",
      "[213]\ttraining's auc: 0.874323\tvalid_1's auc: 0.731706\n",
      "[214]\ttraining's auc: 0.874329\tvalid_1's auc: 0.73168\n",
      "[215]\ttraining's auc: 0.874342\tvalid_1's auc: 0.731669\n",
      "[216]\ttraining's auc: 0.874349\tvalid_1's auc: 0.731671\n",
      "[217]\ttraining's auc: 0.874354\tvalid_1's auc: 0.731667\n",
      "[218]\ttraining's auc: 0.874367\tvalid_1's auc: 0.731673\n",
      "[219]\ttraining's auc: 0.874373\tvalid_1's auc: 0.731688\n",
      "[220]\ttraining's auc: 0.874378\tvalid_1's auc: 0.73169\n",
      "[221]\ttraining's auc: 0.874384\tvalid_1's auc: 0.731696\n",
      "[222]\ttraining's auc: 0.874392\tvalid_1's auc: 0.73172\n",
      "[223]\ttraining's auc: 0.8744\tvalid_1's auc: 0.731712\n",
      "[224]\ttraining's auc: 0.874408\tvalid_1's auc: 0.73174\n",
      "[225]\ttraining's auc: 0.874413\tvalid_1's auc: 0.731707\n",
      "[226]\ttraining's auc: 0.874417\tvalid_1's auc: 0.731712\n",
      "[227]\ttraining's auc: 0.87442\tvalid_1's auc: 0.731713\n",
      "[228]\ttraining's auc: 0.874425\tvalid_1's auc: 0.731714\n",
      "[229]\ttraining's auc: 0.874434\tvalid_1's auc: 0.731746\n",
      "[230]\ttraining's auc: 0.874438\tvalid_1's auc: 0.731733\n",
      "[231]\ttraining's auc: 0.874447\tvalid_1's auc: 0.731783\n",
      "[232]\ttraining's auc: 0.87445\tvalid_1's auc: 0.731788\n",
      "[233]\ttraining's auc: 0.874455\tvalid_1's auc: 0.731823\n",
      "[234]\ttraining's auc: 0.874466\tvalid_1's auc: 0.731822\n",
      "[235]\ttraining's auc: 0.874469\tvalid_1's auc: 0.73181\n",
      "[236]\ttraining's auc: 0.874473\tvalid_1's auc: 0.731812\n",
      "[237]\ttraining's auc: 0.874476\tvalid_1's auc: 0.731817\n",
      "[238]\ttraining's auc: 0.87448\tvalid_1's auc: 0.731827\n",
      "[239]\ttraining's auc: 0.874482\tvalid_1's auc: 0.731822\n",
      "[240]\ttraining's auc: 0.874486\tvalid_1's auc: 0.731821\n",
      "[241]\ttraining's auc: 0.874489\tvalid_1's auc: 0.731827\n",
      "[242]\ttraining's auc: 0.87449\tvalid_1's auc: 0.731826\n",
      "[243]\ttraining's auc: 0.874497\tvalid_1's auc: 0.731816\n",
      "[244]\ttraining's auc: 0.8745\tvalid_1's auc: 0.731806\n",
      "[245]\ttraining's auc: 0.874504\tvalid_1's auc: 0.731802\n",
      "[246]\ttraining's auc: 0.874509\tvalid_1's auc: 0.731798\n",
      "[247]\ttraining's auc: 0.874515\tvalid_1's auc: 0.731799\n",
      "[248]\ttraining's auc: 0.874521\tvalid_1's auc: 0.731799\n",
      "[249]\ttraining's auc: 0.874527\tvalid_1's auc: 0.731791\n",
      "[250]\ttraining's auc: 0.874531\tvalid_1's auc: 0.731778\n",
      "[251]\ttraining's auc: 0.87454\tvalid_1's auc: 0.731763\n",
      "[252]\ttraining's auc: 0.874545\tvalid_1's auc: 0.731792\n",
      "[253]\ttraining's auc: 0.874549\tvalid_1's auc: 0.731781\n",
      "[254]\ttraining's auc: 0.874564\tvalid_1's auc: 0.731774\n",
      "[255]\ttraining's auc: 0.874569\tvalid_1's auc: 0.731765\n",
      "[256]\ttraining's auc: 0.874576\tvalid_1's auc: 0.731772\n",
      "[257]\ttraining's auc: 0.874583\tvalid_1's auc: 0.731819\n",
      "[258]\ttraining's auc: 0.874587\tvalid_1's auc: 0.731826\n",
      "[259]\ttraining's auc: 0.874592\tvalid_1's auc: 0.731834\n",
      "[260]\ttraining's auc: 0.8746\tvalid_1's auc: 0.731838\n",
      "[261]\ttraining's auc: 0.874607\tvalid_1's auc: 0.731863\n",
      "[262]\ttraining's auc: 0.874618\tvalid_1's auc: 0.731916\n",
      "[263]\ttraining's auc: 0.874625\tvalid_1's auc: 0.731924\n",
      "[264]\ttraining's auc: 0.87463\tvalid_1's auc: 0.73192\n",
      "[265]\ttraining's auc: 0.874634\tvalid_1's auc: 0.731918\n",
      "[266]\ttraining's auc: 0.874638\tvalid_1's auc: 0.73192\n",
      "[267]\ttraining's auc: 0.874641\tvalid_1's auc: 0.731921\n",
      "[268]\ttraining's auc: 0.874645\tvalid_1's auc: 0.731912\n",
      "[269]\ttraining's auc: 0.874651\tvalid_1's auc: 0.731914\n",
      "[270]\ttraining's auc: 0.874657\tvalid_1's auc: 0.731919\n",
      "[271]\ttraining's auc: 0.874659\tvalid_1's auc: 0.731919\n",
      "[272]\ttraining's auc: 0.874661\tvalid_1's auc: 0.731913\n",
      "[273]\ttraining's auc: 0.874664\tvalid_1's auc: 0.731901\n",
      "[274]\ttraining's auc: 0.874668\tvalid_1's auc: 0.731901\n",
      "[275]\ttraining's auc: 0.874672\tvalid_1's auc: 0.731904\n",
      "[276]\ttraining's auc: 0.874678\tvalid_1's auc: 0.731908\n",
      "[277]\ttraining's auc: 0.874681\tvalid_1's auc: 0.731914\n",
      "[278]\ttraining's auc: 0.874684\tvalid_1's auc: 0.731912\n",
      "[279]\ttraining's auc: 0.874689\tvalid_1's auc: 0.731912\n",
      "[280]\ttraining's auc: 0.874692\tvalid_1's auc: 0.73192\n",
      "[281]\ttraining's auc: 0.874698\tvalid_1's auc: 0.731948\n",
      "[282]\ttraining's auc: 0.874707\tvalid_1's auc: 0.731955\n",
      "[283]\ttraining's auc: 0.874711\tvalid_1's auc: 0.731957\n",
      "[284]\ttraining's auc: 0.874718\tvalid_1's auc: 0.73198\n",
      "[285]\ttraining's auc: 0.874722\tvalid_1's auc: 0.73197\n",
      "[286]\ttraining's auc: 0.874725\tvalid_1's auc: 0.731966\n",
      "[287]\ttraining's auc: 0.87473\tvalid_1's auc: 0.731986\n",
      "[288]\ttraining's auc: 0.874744\tvalid_1's auc: 0.731953\n",
      "[289]\ttraining's auc: 0.874747\tvalid_1's auc: 0.731945\n",
      "[290]\ttraining's auc: 0.874755\tvalid_1's auc: 0.731984\n",
      "[291]\ttraining's auc: 0.874759\tvalid_1's auc: 0.731982\n",
      "[292]\ttraining's auc: 0.874768\tvalid_1's auc: 0.732004\n",
      "[293]\ttraining's auc: 0.874772\tvalid_1's auc: 0.732008\n",
      "[294]\ttraining's auc: 0.874778\tvalid_1's auc: 0.732015\n",
      "[295]\ttraining's auc: 0.874779\tvalid_1's auc: 0.732014\n",
      "[296]\ttraining's auc: 0.874782\tvalid_1's auc: 0.73201\n",
      "[297]\ttraining's auc: 0.874791\tvalid_1's auc: 0.732031\n",
      "[298]\ttraining's auc: 0.874794\tvalid_1's auc: 0.732032\n",
      "[299]\ttraining's auc: 0.874802\tvalid_1's auc: 0.732048\n",
      "[300]\ttraining's auc: 0.874803\tvalid_1's auc: 0.732049\n",
      "[301]\ttraining's auc: 0.874808\tvalid_1's auc: 0.732032\n",
      "[302]\ttraining's auc: 0.874814\tvalid_1's auc: 0.732051\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[303]\ttraining's auc: 0.874819\tvalid_1's auc: 0.732073\n",
      "[304]\ttraining's auc: 0.874828\tvalid_1's auc: 0.732051\n",
      "[305]\ttraining's auc: 0.87484\tvalid_1's auc: 0.732064\n",
      "[306]\ttraining's auc: 0.874851\tvalid_1's auc: 0.732169\n",
      "[307]\ttraining's auc: 0.874856\tvalid_1's auc: 0.732181\n",
      "[308]\ttraining's auc: 0.874863\tvalid_1's auc: 0.732164\n",
      "[309]\ttraining's auc: 0.874867\tvalid_1's auc: 0.732159\n",
      "[310]\ttraining's auc: 0.87487\tvalid_1's auc: 0.732144\n",
      "[311]\ttraining's auc: 0.874873\tvalid_1's auc: 0.732148\n",
      "[312]\ttraining's auc: 0.874876\tvalid_1's auc: 0.73215\n",
      "[313]\ttraining's auc: 0.874879\tvalid_1's auc: 0.732148\n",
      "[314]\ttraining's auc: 0.874885\tvalid_1's auc: 0.732132\n",
      "[315]\ttraining's auc: 0.874887\tvalid_1's auc: 0.732131\n",
      "[316]\ttraining's auc: 0.87489\tvalid_1's auc: 0.732141\n",
      "[317]\ttraining's auc: 0.874893\tvalid_1's auc: 0.732135\n",
      "[318]\ttraining's auc: 0.874898\tvalid_1's auc: 0.732141\n",
      "[319]\ttraining's auc: 0.874914\tvalid_1's auc: 0.732147\n",
      "[320]\ttraining's auc: 0.874922\tvalid_1's auc: 0.732163\n",
      "[321]\ttraining's auc: 0.874931\tvalid_1's auc: 0.732211\n",
      "[322]\ttraining's auc: 0.874938\tvalid_1's auc: 0.732214\n",
      "[323]\ttraining's auc: 0.87494\tvalid_1's auc: 0.732217\n",
      "[324]\ttraining's auc: 0.874942\tvalid_1's auc: 0.732218\n",
      "[325]\ttraining's auc: 0.874948\tvalid_1's auc: 0.732232\n",
      "[326]\ttraining's auc: 0.874952\tvalid_1's auc: 0.732233\n",
      "[327]\ttraining's auc: 0.874956\tvalid_1's auc: 0.732235\n",
      "[328]\ttraining's auc: 0.87496\tvalid_1's auc: 0.73223\n",
      "[329]\ttraining's auc: 0.874963\tvalid_1's auc: 0.732232\n",
      "[330]\ttraining's auc: 0.874966\tvalid_1's auc: 0.732219\n",
      "[331]\ttraining's auc: 0.874969\tvalid_1's auc: 0.732223\n",
      "[332]\ttraining's auc: 0.874979\tvalid_1's auc: 0.732236\n",
      "[333]\ttraining's auc: 0.874985\tvalid_1's auc: 0.732232\n",
      "[334]\ttraining's auc: 0.874988\tvalid_1's auc: 0.73223\n",
      "[335]\ttraining's auc: 0.874992\tvalid_1's auc: 0.732228\n",
      "[336]\ttraining's auc: 0.874996\tvalid_1's auc: 0.732234\n",
      "[337]\ttraining's auc: 0.875001\tvalid_1's auc: 0.732276\n",
      "[338]\ttraining's auc: 0.875004\tvalid_1's auc: 0.732288\n",
      "[339]\ttraining's auc: 0.875007\tvalid_1's auc: 0.732291\n",
      "[340]\ttraining's auc: 0.875012\tvalid_1's auc: 0.732299\n",
      "[341]\ttraining's auc: 0.875015\tvalid_1's auc: 0.732291\n",
      "[342]\ttraining's auc: 0.875021\tvalid_1's auc: 0.732306\n",
      "[343]\ttraining's auc: 0.875025\tvalid_1's auc: 0.732307\n",
      "[344]\ttraining's auc: 0.875028\tvalid_1's auc: 0.732302\n",
      "[345]\ttraining's auc: 0.875036\tvalid_1's auc: 0.732289\n",
      "[346]\ttraining's auc: 0.875039\tvalid_1's auc: 0.732301\n",
      "[347]\ttraining's auc: 0.875042\tvalid_1's auc: 0.732325\n",
      "[348]\ttraining's auc: 0.875047\tvalid_1's auc: 0.732324\n",
      "[349]\ttraining's auc: 0.875051\tvalid_1's auc: 0.732317\n",
      "[350]\ttraining's auc: 0.875055\tvalid_1's auc: 0.732324\n",
      "[351]\ttraining's auc: 0.875058\tvalid_1's auc: 0.732348\n",
      "[352]\ttraining's auc: 0.875062\tvalid_1's auc: 0.732346\n",
      "[353]\ttraining's auc: 0.875066\tvalid_1's auc: 0.732351\n",
      "[354]\ttraining's auc: 0.875075\tvalid_1's auc: 0.732407\n",
      "[355]\ttraining's auc: 0.87508\tvalid_1's auc: 0.732405\n",
      "[356]\ttraining's auc: 0.875094\tvalid_1's auc: 0.732404\n",
      "[357]\ttraining's auc: 0.875096\tvalid_1's auc: 0.732411\n",
      "[358]\ttraining's auc: 0.8751\tvalid_1's auc: 0.73241\n",
      "[359]\ttraining's auc: 0.875104\tvalid_1's auc: 0.732404\n",
      "[360]\ttraining's auc: 0.875108\tvalid_1's auc: 0.732403\n",
      "[361]\ttraining's auc: 0.87511\tvalid_1's auc: 0.732414\n",
      "[362]\ttraining's auc: 0.875113\tvalid_1's auc: 0.732402\n",
      "[363]\ttraining's auc: 0.875116\tvalid_1's auc: 0.732405\n",
      "[364]\ttraining's auc: 0.87512\tvalid_1's auc: 0.73242\n",
      "[365]\ttraining's auc: 0.875123\tvalid_1's auc: 0.73242\n",
      "[366]\ttraining's auc: 0.875127\tvalid_1's auc: 0.732415\n",
      "[367]\ttraining's auc: 0.875133\tvalid_1's auc: 0.732411\n",
      "[368]\ttraining's auc: 0.875137\tvalid_1's auc: 0.73242\n",
      "[369]\ttraining's auc: 0.87514\tvalid_1's auc: 0.732437\n",
      "[370]\ttraining's auc: 0.875144\tvalid_1's auc: 0.732446\n",
      "[371]\ttraining's auc: 0.875147\tvalid_1's auc: 0.732442\n",
      "[372]\ttraining's auc: 0.875152\tvalid_1's auc: 0.732439\n",
      "[373]\ttraining's auc: 0.875156\tvalid_1's auc: 0.732447\n",
      "[374]\ttraining's auc: 0.875163\tvalid_1's auc: 0.732447\n",
      "[375]\ttraining's auc: 0.875168\tvalid_1's auc: 0.732447\n",
      "[376]\ttraining's auc: 0.875173\tvalid_1's auc: 0.73246\n",
      "[377]\ttraining's auc: 0.875177\tvalid_1's auc: 0.732447\n",
      "[378]\ttraining's auc: 0.87518\tvalid_1's auc: 0.732443\n",
      "[379]\ttraining's auc: 0.875184\tvalid_1's auc: 0.732449\n",
      "[380]\ttraining's auc: 0.875188\tvalid_1's auc: 0.732447\n",
      "[381]\ttraining's auc: 0.87519\tvalid_1's auc: 0.732456\n",
      "[382]\ttraining's auc: 0.875192\tvalid_1's auc: 0.732455\n",
      "[383]\ttraining's auc: 0.875202\tvalid_1's auc: 0.732479\n",
      "[384]\ttraining's auc: 0.875206\tvalid_1's auc: 0.732479\n",
      "[385]\ttraining's auc: 0.875209\tvalid_1's auc: 0.732471\n",
      "[386]\ttraining's auc: 0.875212\tvalid_1's auc: 0.732473\n",
      "[387]\ttraining's auc: 0.875217\tvalid_1's auc: 0.732478\n",
      "[388]\ttraining's auc: 0.875221\tvalid_1's auc: 0.732453\n",
      "[389]\ttraining's auc: 0.875225\tvalid_1's auc: 0.732482\n",
      "[390]\ttraining's auc: 0.875232\tvalid_1's auc: 0.732486\n",
      "[391]\ttraining's auc: 0.875234\tvalid_1's auc: 0.732476\n",
      "[392]\ttraining's auc: 0.875237\tvalid_1's auc: 0.732485\n",
      "[393]\ttraining's auc: 0.875242\tvalid_1's auc: 0.732487\n",
      "[394]\ttraining's auc: 0.875248\tvalid_1's auc: 0.732535\n",
      "[395]\ttraining's auc: 0.875252\tvalid_1's auc: 0.73254\n",
      "[396]\ttraining's auc: 0.875257\tvalid_1's auc: 0.732543\n",
      "[397]\ttraining's auc: 0.875261\tvalid_1's auc: 0.732543\n",
      "[398]\ttraining's auc: 0.875264\tvalid_1's auc: 0.732545\n",
      "[399]\ttraining's auc: 0.875271\tvalid_1's auc: 0.73255\n",
      "[400]\ttraining's auc: 0.875277\tvalid_1's auc: 0.732526\n",
      "[401]\ttraining's auc: 0.875283\tvalid_1's auc: 0.73254\n",
      "[402]\ttraining's auc: 0.875286\tvalid_1's auc: 0.732536\n",
      "[403]\ttraining's auc: 0.87529\tvalid_1's auc: 0.732549\n",
      "[404]\ttraining's auc: 0.875294\tvalid_1's auc: 0.732563\n",
      "[405]\ttraining's auc: 0.875299\tvalid_1's auc: 0.732567\n",
      "[406]\ttraining's auc: 0.875302\tvalid_1's auc: 0.732569\n",
      "[407]\ttraining's auc: 0.875308\tvalid_1's auc: 0.732543\n",
      "[408]\ttraining's auc: 0.875312\tvalid_1's auc: 0.73255\n",
      "[409]\ttraining's auc: 0.875315\tvalid_1's auc: 0.73255\n",
      "[410]\ttraining's auc: 0.875318\tvalid_1's auc: 0.732547\n",
      "[411]\ttraining's auc: 0.875321\tvalid_1's auc: 0.732543\n",
      "[412]\ttraining's auc: 0.875326\tvalid_1's auc: 0.732535\n",
      "[413]\ttraining's auc: 0.875328\tvalid_1's auc: 0.73254\n",
      "[414]\ttraining's auc: 0.875333\tvalid_1's auc: 0.732542\n",
      "[415]\ttraining's auc: 0.875336\tvalid_1's auc: 0.732543\n",
      "[416]\ttraining's auc: 0.87534\tvalid_1's auc: 0.732541\n",
      "[417]\ttraining's auc: 0.875345\tvalid_1's auc: 0.732543\n",
      "[418]\ttraining's auc: 0.875349\tvalid_1's auc: 0.732544\n",
      "[419]\ttraining's auc: 0.875352\tvalid_1's auc: 0.732545\n",
      "[420]\ttraining's auc: 0.875357\tvalid_1's auc: 0.732551\n",
      "[421]\ttraining's auc: 0.87536\tvalid_1's auc: 0.732559\n",
      "[422]\ttraining's auc: 0.875366\tvalid_1's auc: 0.732557\n",
      "[423]\ttraining's auc: 0.87537\tvalid_1's auc: 0.732556\n",
      "[424]\ttraining's auc: 0.875374\tvalid_1's auc: 0.732543\n",
      "[425]\ttraining's auc: 0.875379\tvalid_1's auc: 0.732518\n",
      "[426]\ttraining's auc: 0.875385\tvalid_1's auc: 0.732514\n",
      "[427]\ttraining's auc: 0.875387\tvalid_1's auc: 0.732509\n",
      "[428]\ttraining's auc: 0.875394\tvalid_1's auc: 0.732503\n",
      "[429]\ttraining's auc: 0.875397\tvalid_1's auc: 0.732501\n",
      "[430]\ttraining's auc: 0.8754\tvalid_1's auc: 0.73249\n",
      "[431]\ttraining's auc: 0.875403\tvalid_1's auc: 0.732486\n",
      "[432]\ttraining's auc: 0.875407\tvalid_1's auc: 0.732492\n",
      "[433]\ttraining's auc: 0.87541\tvalid_1's auc: 0.732495\n",
      "[434]\ttraining's auc: 0.875416\tvalid_1's auc: 0.732512\n",
      "[435]\ttraining's auc: 0.875422\tvalid_1's auc: 0.732538\n",
      "[436]\ttraining's auc: 0.875425\tvalid_1's auc: 0.732529\n",
      "[437]\ttraining's auc: 0.875431\tvalid_1's auc: 0.732529\n",
      "[438]\ttraining's auc: 0.875434\tvalid_1's auc: 0.732517\n",
      "[439]\ttraining's auc: 0.875437\tvalid_1's auc: 0.732523\n",
      "[440]\ttraining's auc: 0.875439\tvalid_1's auc: 0.732526\n",
      "[441]\ttraining's auc: 0.875444\tvalid_1's auc: 0.732497\n",
      "[442]\ttraining's auc: 0.875449\tvalid_1's auc: 0.732512\n",
      "[443]\ttraining's auc: 0.875453\tvalid_1's auc: 0.732491\n",
      "[444]\ttraining's auc: 0.875456\tvalid_1's auc: 0.732487\n",
      "[445]\ttraining's auc: 0.875462\tvalid_1's auc: 0.732496\n",
      "[446]\ttraining's auc: 0.875466\tvalid_1's auc: 0.732468\n",
      "[447]\ttraining's auc: 0.87547\tvalid_1's auc: 0.732471\n",
      "[448]\ttraining's auc: 0.875477\tvalid_1's auc: 0.732466\n",
      "[449]\ttraining's auc: 0.875479\tvalid_1's auc: 0.732466\n",
      "[450]\ttraining's auc: 0.875483\tvalid_1's auc: 0.732477\n",
      "[451]\ttraining's auc: 0.875486\tvalid_1's auc: 0.732484\n",
      "[452]\ttraining's auc: 0.875491\tvalid_1's auc: 0.732488\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[453]\ttraining's auc: 0.875493\tvalid_1's auc: 0.73249\n",
      "[454]\ttraining's auc: 0.875496\tvalid_1's auc: 0.73249\n",
      "[455]\ttraining's auc: 0.875501\tvalid_1's auc: 0.732486\n",
      "[456]\ttraining's auc: 0.875504\tvalid_1's auc: 0.732481\n",
      "[457]\ttraining's auc: 0.875508\tvalid_1's auc: 0.732478\n",
      "[458]\ttraining's auc: 0.875513\tvalid_1's auc: 0.732484\n",
      "[459]\ttraining's auc: 0.875516\tvalid_1's auc: 0.732483\n",
      "[460]\ttraining's auc: 0.875519\tvalid_1's auc: 0.732479\n",
      "[461]\ttraining's auc: 0.875523\tvalid_1's auc: 0.732478\n",
      "[462]\ttraining's auc: 0.875526\tvalid_1's auc: 0.732475\n",
      "[463]\ttraining's auc: 0.875529\tvalid_1's auc: 0.732483\n",
      "[464]\ttraining's auc: 0.875534\tvalid_1's auc: 0.732493\n",
      "[465]\ttraining's auc: 0.875536\tvalid_1's auc: 0.732494\n",
      "[466]\ttraining's auc: 0.875541\tvalid_1's auc: 0.732499\n",
      "[467]\ttraining's auc: 0.875544\tvalid_1's auc: 0.732496\n",
      "[468]\ttraining's auc: 0.875548\tvalid_1's auc: 0.732492\n",
      "[469]\ttraining's auc: 0.875551\tvalid_1's auc: 0.732488\n",
      "[470]\ttraining's auc: 0.875557\tvalid_1's auc: 0.732485\n",
      "[471]\ttraining's auc: 0.875559\tvalid_1's auc: 0.732485\n",
      "[472]\ttraining's auc: 0.875561\tvalid_1's auc: 0.732486\n",
      "[473]\ttraining's auc: 0.875566\tvalid_1's auc: 0.732497\n",
      "[474]\ttraining's auc: 0.87557\tvalid_1's auc: 0.732506\n",
      "[475]\ttraining's auc: 0.875574\tvalid_1's auc: 0.732493\n",
      "[476]\ttraining's auc: 0.875579\tvalid_1's auc: 0.732489\n",
      "[477]\ttraining's auc: 0.875584\tvalid_1's auc: 0.732492\n",
      "[478]\ttraining's auc: 0.875588\tvalid_1's auc: 0.732493\n",
      "[479]\ttraining's auc: 0.87559\tvalid_1's auc: 0.732497\n",
      "[480]\ttraining's auc: 0.875594\tvalid_1's auc: 0.73249\n",
      "[481]\ttraining's auc: 0.875596\tvalid_1's auc: 0.732482\n",
      "[482]\ttraining's auc: 0.8756\tvalid_1's auc: 0.732477\n",
      "[483]\ttraining's auc: 0.875604\tvalid_1's auc: 0.732476\n",
      "[484]\ttraining's auc: 0.875611\tvalid_1's auc: 0.732482\n",
      "[485]\ttraining's auc: 0.875614\tvalid_1's auc: 0.732483\n",
      "[486]\ttraining's auc: 0.875619\tvalid_1's auc: 0.732477\n",
      "[487]\ttraining's auc: 0.875624\tvalid_1's auc: 0.732478\n",
      "[488]\ttraining's auc: 0.875628\tvalid_1's auc: 0.73248\n",
      "[489]\ttraining's auc: 0.875631\tvalid_1's auc: 0.732461\n",
      "[490]\ttraining's auc: 0.875634\tvalid_1's auc: 0.732452\n",
      "[491]\ttraining's auc: 0.875637\tvalid_1's auc: 0.73246\n",
      "[492]\ttraining's auc: 0.875641\tvalid_1's auc: 0.732486\n",
      "[493]\ttraining's auc: 0.875648\tvalid_1's auc: 0.732483\n",
      "[494]\ttraining's auc: 0.875651\tvalid_1's auc: 0.73248\n",
      "[495]\ttraining's auc: 0.875656\tvalid_1's auc: 0.732483\n",
      "[496]\ttraining's auc: 0.875659\tvalid_1's auc: 0.732489\n",
      "[497]\ttraining's auc: 0.875662\tvalid_1's auc: 0.732492\n",
      "[498]\ttraining's auc: 0.875665\tvalid_1's auc: 0.732493\n",
      "[499]\ttraining's auc: 0.875676\tvalid_1's auc: 0.732507\n",
      "[500]\ttraining's auc: 0.875679\tvalid_1's auc: 0.732507\n",
      "[501]\ttraining's auc: 0.875682\tvalid_1's auc: 0.732504\n",
      "[502]\ttraining's auc: 0.875685\tvalid_1's auc: 0.732507\n",
      "[503]\ttraining's auc: 0.875689\tvalid_1's auc: 0.732481\n",
      "[504]\ttraining's auc: 0.875693\tvalid_1's auc: 0.732485\n",
      "[505]\ttraining's auc: 0.875695\tvalid_1's auc: 0.732482\n",
      "[506]\ttraining's auc: 0.875698\tvalid_1's auc: 0.732489\n",
      "[507]\ttraining's auc: 0.8757\tvalid_1's auc: 0.732481\n",
      "[508]\ttraining's auc: 0.875703\tvalid_1's auc: 0.732483\n",
      "[509]\ttraining's auc: 0.875706\tvalid_1's auc: 0.732476\n",
      "[510]\ttraining's auc: 0.87571\tvalid_1's auc: 0.732471\n",
      "[511]\ttraining's auc: 0.875712\tvalid_1's auc: 0.732485\n",
      "[512]\ttraining's auc: 0.875719\tvalid_1's auc: 0.732547\n",
      "[513]\ttraining's auc: 0.875721\tvalid_1's auc: 0.732539\n",
      "[514]\ttraining's auc: 0.875728\tvalid_1's auc: 0.732522\n",
      "[515]\ttraining's auc: 0.875732\tvalid_1's auc: 0.732515\n",
      "[516]\ttraining's auc: 0.875736\tvalid_1's auc: 0.732526\n",
      "[517]\ttraining's auc: 0.875738\tvalid_1's auc: 0.732528\n",
      "[518]\ttraining's auc: 0.875741\tvalid_1's auc: 0.732548\n",
      "[519]\ttraining's auc: 0.875744\tvalid_1's auc: 0.732546\n",
      "[520]\ttraining's auc: 0.875747\tvalid_1's auc: 0.73252\n",
      "[521]\ttraining's auc: 0.875752\tvalid_1's auc: 0.732522\n",
      "[522]\ttraining's auc: 0.875755\tvalid_1's auc: 0.732514\n",
      "[523]\ttraining's auc: 0.875756\tvalid_1's auc: 0.732496\n",
      "[524]\ttraining's auc: 0.87576\tvalid_1's auc: 0.732499\n",
      "[525]\ttraining's auc: 0.875762\tvalid_1's auc: 0.732478\n",
      "[526]\ttraining's auc: 0.875765\tvalid_1's auc: 0.732485\n",
      "[527]\ttraining's auc: 0.875767\tvalid_1's auc: 0.732485\n",
      "[528]\ttraining's auc: 0.87577\tvalid_1's auc: 0.732485\n",
      "[529]\ttraining's auc: 0.875774\tvalid_1's auc: 0.732515\n",
      "[530]\ttraining's auc: 0.875775\tvalid_1's auc: 0.732513\n",
      "[531]\ttraining's auc: 0.875781\tvalid_1's auc: 0.732488\n",
      "[532]\ttraining's auc: 0.875785\tvalid_1's auc: 0.732484\n",
      "[533]\ttraining's auc: 0.875787\tvalid_1's auc: 0.732478\n",
      "[534]\ttraining's auc: 0.875789\tvalid_1's auc: 0.732482\n",
      "[535]\ttraining's auc: 0.875793\tvalid_1's auc: 0.732495\n",
      "[536]\ttraining's auc: 0.875797\tvalid_1's auc: 0.732484\n",
      "[537]\ttraining's auc: 0.875801\tvalid_1's auc: 0.732477\n",
      "[538]\ttraining's auc: 0.875804\tvalid_1's auc: 0.73247\n",
      "[539]\ttraining's auc: 0.875806\tvalid_1's auc: 0.732471\n",
      "[540]\ttraining's auc: 0.87581\tvalid_1's auc: 0.732467\n",
      "[541]\ttraining's auc: 0.875812\tvalid_1's auc: 0.732475\n",
      "[542]\ttraining's auc: 0.875815\tvalid_1's auc: 0.732461\n",
      "[543]\ttraining's auc: 0.875819\tvalid_1's auc: 0.732459\n",
      "[544]\ttraining's auc: 0.875823\tvalid_1's auc: 0.732473\n",
      "[545]\ttraining's auc: 0.875826\tvalid_1's auc: 0.732474\n",
      "[546]\ttraining's auc: 0.875827\tvalid_1's auc: 0.732468\n",
      "[547]\ttraining's auc: 0.875831\tvalid_1's auc: 0.732469\n",
      "[548]\ttraining's auc: 0.875839\tvalid_1's auc: 0.732486\n",
      "[549]\ttraining's auc: 0.875842\tvalid_1's auc: 0.732485\n",
      "[550]\ttraining's auc: 0.875843\tvalid_1's auc: 0.732488\n",
      "[551]\ttraining's auc: 0.875847\tvalid_1's auc: 0.732491\n",
      "[552]\ttraining's auc: 0.87585\tvalid_1's auc: 0.73249\n",
      "[553]\ttraining's auc: 0.875853\tvalid_1's auc: 0.732493\n",
      "[554]\ttraining's auc: 0.875858\tvalid_1's auc: 0.732482\n",
      "[555]\ttraining's auc: 0.875861\tvalid_1's auc: 0.732482\n",
      "[556]\ttraining's auc: 0.875867\tvalid_1's auc: 0.732451\n",
      "[557]\ttraining's auc: 0.875871\tvalid_1's auc: 0.732459\n",
      "[558]\ttraining's auc: 0.875874\tvalid_1's auc: 0.732458\n",
      "[559]\ttraining's auc: 0.875876\tvalid_1's auc: 0.73245\n",
      "[560]\ttraining's auc: 0.875879\tvalid_1's auc: 0.73245\n",
      "[561]\ttraining's auc: 0.875882\tvalid_1's auc: 0.732453\n",
      "[562]\ttraining's auc: 0.875885\tvalid_1's auc: 0.732444\n",
      "[563]\ttraining's auc: 0.87589\tvalid_1's auc: 0.732436\n",
      "[564]\ttraining's auc: 0.875892\tvalid_1's auc: 0.732431\n",
      "[565]\ttraining's auc: 0.875897\tvalid_1's auc: 0.732401\n",
      "[566]\ttraining's auc: 0.8759\tvalid_1's auc: 0.732392\n",
      "[567]\ttraining's auc: 0.875903\tvalid_1's auc: 0.732403\n",
      "[568]\ttraining's auc: 0.875907\tvalid_1's auc: 0.732394\n",
      "[569]\ttraining's auc: 0.875912\tvalid_1's auc: 0.732399\n",
      "[570]\ttraining's auc: 0.875915\tvalid_1's auc: 0.732408\n",
      "[571]\ttraining's auc: 0.875918\tvalid_1's auc: 0.732364\n",
      "[572]\ttraining's auc: 0.875922\tvalid_1's auc: 0.732363\n",
      "[573]\ttraining's auc: 0.875929\tvalid_1's auc: 0.732377\n",
      "[574]\ttraining's auc: 0.875932\tvalid_1's auc: 0.732403\n",
      "[575]\ttraining's auc: 0.875937\tvalid_1's auc: 0.732387\n",
      "[576]\ttraining's auc: 0.87594\tvalid_1's auc: 0.732355\n",
      "[577]\ttraining's auc: 0.875942\tvalid_1's auc: 0.732347\n",
      "[578]\ttraining's auc: 0.87595\tvalid_1's auc: 0.732339\n",
      "[579]\ttraining's auc: 0.875956\tvalid_1's auc: 0.732354\n",
      "[580]\ttraining's auc: 0.875969\tvalid_1's auc: 0.732656\n",
      "[581]\ttraining's auc: 0.875973\tvalid_1's auc: 0.732654\n",
      "[582]\ttraining's auc: 0.875982\tvalid_1's auc: 0.732701\n",
      "[583]\ttraining's auc: 0.875983\tvalid_1's auc: 0.7327\n",
      "[584]\ttraining's auc: 0.875988\tvalid_1's auc: 0.732704\n",
      "[585]\ttraining's auc: 0.875994\tvalid_1's auc: 0.732705\n",
      "[586]\ttraining's auc: 0.875997\tvalid_1's auc: 0.732703\n",
      "[587]\ttraining's auc: 0.876\tvalid_1's auc: 0.732695\n",
      "[588]\ttraining's auc: 0.876003\tvalid_1's auc: 0.732708\n",
      "[589]\ttraining's auc: 0.876006\tvalid_1's auc: 0.73271\n",
      "[590]\ttraining's auc: 0.876009\tvalid_1's auc: 0.732706\n",
      "[591]\ttraining's auc: 0.876015\tvalid_1's auc: 0.732744\n",
      "[592]\ttraining's auc: 0.876017\tvalid_1's auc: 0.732752\n",
      "[593]\ttraining's auc: 0.87602\tvalid_1's auc: 0.732752\n",
      "[594]\ttraining's auc: 0.876024\tvalid_1's auc: 0.732767\n",
      "[595]\ttraining's auc: 0.876026\tvalid_1's auc: 0.732769\n",
      "[596]\ttraining's auc: 0.876029\tvalid_1's auc: 0.732768\n",
      "[597]\ttraining's auc: 0.876033\tvalid_1's auc: 0.732759\n",
      "[598]\ttraining's auc: 0.876035\tvalid_1's auc: 0.732766\n",
      "[599]\ttraining's auc: 0.876038\tvalid_1's auc: 0.732765\n",
      "[600]\ttraining's auc: 0.876042\tvalid_1's auc: 0.732765\n",
      "[601]\ttraining's auc: 0.876046\tvalid_1's auc: 0.732787\n",
      "[602]\ttraining's auc: 0.876049\tvalid_1's auc: 0.732793\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[603]\ttraining's auc: 0.876052\tvalid_1's auc: 0.73279\n",
      "[604]\ttraining's auc: 0.876054\tvalid_1's auc: 0.732785\n",
      "[605]\ttraining's auc: 0.876057\tvalid_1's auc: 0.732785\n",
      "[606]\ttraining's auc: 0.876064\tvalid_1's auc: 0.732775\n",
      "[607]\ttraining's auc: 0.876068\tvalid_1's auc: 0.73277\n",
      "[608]\ttraining's auc: 0.876072\tvalid_1's auc: 0.732768\n",
      "[609]\ttraining's auc: 0.876076\tvalid_1's auc: 0.732766\n",
      "[610]\ttraining's auc: 0.876079\tvalid_1's auc: 0.73275\n",
      "[611]\ttraining's auc: 0.876081\tvalid_1's auc: 0.732752\n",
      "[612]\ttraining's auc: 0.876087\tvalid_1's auc: 0.732752\n",
      "[613]\ttraining's auc: 0.876091\tvalid_1's auc: 0.732759\n",
      "[614]\ttraining's auc: 0.876095\tvalid_1's auc: 0.732739\n",
      "[615]\ttraining's auc: 0.876099\tvalid_1's auc: 0.732728\n",
      "[616]\ttraining's auc: 0.876103\tvalid_1's auc: 0.732729\n",
      "[617]\ttraining's auc: 0.876106\tvalid_1's auc: 0.732718\n",
      "[618]\ttraining's auc: 0.876109\tvalid_1's auc: 0.732715\n",
      "[619]\ttraining's auc: 0.876115\tvalid_1's auc: 0.73282\n",
      "[620]\ttraining's auc: 0.876119\tvalid_1's auc: 0.73282\n",
      "[621]\ttraining's auc: 0.876121\tvalid_1's auc: 0.732831\n",
      "[622]\ttraining's auc: 0.876125\tvalid_1's auc: 0.732824\n",
      "[623]\ttraining's auc: 0.87613\tvalid_1's auc: 0.73284\n",
      "[624]\ttraining's auc: 0.876133\tvalid_1's auc: 0.732849\n",
      "[625]\ttraining's auc: 0.876136\tvalid_1's auc: 0.732845\n",
      "[626]\ttraining's auc: 0.87614\tvalid_1's auc: 0.732824\n",
      "[627]\ttraining's auc: 0.876142\tvalid_1's auc: 0.732823\n",
      "[628]\ttraining's auc: 0.876147\tvalid_1's auc: 0.732823\n",
      "[629]\ttraining's auc: 0.876152\tvalid_1's auc: 0.73288\n",
      "[630]\ttraining's auc: 0.876155\tvalid_1's auc: 0.732875\n",
      "[631]\ttraining's auc: 0.876161\tvalid_1's auc: 0.732884\n",
      "[632]\ttraining's auc: 0.876164\tvalid_1's auc: 0.732891\n",
      "[633]\ttraining's auc: 0.876167\tvalid_1's auc: 0.732891\n",
      "[634]\ttraining's auc: 0.87617\tvalid_1's auc: 0.732899\n",
      "[635]\ttraining's auc: 0.876172\tvalid_1's auc: 0.7329\n",
      "[636]\ttraining's auc: 0.876175\tvalid_1's auc: 0.732899\n",
      "[637]\ttraining's auc: 0.876178\tvalid_1's auc: 0.732907\n",
      "[638]\ttraining's auc: 0.876181\tvalid_1's auc: 0.732899\n",
      "[639]\ttraining's auc: 0.876185\tvalid_1's auc: 0.732898\n",
      "[640]\ttraining's auc: 0.87619\tvalid_1's auc: 0.732921\n",
      "[641]\ttraining's auc: 0.876194\tvalid_1's auc: 0.732919\n",
      "[642]\ttraining's auc: 0.876198\tvalid_1's auc: 0.732919\n",
      "[643]\ttraining's auc: 0.876202\tvalid_1's auc: 0.732906\n",
      "[644]\ttraining's auc: 0.876205\tvalid_1's auc: 0.732905\n",
      "[645]\ttraining's auc: 0.876208\tvalid_1's auc: 0.732923\n",
      "[646]\ttraining's auc: 0.87621\tvalid_1's auc: 0.73294\n",
      "[647]\ttraining's auc: 0.876214\tvalid_1's auc: 0.732939\n",
      "[648]\ttraining's auc: 0.876218\tvalid_1's auc: 0.732938\n",
      "[649]\ttraining's auc: 0.876222\tvalid_1's auc: 0.73295\n",
      "[650]\ttraining's auc: 0.876225\tvalid_1's auc: 0.732925\n",
      "[651]\ttraining's auc: 0.876228\tvalid_1's auc: 0.732924\n",
      "[652]\ttraining's auc: 0.876232\tvalid_1's auc: 0.732912\n",
      "[653]\ttraining's auc: 0.876235\tvalid_1's auc: 0.732908\n",
      "[654]\ttraining's auc: 0.876239\tvalid_1's auc: 0.732918\n",
      "[655]\ttraining's auc: 0.876242\tvalid_1's auc: 0.732902\n",
      "[656]\ttraining's auc: 0.876244\tvalid_1's auc: 0.732901\n",
      "[657]\ttraining's auc: 0.876249\tvalid_1's auc: 0.732901\n",
      "[658]\ttraining's auc: 0.876252\tvalid_1's auc: 0.732915\n",
      "[659]\ttraining's auc: 0.876258\tvalid_1's auc: 0.732966\n",
      "[660]\ttraining's auc: 0.87626\tvalid_1's auc: 0.732969\n",
      "[661]\ttraining's auc: 0.876263\tvalid_1's auc: 0.732958\n",
      "[662]\ttraining's auc: 0.87627\tvalid_1's auc: 0.732957\n",
      "[663]\ttraining's auc: 0.876272\tvalid_1's auc: 0.732961\n",
      "[664]\ttraining's auc: 0.876276\tvalid_1's auc: 0.732961\n",
      "[665]\ttraining's auc: 0.876278\tvalid_1's auc: 0.732968\n",
      "[666]\ttraining's auc: 0.876281\tvalid_1's auc: 0.732971\n",
      "[667]\ttraining's auc: 0.876283\tvalid_1's auc: 0.732969\n",
      "[668]\ttraining's auc: 0.876286\tvalid_1's auc: 0.73297\n",
      "[669]\ttraining's auc: 0.876291\tvalid_1's auc: 0.733017\n",
      "[670]\ttraining's auc: 0.876294\tvalid_1's auc: 0.733002\n",
      "[671]\ttraining's auc: 0.8763\tvalid_1's auc: 0.733014\n",
      "[672]\ttraining's auc: 0.876303\tvalid_1's auc: 0.733002\n",
      "[673]\ttraining's auc: 0.876307\tvalid_1's auc: 0.732982\n",
      "[674]\ttraining's auc: 0.876311\tvalid_1's auc: 0.732981\n",
      "[675]\ttraining's auc: 0.876315\tvalid_1's auc: 0.73298\n",
      "[676]\ttraining's auc: 0.876319\tvalid_1's auc: 0.732975\n",
      "[677]\ttraining's auc: 0.876322\tvalid_1's auc: 0.732981\n",
      "[678]\ttraining's auc: 0.876325\tvalid_1's auc: 0.73298\n",
      "[679]\ttraining's auc: 0.876328\tvalid_1's auc: 0.732986\n",
      "[680]\ttraining's auc: 0.87633\tvalid_1's auc: 0.732984\n",
      "[681]\ttraining's auc: 0.876333\tvalid_1's auc: 0.732982\n",
      "[682]\ttraining's auc: 0.87634\tvalid_1's auc: 0.732996\n",
      "[683]\ttraining's auc: 0.876342\tvalid_1's auc: 0.733003\n",
      "[684]\ttraining's auc: 0.876347\tvalid_1's auc: 0.732992\n",
      "[685]\ttraining's auc: 0.876349\tvalid_1's auc: 0.733\n",
      "[686]\ttraining's auc: 0.876354\tvalid_1's auc: 0.73301\n",
      "[687]\ttraining's auc: 0.876357\tvalid_1's auc: 0.733012\n",
      "[688]\ttraining's auc: 0.876363\tvalid_1's auc: 0.733027\n",
      "[689]\ttraining's auc: 0.876366\tvalid_1's auc: 0.733037\n",
      "[690]\ttraining's auc: 0.876369\tvalid_1's auc: 0.733034\n",
      "[691]\ttraining's auc: 0.876373\tvalid_1's auc: 0.73303\n",
      "[692]\ttraining's auc: 0.876376\tvalid_1's auc: 0.733021\n",
      "[693]\ttraining's auc: 0.876382\tvalid_1's auc: 0.733027\n",
      "[694]\ttraining's auc: 0.876386\tvalid_1's auc: 0.733027\n",
      "[695]\ttraining's auc: 0.876389\tvalid_1's auc: 0.733026\n",
      "[696]\ttraining's auc: 0.876392\tvalid_1's auc: 0.733034\n",
      "[697]\ttraining's auc: 0.876394\tvalid_1's auc: 0.733025\n",
      "[698]\ttraining's auc: 0.876403\tvalid_1's auc: 0.733022\n",
      "[699]\ttraining's auc: 0.876407\tvalid_1's auc: 0.73302\n",
      "[700]\ttraining's auc: 0.876407\tvalid_1's auc: 0.73302\n",
      "[701]\ttraining's auc: 0.876411\tvalid_1's auc: 0.733021\n",
      "[702]\ttraining's auc: 0.876415\tvalid_1's auc: 0.73303\n",
      "[703]\ttraining's auc: 0.876418\tvalid_1's auc: 0.733028\n",
      "[704]\ttraining's auc: 0.87642\tvalid_1's auc: 0.733015\n",
      "[705]\ttraining's auc: 0.876423\tvalid_1's auc: 0.73301\n",
      "[706]\ttraining's auc: 0.876426\tvalid_1's auc: 0.733003\n",
      "[707]\ttraining's auc: 0.876429\tvalid_1's auc: 0.732996\n",
      "[708]\ttraining's auc: 0.876432\tvalid_1's auc: 0.733002\n",
      "[709]\ttraining's auc: 0.876435\tvalid_1's auc: 0.732992\n",
      "[710]\ttraining's auc: 0.876438\tvalid_1's auc: 0.732996\n",
      "[711]\ttraining's auc: 0.876442\tvalid_1's auc: 0.732996\n",
      "[712]\ttraining's auc: 0.876445\tvalid_1's auc: 0.733004\n",
      "[713]\ttraining's auc: 0.876448\tvalid_1's auc: 0.733004\n",
      "[714]\ttraining's auc: 0.876451\tvalid_1's auc: 0.733006\n",
      "[715]\ttraining's auc: 0.876453\tvalid_1's auc: 0.733006\n",
      "[716]\ttraining's auc: 0.876459\tvalid_1's auc: 0.732983\n",
      "[717]\ttraining's auc: 0.876469\tvalid_1's auc: 0.733007\n",
      "[718]\ttraining's auc: 0.876472\tvalid_1's auc: 0.733029\n",
      "[719]\ttraining's auc: 0.876475\tvalid_1's auc: 0.733032\n",
      "[720]\ttraining's auc: 0.876478\tvalid_1's auc: 0.733039\n",
      "[721]\ttraining's auc: 0.876482\tvalid_1's auc: 0.733018\n",
      "[722]\ttraining's auc: 0.876485\tvalid_1's auc: 0.73301\n",
      "[723]\ttraining's auc: 0.87649\tvalid_1's auc: 0.733061\n",
      "[724]\ttraining's auc: 0.876492\tvalid_1's auc: 0.733068\n",
      "[725]\ttraining's auc: 0.876495\tvalid_1's auc: 0.733075\n",
      "[726]\ttraining's auc: 0.876499\tvalid_1's auc: 0.733065\n",
      "[727]\ttraining's auc: 0.876502\tvalid_1's auc: 0.733062\n",
      "[728]\ttraining's auc: 0.876506\tvalid_1's auc: 0.73307\n",
      "[729]\ttraining's auc: 0.876509\tvalid_1's auc: 0.73307\n",
      "[730]\ttraining's auc: 0.876512\tvalid_1's auc: 0.733065\n",
      "[731]\ttraining's auc: 0.876517\tvalid_1's auc: 0.733064\n",
      "[732]\ttraining's auc: 0.876518\tvalid_1's auc: 0.733075\n",
      "[733]\ttraining's auc: 0.876523\tvalid_1's auc: 0.733087\n",
      "[734]\ttraining's auc: 0.876525\tvalid_1's auc: 0.733087\n",
      "[735]\ttraining's auc: 0.876528\tvalid_1's auc: 0.733058\n",
      "[736]\ttraining's auc: 0.876532\tvalid_1's auc: 0.73307\n",
      "[737]\ttraining's auc: 0.876535\tvalid_1's auc: 0.733073\n",
      "[738]\ttraining's auc: 0.876539\tvalid_1's auc: 0.73307\n",
      "[739]\ttraining's auc: 0.876542\tvalid_1's auc: 0.733065\n",
      "[740]\ttraining's auc: 0.876546\tvalid_1's auc: 0.733071\n",
      "[741]\ttraining's auc: 0.876549\tvalid_1's auc: 0.733074\n",
      "[742]\ttraining's auc: 0.87655\tvalid_1's auc: 0.733075\n",
      "[743]\ttraining's auc: 0.876553\tvalid_1's auc: 0.733091\n",
      "[744]\ttraining's auc: 0.876555\tvalid_1's auc: 0.733093\n",
      "[745]\ttraining's auc: 0.876558\tvalid_1's auc: 0.733095\n",
      "[746]\ttraining's auc: 0.876561\tvalid_1's auc: 0.733088\n",
      "[747]\ttraining's auc: 0.876563\tvalid_1's auc: 0.733091\n",
      "[748]\ttraining's auc: 0.876566\tvalid_1's auc: 0.733098\n",
      "[749]\ttraining's auc: 0.876568\tvalid_1's auc: 0.7331\n",
      "[750]\ttraining's auc: 0.876571\tvalid_1's auc: 0.73309\n",
      "[751]\ttraining's auc: 0.876573\tvalid_1's auc: 0.733082\n",
      "[752]\ttraining's auc: 0.876576\tvalid_1's auc: 0.733095\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[753]\ttraining's auc: 0.876579\tvalid_1's auc: 0.733098\n",
      "[754]\ttraining's auc: 0.876582\tvalid_1's auc: 0.733094\n",
      "[755]\ttraining's auc: 0.876585\tvalid_1's auc: 0.733089\n",
      "[756]\ttraining's auc: 0.87659\tvalid_1's auc: 0.733094\n",
      "[757]\ttraining's auc: 0.876592\tvalid_1's auc: 0.733092\n",
      "[758]\ttraining's auc: 0.876596\tvalid_1's auc: 0.733085\n",
      "[759]\ttraining's auc: 0.876598\tvalid_1's auc: 0.733083\n",
      "[760]\ttraining's auc: 0.876601\tvalid_1's auc: 0.733071\n",
      "[761]\ttraining's auc: 0.876604\tvalid_1's auc: 0.733082\n",
      "[762]\ttraining's auc: 0.876605\tvalid_1's auc: 0.733077\n",
      "[763]\ttraining's auc: 0.876608\tvalid_1's auc: 0.733066\n",
      "[764]\ttraining's auc: 0.876611\tvalid_1's auc: 0.733066\n",
      "[765]\ttraining's auc: 0.876613\tvalid_1's auc: 0.733075\n",
      "[766]\ttraining's auc: 0.876616\tvalid_1's auc: 0.733071\n",
      "[767]\ttraining's auc: 0.876619\tvalid_1's auc: 0.733063\n",
      "[768]\ttraining's auc: 0.876623\tvalid_1's auc: 0.733064\n",
      "[769]\ttraining's auc: 0.876626\tvalid_1's auc: 0.733055\n",
      "[770]\ttraining's auc: 0.876629\tvalid_1's auc: 0.733052\n",
      "[771]\ttraining's auc: 0.876632\tvalid_1's auc: 0.733063\n",
      "[772]\ttraining's auc: 0.876636\tvalid_1's auc: 0.733063\n",
      "[773]\ttraining's auc: 0.876642\tvalid_1's auc: 0.73307\n",
      "[774]\ttraining's auc: 0.876644\tvalid_1's auc: 0.733078\n",
      "[775]\ttraining's auc: 0.876647\tvalid_1's auc: 0.733085\n",
      "[776]\ttraining's auc: 0.87665\tvalid_1's auc: 0.733093\n",
      "[777]\ttraining's auc: 0.876652\tvalid_1's auc: 0.733093\n",
      "[778]\ttraining's auc: 0.876657\tvalid_1's auc: 0.733092\n",
      "[779]\ttraining's auc: 0.876659\tvalid_1's auc: 0.733085\n",
      "[780]\ttraining's auc: 0.876661\tvalid_1's auc: 0.733079\n",
      "[781]\ttraining's auc: 0.876665\tvalid_1's auc: 0.73308\n",
      "[782]\ttraining's auc: 0.876668\tvalid_1's auc: 0.733125\n",
      "[783]\ttraining's auc: 0.87667\tvalid_1's auc: 0.733126\n",
      "[784]\ttraining's auc: 0.876674\tvalid_1's auc: 0.733122\n",
      "[785]\ttraining's auc: 0.876677\tvalid_1's auc: 0.733119\n",
      "[786]\ttraining's auc: 0.87668\tvalid_1's auc: 0.733119\n",
      "[787]\ttraining's auc: 0.876683\tvalid_1's auc: 0.733103\n",
      "[788]\ttraining's auc: 0.876687\tvalid_1's auc: 0.733086\n",
      "[789]\ttraining's auc: 0.87669\tvalid_1's auc: 0.733076\n",
      "[790]\ttraining's auc: 0.876692\tvalid_1's auc: 0.733064\n",
      "[791]\ttraining's auc: 0.876696\tvalid_1's auc: 0.73307\n",
      "[792]\ttraining's auc: 0.8767\tvalid_1's auc: 0.733066\n",
      "[793]\ttraining's auc: 0.876702\tvalid_1's auc: 0.733074\n",
      "[794]\ttraining's auc: 0.876705\tvalid_1's auc: 0.733079\n",
      "[795]\ttraining's auc: 0.876708\tvalid_1's auc: 0.733077\n",
      "[796]\ttraining's auc: 0.87671\tvalid_1's auc: 0.733082\n",
      "[797]\ttraining's auc: 0.876713\tvalid_1's auc: 0.733091\n",
      "[798]\ttraining's auc: 0.876716\tvalid_1's auc: 0.733097\n",
      "[799]\ttraining's auc: 0.876721\tvalid_1's auc: 0.733069\n",
      "[800]\ttraining's auc: 0.876724\tvalid_1's auc: 0.733094\n",
      "[801]\ttraining's auc: 0.876727\tvalid_1's auc: 0.733108\n",
      "[802]\ttraining's auc: 0.876729\tvalid_1's auc: 0.733107\n",
      "[803]\ttraining's auc: 0.876732\tvalid_1's auc: 0.733104\n",
      "[804]\ttraining's auc: 0.876735\tvalid_1's auc: 0.733104\n",
      "[805]\ttraining's auc: 0.876738\tvalid_1's auc: 0.733099\n",
      "[806]\ttraining's auc: 0.876741\tvalid_1's auc: 0.733093\n",
      "[807]\ttraining's auc: 0.876747\tvalid_1's auc: 0.733112\n",
      "[808]\ttraining's auc: 0.87675\tvalid_1's auc: 0.733098\n",
      "[809]\ttraining's auc: 0.876753\tvalid_1's auc: 0.733091\n",
      "[810]\ttraining's auc: 0.876756\tvalid_1's auc: 0.733093\n",
      "[811]\ttraining's auc: 0.876759\tvalid_1's auc: 0.733085\n",
      "[812]\ttraining's auc: 0.876761\tvalid_1's auc: 0.733082\n",
      "[813]\ttraining's auc: 0.876764\tvalid_1's auc: 0.733087\n",
      "[814]\ttraining's auc: 0.876767\tvalid_1's auc: 0.733086\n",
      "[815]\ttraining's auc: 0.87677\tvalid_1's auc: 0.733087\n",
      "[816]\ttraining's auc: 0.876772\tvalid_1's auc: 0.733081\n",
      "[817]\ttraining's auc: 0.876776\tvalid_1's auc: 0.733117\n",
      "[818]\ttraining's auc: 0.876779\tvalid_1's auc: 0.73312\n",
      "[819]\ttraining's auc: 0.876781\tvalid_1's auc: 0.733131\n",
      "[820]\ttraining's auc: 0.876784\tvalid_1's auc: 0.733138\n",
      "[821]\ttraining's auc: 0.876788\tvalid_1's auc: 0.733132\n",
      "[822]\ttraining's auc: 0.876789\tvalid_1's auc: 0.733128\n",
      "[823]\ttraining's auc: 0.876794\tvalid_1's auc: 0.73312\n",
      "[824]\ttraining's auc: 0.876797\tvalid_1's auc: 0.733112\n",
      "[825]\ttraining's auc: 0.8768\tvalid_1's auc: 0.733096\n",
      "[826]\ttraining's auc: 0.876803\tvalid_1's auc: 0.7331\n",
      "[827]\ttraining's auc: 0.876806\tvalid_1's auc: 0.73311\n",
      "[828]\ttraining's auc: 0.87681\tvalid_1's auc: 0.733101\n",
      "[829]\ttraining's auc: 0.876813\tvalid_1's auc: 0.733103\n",
      "[830]\ttraining's auc: 0.876816\tvalid_1's auc: 0.73311\n",
      "[831]\ttraining's auc: 0.876819\tvalid_1's auc: 0.733109\n",
      "[832]\ttraining's auc: 0.876822\tvalid_1's auc: 0.733099\n",
      "[833]\ttraining's auc: 0.876825\tvalid_1's auc: 0.733093\n",
      "[834]\ttraining's auc: 0.876829\tvalid_1's auc: 0.733098\n",
      "[835]\ttraining's auc: 0.876832\tvalid_1's auc: 0.733104\n",
      "[836]\ttraining's auc: 0.876834\tvalid_1's auc: 0.733106\n",
      "[837]\ttraining's auc: 0.876837\tvalid_1's auc: 0.733104\n",
      "[838]\ttraining's auc: 0.87684\tvalid_1's auc: 0.733109\n",
      "[839]\ttraining's auc: 0.876843\tvalid_1's auc: 0.733107\n",
      "[840]\ttraining's auc: 0.876846\tvalid_1's auc: 0.733104\n",
      "[841]\ttraining's auc: 0.876849\tvalid_1's auc: 0.73311\n",
      "[842]\ttraining's auc: 0.876853\tvalid_1's auc: 0.73312\n",
      "[843]\ttraining's auc: 0.876858\tvalid_1's auc: 0.733126\n",
      "[844]\ttraining's auc: 0.876861\tvalid_1's auc: 0.733099\n",
      "[845]\ttraining's auc: 0.876864\tvalid_1's auc: 0.733114\n",
      "[846]\ttraining's auc: 0.876867\tvalid_1's auc: 0.733112\n",
      "[847]\ttraining's auc: 0.876871\tvalid_1's auc: 0.733128\n",
      "[848]\ttraining's auc: 0.876875\tvalid_1's auc: 0.733134\n",
      "[849]\ttraining's auc: 0.876878\tvalid_1's auc: 0.733141\n",
      "[850]\ttraining's auc: 0.87688\tvalid_1's auc: 0.733136\n",
      "[851]\ttraining's auc: 0.876883\tvalid_1's auc: 0.733135\n",
      "[852]\ttraining's auc: 0.876886\tvalid_1's auc: 0.733141\n",
      "[853]\ttraining's auc: 0.876889\tvalid_1's auc: 0.733138\n",
      "[854]\ttraining's auc: 0.876892\tvalid_1's auc: 0.733142\n",
      "[855]\ttraining's auc: 0.876895\tvalid_1's auc: 0.733149\n",
      "[856]\ttraining's auc: 0.876898\tvalid_1's auc: 0.733147\n",
      "[857]\ttraining's auc: 0.876901\tvalid_1's auc: 0.733148\n",
      "[858]\ttraining's auc: 0.876903\tvalid_1's auc: 0.733141\n",
      "[859]\ttraining's auc: 0.876909\tvalid_1's auc: 0.733152\n",
      "[860]\ttraining's auc: 0.876912\tvalid_1's auc: 0.733146\n",
      "[861]\ttraining's auc: 0.876917\tvalid_1's auc: 0.733148\n",
      "[862]\ttraining's auc: 0.876921\tvalid_1's auc: 0.73314\n",
      "[863]\ttraining's auc: 0.876924\tvalid_1's auc: 0.733144\n",
      "[864]\ttraining's auc: 0.87693\tvalid_1's auc: 0.733169\n",
      "[865]\ttraining's auc: 0.876933\tvalid_1's auc: 0.733159\n",
      "[866]\ttraining's auc: 0.876935\tvalid_1's auc: 0.73315\n",
      "[867]\ttraining's auc: 0.876938\tvalid_1's auc: 0.733158\n",
      "[868]\ttraining's auc: 0.876941\tvalid_1's auc: 0.733153\n",
      "[869]\ttraining's auc: 0.876944\tvalid_1's auc: 0.733134\n",
      "[870]\ttraining's auc: 0.876946\tvalid_1's auc: 0.733134\n",
      "[871]\ttraining's auc: 0.876949\tvalid_1's auc: 0.733133\n",
      "[872]\ttraining's auc: 0.876953\tvalid_1's auc: 0.733137\n",
      "[873]\ttraining's auc: 0.876956\tvalid_1's auc: 0.733139\n",
      "[874]\ttraining's auc: 0.876958\tvalid_1's auc: 0.733144\n",
      "[875]\ttraining's auc: 0.876961\tvalid_1's auc: 0.733137\n",
      "[876]\ttraining's auc: 0.876965\tvalid_1's auc: 0.733138\n",
      "[877]\ttraining's auc: 0.876968\tvalid_1's auc: 0.733142\n",
      "[878]\ttraining's auc: 0.87697\tvalid_1's auc: 0.733141\n",
      "[879]\ttraining's auc: 0.876973\tvalid_1's auc: 0.733144\n",
      "[880]\ttraining's auc: 0.876977\tvalid_1's auc: 0.733133\n",
      "[881]\ttraining's auc: 0.876982\tvalid_1's auc: 0.733129\n",
      "[882]\ttraining's auc: 0.876984\tvalid_1's auc: 0.733125\n",
      "[883]\ttraining's auc: 0.876988\tvalid_1's auc: 0.733129\n",
      "[884]\ttraining's auc: 0.876991\tvalid_1's auc: 0.733135\n",
      "[885]\ttraining's auc: 0.876995\tvalid_1's auc: 0.73315\n",
      "[886]\ttraining's auc: 0.876999\tvalid_1's auc: 0.733153\n",
      "[887]\ttraining's auc: 0.877001\tvalid_1's auc: 0.733147\n",
      "[888]\ttraining's auc: 0.877003\tvalid_1's auc: 0.733139\n",
      "[889]\ttraining's auc: 0.877005\tvalid_1's auc: 0.733151\n",
      "[890]\ttraining's auc: 0.877008\tvalid_1's auc: 0.733176\n",
      "[891]\ttraining's auc: 0.877011\tvalid_1's auc: 0.733167\n",
      "[892]\ttraining's auc: 0.877014\tvalid_1's auc: 0.73317\n",
      "[893]\ttraining's auc: 0.877016\tvalid_1's auc: 0.733198\n",
      "[894]\ttraining's auc: 0.87702\tvalid_1's auc: 0.733188\n",
      "[895]\ttraining's auc: 0.877024\tvalid_1's auc: 0.733183\n",
      "[896]\ttraining's auc: 0.877027\tvalid_1's auc: 0.733182\n",
      "[897]\ttraining's auc: 0.87703\tvalid_1's auc: 0.733178\n",
      "[898]\ttraining's auc: 0.877034\tvalid_1's auc: 0.733169\n",
      "[899]\ttraining's auc: 0.877036\tvalid_1's auc: 0.733169\n",
      "[900]\ttraining's auc: 0.877042\tvalid_1's auc: 0.733177\n",
      "[901]\ttraining's auc: 0.877043\tvalid_1's auc: 0.733161\n",
      "[902]\ttraining's auc: 0.877048\tvalid_1's auc: 0.733154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[903]\ttraining's auc: 0.87705\tvalid_1's auc: 0.733149\n",
      "[904]\ttraining's auc: 0.877051\tvalid_1's auc: 0.73314\n",
      "[905]\ttraining's auc: 0.877055\tvalid_1's auc: 0.733143\n",
      "[906]\ttraining's auc: 0.877059\tvalid_1's auc: 0.733145\n",
      "[907]\ttraining's auc: 0.877062\tvalid_1's auc: 0.733164\n",
      "[908]\ttraining's auc: 0.877065\tvalid_1's auc: 0.733163\n",
      "[909]\ttraining's auc: 0.877068\tvalid_1's auc: 0.733175\n",
      "[910]\ttraining's auc: 0.877072\tvalid_1's auc: 0.733174\n",
      "[911]\ttraining's auc: 0.877075\tvalid_1's auc: 0.73317\n",
      "[912]\ttraining's auc: 0.877078\tvalid_1's auc: 0.733173\n",
      "[913]\ttraining's auc: 0.877081\tvalid_1's auc: 0.733177\n",
      "[914]\ttraining's auc: 0.877084\tvalid_1's auc: 0.73318\n",
      "[915]\ttraining's auc: 0.877086\tvalid_1's auc: 0.733177\n",
      "[916]\ttraining's auc: 0.877091\tvalid_1's auc: 0.733156\n",
      "[917]\ttraining's auc: 0.877094\tvalid_1's auc: 0.733161\n",
      "[918]\ttraining's auc: 0.877098\tvalid_1's auc: 0.733137\n",
      "[919]\ttraining's auc: 0.877099\tvalid_1's auc: 0.733135\n",
      "[920]\ttraining's auc: 0.877104\tvalid_1's auc: 0.733133\n",
      "[921]\ttraining's auc: 0.877108\tvalid_1's auc: 0.733149\n",
      "[922]\ttraining's auc: 0.877112\tvalid_1's auc: 0.733141\n",
      "[923]\ttraining's auc: 0.877115\tvalid_1's auc: 0.733137\n",
      "[924]\ttraining's auc: 0.877118\tvalid_1's auc: 0.733139\n",
      "[925]\ttraining's auc: 0.87712\tvalid_1's auc: 0.73314\n",
      "[926]\ttraining's auc: 0.877124\tvalid_1's auc: 0.733145\n",
      "[927]\ttraining's auc: 0.877127\tvalid_1's auc: 0.733154\n",
      "[928]\ttraining's auc: 0.877129\tvalid_1's auc: 0.733156\n",
      "[929]\ttraining's auc: 0.877133\tvalid_1's auc: 0.733155\n",
      "[930]\ttraining's auc: 0.877139\tvalid_1's auc: 0.733149\n",
      "[931]\ttraining's auc: 0.877144\tvalid_1's auc: 0.733119\n",
      "[932]\ttraining's auc: 0.877153\tvalid_1's auc: 0.733306\n",
      "[933]\ttraining's auc: 0.877157\tvalid_1's auc: 0.733311\n",
      "[934]\ttraining's auc: 0.877159\tvalid_1's auc: 0.733305\n",
      "[935]\ttraining's auc: 0.877163\tvalid_1's auc: 0.733313\n",
      "[936]\ttraining's auc: 0.877166\tvalid_1's auc: 0.733314\n",
      "[937]\ttraining's auc: 0.877168\tvalid_1's auc: 0.733318\n",
      "[938]\ttraining's auc: 0.877171\tvalid_1's auc: 0.733312\n",
      "[939]\ttraining's auc: 0.877174\tvalid_1's auc: 0.733309\n",
      "[940]\ttraining's auc: 0.877177\tvalid_1's auc: 0.733323\n",
      "[941]\ttraining's auc: 0.877181\tvalid_1's auc: 0.733328\n",
      "[942]\ttraining's auc: 0.877184\tvalid_1's auc: 0.733332\n",
      "[943]\ttraining's auc: 0.87719\tvalid_1's auc: 0.733329\n",
      "[944]\ttraining's auc: 0.877193\tvalid_1's auc: 0.733345\n",
      "[945]\ttraining's auc: 0.877196\tvalid_1's auc: 0.733347\n",
      "[946]\ttraining's auc: 0.877198\tvalid_1's auc: 0.733351\n",
      "[947]\ttraining's auc: 0.877201\tvalid_1's auc: 0.733345\n",
      "[948]\ttraining's auc: 0.877203\tvalid_1's auc: 0.733343\n",
      "[949]\ttraining's auc: 0.877207\tvalid_1's auc: 0.733325\n",
      "[950]\ttraining's auc: 0.877211\tvalid_1's auc: 0.733299\n",
      "[951]\ttraining's auc: 0.877213\tvalid_1's auc: 0.7333\n",
      "[952]\ttraining's auc: 0.877218\tvalid_1's auc: 0.7333\n",
      "[953]\ttraining's auc: 0.877222\tvalid_1's auc: 0.7333\n",
      "[954]\ttraining's auc: 0.877225\tvalid_1's auc: 0.733306\n",
      "[955]\ttraining's auc: 0.877228\tvalid_1's auc: 0.733304\n",
      "[956]\ttraining's auc: 0.877232\tvalid_1's auc: 0.733309\n",
      "[957]\ttraining's auc: 0.877236\tvalid_1's auc: 0.733307\n",
      "[958]\ttraining's auc: 0.877239\tvalid_1's auc: 0.733296\n",
      "[959]\ttraining's auc: 0.877242\tvalid_1's auc: 0.733292\n",
      "[960]\ttraining's auc: 0.877244\tvalid_1's auc: 0.73331\n",
      "[961]\ttraining's auc: 0.877247\tvalid_1's auc: 0.733316\n",
      "[962]\ttraining's auc: 0.877252\tvalid_1's auc: 0.733315\n",
      "[963]\ttraining's auc: 0.877255\tvalid_1's auc: 0.733315\n",
      "[964]\ttraining's auc: 0.877258\tvalid_1's auc: 0.733292\n",
      "[965]\ttraining's auc: 0.87726\tvalid_1's auc: 0.733289\n",
      "[966]\ttraining's auc: 0.877264\tvalid_1's auc: 0.73327\n",
      "[967]\ttraining's auc: 0.877267\tvalid_1's auc: 0.733267\n",
      "[968]\ttraining's auc: 0.877269\tvalid_1's auc: 0.73327\n",
      "[969]\ttraining's auc: 0.877273\tvalid_1's auc: 0.733263\n",
      "[970]\ttraining's auc: 0.877277\tvalid_1's auc: 0.733279\n",
      "[971]\ttraining's auc: 0.877279\tvalid_1's auc: 0.733276\n",
      "[972]\ttraining's auc: 0.877282\tvalid_1's auc: 0.733261\n",
      "[973]\ttraining's auc: 0.877285\tvalid_1's auc: 0.733259\n",
      "[974]\ttraining's auc: 0.877287\tvalid_1's auc: 0.733258\n",
      "[975]\ttraining's auc: 0.87729\tvalid_1's auc: 0.733259\n",
      "[976]\ttraining's auc: 0.877293\tvalid_1's auc: 0.733266\n",
      "[977]\ttraining's auc: 0.877296\tvalid_1's auc: 0.733274\n",
      "[978]\ttraining's auc: 0.877297\tvalid_1's auc: 0.733276\n",
      "[979]\ttraining's auc: 0.8773\tvalid_1's auc: 0.733279\n",
      "[980]\ttraining's auc: 0.877302\tvalid_1's auc: 0.73328\n",
      "[981]\ttraining's auc: 0.877306\tvalid_1's auc: 0.733288\n",
      "[982]\ttraining's auc: 0.877309\tvalid_1's auc: 0.733281\n",
      "[983]\ttraining's auc: 0.877314\tvalid_1's auc: 0.73328\n",
      "[984]\ttraining's auc: 0.877317\tvalid_1's auc: 0.733269\n",
      "[985]\ttraining's auc: 0.877321\tvalid_1's auc: 0.733272\n",
      "[986]\ttraining's auc: 0.877323\tvalid_1's auc: 0.733274\n",
      "[987]\ttraining's auc: 0.877326\tvalid_1's auc: 0.733265\n",
      "[988]\ttraining's auc: 0.877329\tvalid_1's auc: 0.733271\n",
      "[989]\ttraining's auc: 0.877332\tvalid_1's auc: 0.733278\n",
      "[990]\ttraining's auc: 0.877337\tvalid_1's auc: 0.733247\n",
      "[991]\ttraining's auc: 0.87734\tvalid_1's auc: 0.73325\n",
      "[992]\ttraining's auc: 0.877343\tvalid_1's auc: 0.733245\n",
      "[993]\ttraining's auc: 0.877345\tvalid_1's auc: 0.733236\n",
      "[994]\ttraining's auc: 0.877348\tvalid_1's auc: 0.733246\n",
      "[995]\ttraining's auc: 0.87735\tvalid_1's auc: 0.733259\n",
      "[996]\ttraining's auc: 0.877354\tvalid_1's auc: 0.733266\n",
      "[997]\ttraining's auc: 0.877356\tvalid_1's auc: 0.733272\n",
      "[998]\ttraining's auc: 0.877358\tvalid_1's auc: 0.733267\n",
      "[999]\ttraining's auc: 0.877361\tvalid_1's auc: 0.733272\n",
      "[1000]\ttraining's auc: 0.877364\tvalid_1's auc: 0.733275\n"
     ]
    }
   ],
   "source": [
    "import model_lgbm as mlgbm\n",
    "_, fi, model = mlgbm.crossvalidate_model_by_index(train, train_index, validation_index, cols, params, iterations=1000, validate_on_both=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('context_type', 1658),\n",
       " ('song_listened', 1449),\n",
       " ('context_type_cumulative_mean', 1346),\n",
       " ('user_id', 1221),\n",
       " ('ts_listen_hour_cumulative_mean', 1198),\n",
       " ('ts_listen_dayofweek_cumulative_mean', 1161),\n",
       " ('media_duration', 1159),\n",
       " ('platform_name_cumulative_mean', 1001),\n",
       " ('listen_type_cumulative_mean', 884),\n",
       " ('ExplicitLyrics_cumulative_mean', 832),\n",
       " ('release_date_decade_cumulative_mean', 734),\n",
       " ('genre_id_cumulative_mean', 728),\n",
       " ('release_date_year_cumulative_mean', 701),\n",
       " ('album_id_cumulative_mean', 698),\n",
       " ('Isrc_Country_cumulative_mean', 675),\n",
       " ('cumsum_is_listened_1', 658),\n",
       " ('bpm_min', 651),\n",
       " ('artist_id_cumulative_mean', 631),\n",
       " ('media_duration_std', 630),\n",
       " ('media_duration_min', 615),\n",
       " ('media_duration_max', 603),\n",
       " ('rank_std', 595),\n",
       " ('Isrc_Label_cumulative_mean', 584),\n",
       " ('gain_min', 577),\n",
       " ('bpm_std', 570),\n",
       " ('gain_std', 558),\n",
       " ('gain_max', 547),\n",
       " ('rank_min', 544),\n",
       " ('rank_mean', 532),\n",
       " ('bpm_max', 527),\n",
       " ('rank_max', 526),\n",
       " ('bpm_mean', 518),\n",
       " ('gain_mean', 502),\n",
       " ('media_duration_mean', 502),\n",
       " ('cumsum_is_listened', 484),\n",
       " ('user_age', 478),\n",
       " ('cumsum_listen_type_1', 461),\n",
       " ('release_date_listened_this_year', 414),\n",
       " ('was_is_listened_before', 367),\n",
       " ('cumsum_listen_type', 252),\n",
       " ('country_percent', 206),\n",
       " ('percent_listened', 202),\n",
       " ('release_date_listened_this_decade', 194),\n",
       " ('explicit_lyrics_percent', 165),\n",
       " ('genre_listened', 150),\n",
       " ('artist_listened', 129),\n",
       " ('album_listened', 105),\n",
       " ('user_gender', 78)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlgbm.feature_score(fi, cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "import numpy as np\n",
    "\n",
    "y_test = np.array(train.loc[validation_index]['is_listened'])\n",
    "x_test = train.loc[validation_index][cols]\n",
    "\n",
    "y_predicted = np.array(model.predict(x_test))\n",
    "\n",
    "# Compute ROC curve and ROC area for each class\n",
    "fpr, tpr, _ = roc_curve(y_test, y_predicted)\n",
    "roc_auc = auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4VNXWwOHfSk8gEEIAEaRjCFIlIogCghQBEUUFVGzc\nD5GiggIi2DsiClLlClwvV1FRBKWDVAu9Sa9CIr2EhBSSzP7+OEMygSQEyORkkvU+T56cftaZTGbN\n2fvsvcUYg1JKKZUVL7sDUEoplb9polBKKZUtTRRKKaWypYlCKaVUtjRRKKWUypYmCqWUUtnSRKGu\nmog8JiIL7Y7DbiJSQUTiRMQ7D89ZSUSMiPjk1TndSUS2iUjza9hP34N5SLQdhWcTkYNAGSAViAPm\nA32NMXF2xlUQOV/rfxljFtsYQyXgAOBrjEmxKw5nLAaobozZ6+bzVCKfXHNhpXcUBcN9xpiiQD2g\nPjDE5niuiZ3fkgvKN/Sroa+3yilNFAWIMeYosAArYQAgIv4iMkJEDonIMRGZICKBLuvvF5FNInJO\nRPaJSFvn8uIi8qWIHBGRaBF592IRi4g8JSKrnNPjRWSEaxwiMktEBjinbxSRH0TkhIgcEJHnXbZ7\nU0RmiMg0ETkHPHXpNTnj+Mq5/98iMkxEvFzi+E1ExohIjIjsFJGWl+yb3TX8JiKfisgp4E0RqSoi\nv4rIKRE5KSL/E5EQ5/b/BSoAPzuLmwZdWgwkIstE5B3ncWNFZKGIhLnE84TzGk6JyGsiclBE7sns\nbykigSLyiXP7GBFZ5fp3Ax5z/k1PishQl/0aisgfInLWed1jRMTPZb0RkT4isgfY41w2SkQOO98D\n60XkLpftvUXkVed7I9a5/iYRWeHcZLPz9eji3L6D8/10VkR+F5E6Lsc6KCKDRWQLcF5EfFxfA2fs\n65xxHBORkc5dL57rrPNcjV3fg859bxGRRSJy2rnvq5m9ruoaGWP0x4N/gIPAPc7p8sBWYJTL+k+B\n2UAoEAz8DHzgXNcQiAFaYX1pKAfUcK6bCUwEigClgTXAs851TwGrnNNNgcOkF2OWABKAG53HXA+8\nDvgBVYD9QBvntm8CyUAn57aBmVzfV8AsZ+yVgN1AD5c4UoD+gC/QxXk9oTm8hhSgH+ADBALVnK+F\nP1AK6wPqs8xea+d8JcAAPs75ZcA+4Gbn8ZYBHzrX1cQqGrzT+VqMcF77PVn8Xcc69y8HeAN3OOO6\neM5JznPUBZKACOd+DYBGzmuqBOwAXnQ5rgEWYb0fAp3LHgdKOvd5CTgKBDjXDcR6T4UD4jxfSZdj\nVXM5dn3gOHC7M+Ynna+Zv8vrtwm4yeXcaa8p8AfQ3TldFGiU2eucyXswGDjijD3AOX+73f+bBenH\n9gD05zr/gNY/WhwQ6/xnWgKEONcJcB6o6rJ9Y+CAc3oi8Gkmxyzj/PAJdFnWDVjqnHb9JxXgENDU\nOf9/wK/O6duBQ5ccewgwxTn9JrAim2vzBi4ANV2WPQssc4njH5xJyrlsDdA9h9dwKKtzO7fpBGy8\n5LW+UqIY5rK+NzDfOf068I3LuiDntV2WKLCSZgJQN5N1F89Z/pJr7prFNbwIzHSZN0CLK1z3mYvn\nBnYB92ex3aWJYjzwziXb7AKaubx+z2Ty/r2YKFYAbwFhWVxzVomim+vfSX9y/0fLCQuGTsaYxSLS\nDPgaCAPOYn0rDgLWi8jFbQXrAxisb3ZzMzleRaxv6Edc9vPCunPIwBhjRGQ61j/rCuBRYJrLcW4U\nkbMuu3gDK13mLzumizBnHH+7LPsb61v2RdHG+Wnhsv7GHF5DhnOLSBlgFHAX1rdSL6wPzatx1GU6\nHuubMc6Y0s5njIl3FnllJgzrm/G+qz2PiNwMjAQisf72Plh3da4uve6XgR7OGA1QzBkDWO+R7OJw\nVRF4UkT6uSzzcx4303NfogfwNrBTRA4AbxljfsnBea8mRnUNtI6iADHGLAemYhVrAJzE+mZ6izEm\nxPlT3FgV32D901bN5FCHsb6Nh7nsV8wYc0sWp/4GeEhEKmLdRfzgcpwDLscIMcYEG2PauYadzSWd\nxCqeqeiyrAIQ7TJfTlwygXP9Pzm8hkvP/b5zWW1jTDGsIhnJZvurcQSraBCw6iCwinsycxJIJPO/\nzZWMB3ZiPY1UDHiVjNcALtfhrI8YBDwClDDGhGAV313cJ6v3SGYOA+9d8vcOMsZ8k9m5L2WM2WOM\n6YZVTPgRMENEimS3j8t5q+QwRnUNNFEUPJ8BrUSkrjHGgVWW/amIlAYQkXIi0sa57ZfA0yLSUkS8\nnOtqGGOOAAuBT0SkmHNdVecdy2WMMRuxPtz+DSwwxly8g1gDxDorMAOdFaO1ROS2nFyIMSYV+A54\nT0SCnYloAOl3LGB9qDwvIr4i8jAQAcy92mtwCsYqxosRkXJY5fOujnHtH0gzgPtE5A5n5fKbXP4B\nDoDz7zYZGCnWwwDezgpc/xycJxg4B8SJSA3guRxsnwKcAHxE5HWsO4qL/g28IyLVxVJHRC4muEtf\nj0lALxG53bltERFpLyLBOYgbEXlcREo5r//ie8jhjM1B1q/9L0BZEXlRrIc3gkXk9pycU+WMJooC\nxhhzAqsC+HXnosHAXuBPsZ4sWoxVMYkxZg3wNFaFdwywnPRv709gFRtsxyp+mQGUzebUXwP3OH9f\njCUV6ID1FNYB0pNJ8au4pH5Y9Sz7gVXO4092Wb8aqO489nvAQ8aYi0U6V3sNbwG3Yr0Wc4AfL1n/\nATDM+UTPy1dxDRhjtjmvZTrW3UUcVsVvUha7vIxVibwWOI31DTsn/68vYxX/xWJ9cH97he0XYLW9\n2Y1VbJdIxuKhkVjJeiFWAvoSqxIdrGT3H+fr8YgxZh1WHdUYrNd7L5k8yZaNtsA2EYnDKgLsaoxJ\nMMbEY/1tf3Oeq5HrTsaYWKyHEO7DKpLbA9x9FedVV6AN7pTHEpGnsBrA3Wl3LFdLRIpifWuubow5\nYHc8SmVH7yiUyiMicp+IBDnL3Udg3TEctDcqpa5ME4VSeed+rIr2f7CKy7oavaVXHkCLnpRSSmVL\n7yiUUkply+Ma3IWFhZlKlSrZHYZSSnmU9evXnzTGlLqWfT0uUVSqVIl169bZHYZSSnkUEfn7yltl\nTouelFJKZUsThVJKqWxpolBKKZUtTRRKKaWypYlCKaVUtjRRKKWUypbbEoWITBaR4yLyVxbrRURG\ni8heEdkiIre6KxallFLXzp3tKKZidTf8VRbr78Xq76Y61mA3452/lVJKXQ9jIPEMJJ2BxNNciL3a\ngRozcluiMMasEJFK2WxyP/CVs1O0P0UkRETKOgecUUopdSlj4EIsnNkFJ7aAIwWOrQcvbzi5DRzJ\ncOTPDLsM/LkVG//JbhiWK7OzZXY5Mg6QEuVcdlmiEJGeQE+AChUq5ElwSillm+Tz8Pdi2P5fOLsH\nks5BwkkwKZCSeFWHqlXNl9G/VbzyhtnwiC48jDFfAF8AREZGane3SinPlxwPJzbDgXnW75gDEBcN\n4mUlhax4+4FPEKTEQ+n6ULYxJMdBqXoQVIrtpyqxYWcKjz9zBwBPGEOzYTFUrvzONYdqZ6KIBm5y\nmS/vXKaUUgWHcUDiWSsZ7P0JDs6DM3uuvJ+XLwSEQvFKUPc5CKsNwRUgKCzTzePjk3n33RV8/PEC\nvL2FRk1rUK1aKCJCpUoh13UJdiaK2UBfEZmOVYkdo/UTSimPF3PQSggH5kLsYTi9M/vtK7SAIjdC\nyVugRHXrJ7SGdeeQQ/Pm7aFPn7kcOHAWgB49GlCyZOAV9so5tyUKEfkGaA6EiUgU8AbgC2CMmQDM\nBdphDcAeDzztrliUUsotkhNgzwxYNwLOH4X441lvGxoBN9wGJW6Gap0gNBy8ru8jODr6HC++uIAZ\nM7YDUKdOGSZMaE/jxjddYc+r486nnrpdYb0B+rjr/EoplWtSk+HUNtg8AY5vtH4cydnvU/tfULk9\n3NgYipRxS1h9+sxl1qxdBAX58vbbzXnhhUb4+OR+8ziPqMxWSqk8k3wezu6DqJUQtQwO/QqJp7PZ\nQaBUHSh9KzToDyUjrvtOITspKY60ZPDRR/fg6+vNJ5+0pkKF4m47pyYKpVThlJIEcVFWJfOJLXDu\nIGz7T/b7+ATBzQ9BnZ4QVAaCy4NPQJ6EGxOTyLBhv7J792nmz38MESE8PIzvv3/Y7efWRKGUKvgc\nqVYy+Oc32P+LlRzOH73yfuWbQdWOULsH+LvvG3t2jDF8//12XnxxPkeOxOHtLWzadJT69a+vEd3V\n0EShlCp4Yg5a9QgH51uPoh5emvW2xSpaTxjddDeEd7HaJfjm3hND12PfvtP07TuP+fP3AtC4cXkm\nTOhAnTruqfPIiiYKpZTnizkIq16FU9utJ4/OZ/GkvX8IhNWCm5pDnV4QXC4vo7wqI0b8zmuvLSUx\nMYWQkAA++uge/vWvW/HykjyPRROFUsrzGAdsnwZ/vGm1aM6Mlw8UrwwVWkKDlyCkitXq2UPExyeT\nmJhC9+51GDGiNaVLF7EtFk0USqn8LynG6uxu3y8QvdKqY7hUiXAo3xSqP2DVLfgG5X2c1+HEifPs\n2nWKO++0+rMbPLgJzZtXomnT6+unKTdoolBK5S/GYT2S+tcUOLYWEk5D4qnMt72xCVR/EOr3vaqW\nzPmJw2GYPHkjgwYtwsfHi507+xIaGoi/v0++SBKgiUIplR8knIK9s2DrF3BkddbblaprtXCu9xzc\ncDv4+OddjG7w11/H6dXrF377zepIu1WrKsTHJxMamj8q0y/SRKGUsodxwKphsGksXDh3+foa3SCw\nFFRqDTe1yDdPIuWG8+cv8Pbbyxk58k9SUhyUKVOEzz5rS5cutyCS95XVV6KJQimVd07vgj0zYdPn\nEPfP5esb9IdKba0KaC/vvI8vjzz00PfMn78XEejdO5L33mtJSEjeNNy7FpoolFLuY4zVnmHbf2Dj\n6MvX+xa1Wjo3HAKhN+d9fDYZPLgJx47FMX58e26/vbzd4VyRJgqlVO5KOGUVJ235whqI51IBJeDO\n9627hpBqkA+LWnJTSoqDzz9fzcGDZxk16l4AmjevxLp1PW1pE3EtNFEopXLHlkmw/SuIXnX5Oi8f\n664h8mXwL5b3sdlkzZponn32FzZtsroL6dmzAbfcUhrAY5IEaKJQSl2rC7FwdK2VIE5usVpFX+Rf\nHCq3g1rPWHcOBfyu4VJnzyby6qtLmDBhHcZAxYrFGTOmXVqS8DSaKJRSOedIhZ1fw7wnMl9ftBx0\nXWm1iC6kpk//ixdfnM+xY+fx8fHipZca89prTSlSxDPbeYAmCqVUTqQmw2/DYO3wy9eVbQw33gFN\n3va41tDusHDhPo4dO0+TJjcxfnx7atfO2w783EEThVIqa+f+hl+6wLH14EhJX175Xmg5DopXsi20\n/CIpKYXo6FiqVCkBwPDhrbjrrgo8+WQ9j6qHyI4mCqVUOuOAo+tgzw+Z3z1UbA33fWfb2Az5za+/\nHuC55+bg5SVs3twLPz9vwsKCePrp+naHlqs0USilrOE/t0yCZf0vX+cTAJGD4I43PKr3VXc6diyO\nl19exLRpWwCoUSOMqKhzaXcVBY0mCqUKo9RkOL4Bdn0PZ/fAvtkZ15dvBiVuhlpPw42N7YkxH3I4\nDJMmreeVV5Zw9mwiAQE+DBt2FwMHNsHPr+C2JNdEoVRhEHMA1n1idbh3bF3W293xFtw22OM723OX\nBx74ltmzdwHQpk1Vxo5tR9WqoTZH5X6aKJQqiBypVlJY/wmc3WuNF52Z4pWhcnuo369QdaFxrR58\nsAZr1kQzalRbHn64Zr7swM8dNFEoVVAknoE/34H9c+HMrsvX+wRCnWehSnsoEwkBIXkfo4eZPXsX\nUVHn6N37NgCeeKIuDz4YQXBw4brj0kShlKeK+wcOzIed38ChxZlvU6wS3HQ33D4ESlTP0/A82aFD\nMTz//DxmzdqFv783bdtWo0qVEohIoUsSoIlCKc9hjDUE6IZRsOtbSEm4fJuwWs6BffpYw4IWkqKR\n3JKcnMro0at5441lnD+fTHCwH+++24KKFQv348CaKJTyBNunwbzuly/3Kwbhj0BQGWs40CI35H1s\nBcSff0bx7LO/sGXLMQAefrgmn37ahnLlCk8nhlnRRKFUfnX+qHX3sObDjMu9fCHicWj6IQR5Zidz\n+dFrry1ly5ZjVK4cwpgx7WjXTovqLtJEoVR+cnwz7J0Jf7x1+brSt0KnWRCc/we68QTGGGJjL1Cs\nmFXnMGbMvXz11WaGDm1KUJCvzdHlL5oolLKbMbB5Aizpnfn6W56ElmPBt0jexlWA7dp1kt695yIC\nixZ1R0QIDw/jvfda2h1avqSJQim7xB+H2Z0vH+gnrBbU7gk3d4aiN9oTWwGVmJjCBx+s5MMPf+PC\nhVRKlgzk4MGzVK5cMLveyC2aKJTKa1Gr4PfX4PCyjMtDI+DxtXrn4CaLFu2jd++57N17GoBnnqnH\n8OGtKFlSu0a/ErcmChFpC4wCvIF/G2M+vGR9BeA/QIhzm1eMMXPdGZNStog/CSsHw1+TMy4Xb6j+\nILT/BrwKbl9BdjLG0KPHbKZM2QRAzZqlmDChPXfdVdHmyDyH2xKFiHgDY4FWQBSwVkRmG2Ncxktk\nGPCdMWa8iNQE5gKV3BWTUnnu2AZYOQT+XphxedFycM8EqNrBnrgKERGhUqUQAgN9eP31ZgwY0LhA\nd+DnDu68o2gI7DXG7AcQkenA/YBrojDAxYeUiwP/uDEepfKGMbDxc1j+UsbBfsBqCNf4dX2s1c02\nbTrKkSOx3Huv9Yjr4MFN6N69jtZFXCN3JopywGGX+Sjg9ku2eRNYKCL9gCLAPZkdSER6Aj0BKlSo\nkOuBKpUrzh+DRT0v77K7ZE2o/zzU/pcWL7lZbGwSb7yxjFGjVlOyZCA7d/YlNDQQf38fTRLXwe7K\n7G7AVGPMJyLSGPiviNQyxjhcNzLGfAF8ARAZGWlsiFOpzJ3aDkv7w6m/rL6XXIXVgpbjofyd9sRW\niBhj+OmnnTz//Hyios7h5SU8+mhtfH11oKXc4M5EEQ3c5DJf3rnMVQ+gLYAx5g8RCQDCgONujEup\n67PtP1ZfSwfmZb7+5ofgro8gpErexlVI/f33Wfr2nccvv+wGIDLyRiZO7MCtt5a1ObKCw52JYi1Q\nXUQqYyWIrsCjl2xzCGgJTBWRCCAAOOHGmJS6NsbAvp9h1v2Zr6/1DNw2CELD8zauQs4YQ+fO37F+\n/RGKFfPn/fdb0KtXJN7eeieRm9yWKIwxKSLSF1iA9ejrZGPMNhF5G1hnjJkNvARMEpH+WBXbTxlj\ntGhJ5R+xUbBiMOz8+pIVAveMg8r3QjF9zDKvORwGLy9BRBgxojUTJqzj00/bULZssN2hFUjiaZ/L\nkZGRZt26bIZyVCo3RP8O05tcvtwvGB6Yq/UONjl1Kp5XXrHG3pg0qaPN0XgWEVlvjIm8ln3trsxW\nKn85sga+vuThPG8/uHsU1HwSfAPtiauQM8bw1VebefnlRZw8GY+fnzdvvNGc8uW1C/C8oIlCqdO7\nYcsE2P5fSDiZcV3rL6H2M/bEpQDYseMEzz03h+XL/wagefNKjB/fXpNEHtJEoQqnC3Gw5gNY/X7m\n6+/60Kqc1hHibGOM4fXXl/LRR7+RnOwgLCyITz5pTffudRD9u+QpTRSqcDmxBaZFgiM54/IiZaHE\nzdD8UyhT357YVAYiQnR0LMnJDv7v/27lww/vITRUi/7soIlCFQ4ntsLyly/vc6nWM9BijNY95BP/\n/BPLyZPx1KlTBoDhw1vRo0d9mjTRHhnspIlCFVwxB2DnN7BuJCSeSl/uEwCdF0D5pvbFpjJITXUw\nfvw6hg79lXLlgtm0qRd+ft6EhQURFqZJwm6aKFTB4kiB3167fJxpgIBQ6+mliMe07iEf2bDhCM8+\n+wvr1lldoDRtWpFz55IIC9NxIvKLHCUKEfEDKhhj9ro5HqWujSPFahi3fmTG5QEloUF/qPm4NozL\nZ86dS+K1135lzJi1OByG8uWLMXp0Wzp1qqGV1fnMFROFiLQHRgJ+QGURqQe8YYx5wN3BKXVFZ/fD\n0hdg/y8Zl9fqAXd9AEGl7IlLZcsYQ9OmU9i8+Rje3sKAAY14883mBAf72x2aykRO7ijexuoefCmA\nMWaTiFRza1RKXcnWybCkN6QmZVxep6fVIV9AiD1xqRwREfr3b8S4ceuYOLED9erdYHdIKhs5SRTJ\nxpizl9wKela/H6pgSEmEZQNg8/iMy739oNUkqNld6x7yqQsXUhk58g+8vYWBA62uUZ54oi6PP15H\nO/DzADlJFDtE5BHAy9kT7PPAn+4NSymnpBg4sRl+7We1gXBVvDJ0XQVFb7QnNpUjK1f+Ta9ec9i+\n/QT+/t488URdypQpiojg7a2J3RPkJFH0BV4HHMCPWL3BvurOoFQht+9nWPsxxEVZj7i68vaHW1+0\n6h/07iFfO3kynkGDFjFlyiYAqlcPZdy49pQpU9TmyNTVykmiaGOMGQwMvrhARB7EShpK5Y6T22Dl\nENj/c8bl4gUhVaF8M6j+IFRqqwkinzPGMHXqJgYOXMSpUwn4+XkzZMidvPLKnQQE6BP5nignf7Vh\nXJ4UhmayTKmrF7USZrS6vFK6Rje4+WFrvAefAHtiU9ds2rStnDqVQIsWlRk3rh3h4WF2h6SuQ5aJ\nQkTaYA1TWk5EXB9OL4ZVDKXUtds8ARY/d/nyur2h+SeaHDxMfHwyMTGJlC0bjIgwblw71q79h8ce\nq61tIgqA7O4ojgN/AYnANpflscAr7gxKFVDGWE8sLemTcXnxKtBqIlS8x5641HWZN28PffrMpUqV\nEixa1B0RITw8TO8iCpAsE4UxZiOwUUT+Z4xJzMOYVEFzZo9V/7Dnh4zLi5SFdv+DCnfbE5e6LtHR\n53jxxQXMmLEdgOBgf06dStCuNwqgnNRRlBOR94CaQFp5gDHmZrdFpTyfIxWiVsAfb0HU8ozrSteH\nDt9Cier2xKauS2qqg7Fj1zJs2K/Exl6gSBFf3n77bp5//nZ8fLRNREGUk0QxFXgXGAHcCzyNNrhT\nWTEOWDsCVg7OuDwgFCIHQt1e2mragzkchmbNpvLbb4cB6NSpBqNGtaVCheI2R6bcKSeJIsgYs0BE\nRhhj9gHDRGQd8JqbY1OexJEC3zaDf37PuLzyvdYjrfX76WOtBYCXl9C6dVUOHYphzJh2dOwYbndI\nKg/kJFEkiYgXsE9EegHRQLB7w1IeIznB6pRv66T0ZQGhVrFSl5Xg7WtfbOq6GWP47rtt+Ph40blz\nTQAGD27CgAGNKVrUz+boVF7JSaLoDxTB6rrjPaA4oKPNF3aOFPjvrXBya8bl1TpBxx/17qEA2Lfv\nNL17z2Xhwn2UKhVEixaVKVEiEH9/H/y1k9dC5YqJwhiz2jkZC3QHEJFy7gxK5WOOVGvch02fQ+qF\n9OW3PA0tRoOfds/g6ZKSUvj44995772VJCamUKJEAO+914LixbVtS2GVbaIQkduAcsAqY8xJEbkF\nqyuPFkD5PIhP5RfGwLb/wIqBkHDSWlakrFXE9MhSq6sN5fGWLTvIc8/NYedO62/cvXsdRoxoTenS\nRWyOTNkpu5bZHwCdgc1YFdi/AL2Bj4BeeROeyhdObIXpTeBCrDXvFwx3j4ZbntQipgIkNdVB795W\nkggPL8n48e25++7Kdoel8oHs7ijuB+oaYxJEJBQ4DNQ2xuzPm9BUvvDXFFjgUiVVtzfc9T746+OQ\nBYHDYUhMTCEoyBdvby/Gj2/PihV/M2hQE/z9tQM/ZcnunZBojEkAMMacFpHdmiQKEWNg7uOw8+v0\nZZ1mQ9X77ItJ5aqtW4/Rq9ccatQoyZdf3g9As2aVaNaskr2BqXwnu0RRRUQu9hArWONlp/UYa4x5\n0K2RKfvsnwsz26fP+wVDr2PgG2hfTCrXnD9/gbffXs7IkX+SkuLgwIEznDmTQIkS+vdVmcsuUXS+\nZH6MOwNR+UTUioxJotkIiHzJvnhUrvr551307TuPQ4diEIHevSN5772WhIToE00qa9l1CrgkLwNR\nNju4CH5/DY6sTl/2+AYoU9++mFSuSUlx0KXLDH78cQcA9erdwMSJHWjYUJ90V1emtVWFnSMVvigP\n54+mLyvfDNp/rWNRFyA+Pl4UL+5P0aJ+vPPO3fTt21A78FM5Jsa4r38/EWkLjAK8gX8bYz7MZJtH\ngDexOhrcbIx5NLtjRkZGmnXr1rkh2kLm1A6Y3RliD0HyeWuZlw88vATKN7U3NpUrVq+OAuD2260m\nT6dOxZOQkEL58sXsDEvZRETWG2Mir2XfHN9RiIi/MSbpylumbe8NjAVaAVHAWhGZbYzZ7rJNdWAI\n0MQYc0ZESuc8dHXN9v0MP3XMuKzJO9BomD3xqFx19mwiQ4YsZuLE9dSoEcamTb3w8/OmZEkdJ0Jd\nmysmChFpCHyJ1cdTBRGpC/zLGNPvCrs2BPZefKRWRKZjtc3Y7rLN/wFjjTFnAIwxx6/+ElSOGQMb\nRsGy/unLWn0BtZ627iaURzPG8M03fzFgwAKOHTuPj48XHTuGk5rqwLqpV+ra5OTTYTTQAfgJwBiz\nWURyMiRZOaxGehdFAbdfss3NACLyG9Y7+U1jzPwcHFtdrbgjMNG1zkGgX4z16KvyeHv2nKJ377ks\nXmw1dWrS5CYmTOhArVp6k66uX04ShZcx5u9LBkhPzcXzVweaY/UdtUJEahtjzrpuJCI9gZ4AFSpU\nyKVTFyKrhsHq9zIue/68tosoIJKTU2nR4iuios4RGhrI8OH38PTT9fHy0u5VVO7ISaI47Cx+Ms56\nh37A7hzsFw3c5DJf3rnMVRSw2hiTDBwQkd1YiWOt60bGmC+AL8CqzM7BuRVA/HGYWgsSTqQvazcN\nIh6zLyaVa4wxiAi+vt68914Lli49yPDh91CqlHbgp3JXTp6Pew4YAFQAjgGNnMuuZC1QXUQqi4gf\n0BWYfck2P2HdTSAiYVhFUdpNSG44vRvGl0lPEhVbQ68jmiQKgGPH4ujefSbvvrsibdkTT9RlypT7\nNUkot8iQWbN5AAAgAElEQVTJHUWKMabr1R7YGJMiIn2BBVj1D5ONMdtE5G1gnTFmtnNdaxHZjlWc\nNdAYc+pqz6UuERsNU1yGqOyyXB95LQAcDsOkSet55ZUlnD2bSEhIAC++2IjgYB1FSLnXFdtRiMg+\nYBfwLfCjMSY2LwLLirajyIZxwLIB1pNNFz28BCq0sC8mlSs2bz5Kr15z+PNPq21E27bVGDu2HVWq\nlLA5MuUp3NqOwhhTVUTuwCo6ektENgHTjTHTr+WEyk2MA0Ze8gjkI8vgpma2hKNyR3JyKkOGLOGz\nz/4kNdVQtmxRRo1qy0MP1UR0LBCVR3LUht8Y87sx5nngVuAc8D+3RqWuTmx0xiRRtzcMcGiSKAB8\nfLzYuPEoDoehX7+G7NjRh4cfvkWThMpTOWlwVxSroVxXIAKYBdzh5rhUTqVesPpquqh4ZbhnrH3x\nqOt26FAMqakOKlcugYgwYUJ7YmKSiIzUvreUPXJSmf0X8DMw3Biz0s3xqKvhSIVRLt0y6KOvHi05\nOZVRo1bzxhvLaNy4PIsWdUdEqF69pN2hqUIuJ4miijHG4fZI1NU59Ct83zJ9/s4PNEl4sD/+OEyv\nXnPYsuUYAKGhgcTHJ1OkiJ/NkSmVTaIQkU+MMS8BP4jIZY9G6Qh3NklOgO/vzjhuRPtvoMZVP8Gs\n8oEzZxJ45ZXFfPHFBgAqVw5h7Nh23HtvdZsjUypddncU3zp/68h2+UVSDIwJSZ8PrgBPbIIAfUTS\nEyUlpVCv3kQOHYrB19eLgQPvYOjQpgQF+dodmlIZZDfC3RrnZIQxJkOycDak0xHw8lJyQsYkod2C\nezx/fx969KjPkiUHGD++PTVrlrI7JKUylZMGdxuMMbdesmyjMcaWMTILZYO75AQY7VJpfdtgaHrZ\nGFAqn0tMTOGDD1YSHh7Go4/WBqwhSr29RR93VW7nlgZ3ItIF65HYyiLyo8uqYOBs5nspt/jPLenT\nTd6FRkPti0Vdk0WL9tG791z27j1N6dJFeOCBGgQG+upwpMojZFdHsQY4hdXrq+uD+bHARncGpVz8\npw7EHLCmqz+oScLDHD0ax4ABC/jmm78AuOWWUkyY0IHAQK2HUJ4juzqKA8ABYHHehaMyWDsCTm61\npvVOwqOkpjqYOHE9r766hJiYJAIDfXjjjWb0798YPz8dbU55luyKnpYbY5qJyBnAtSJDAGOMCXV7\ndIWZMbBiYPq8JgmPkppq+PzzNcTEJNGuXXXGjLmXypX16TTlmbIrero43GlYXgSiXBgDU2umzz+z\nx75YVI7FxiaRmmoICQnAz8+bSZPu49ixOB58MEIrq5VHy7ImzaU19k2AtzEmFWgMPAvo6CjucnQt\njPSC0zut+dr/ghLV7I1JZcsYw48/7iAiYiwvvbQgbfmdd1agc2ft5VV5vpw8cvET1jCoVYEpWEOV\nfu3WqAqrqFXwv4bp81Xug9aT7ItHXdHBg2fp2HE6nTt/R3R0LH/9dYLExBS7w1IqV+WkryeHMSZZ\nRB4EPjfGjBYRfeopt8X9A9/elT7/1HYoGWFfPCpbycmpjBz5B2+9tZyEhBSKFfPn/fdb0KtXJN7e\n+sirKlhyNBSqiDwMdAc6OZfps325yZECE8ulz3ddpUkiH4uPT6ZRo3+zdetxALp2rcXIka0pWzbY\n5siUco+cJIpngN5Y3YzvF5HKwDfuDasQOb4J/uvSyL39dCjXxL541BUFBfkSGXkj8fHJjBvXntat\nq9odklJudcUuPABExAe4WKO61xhjWyFsgerC49h6mObSor56Z+g4w754VKaMMXz11WaqVg3lzjsr\nABATk4ifn7c2nFMew61jZovIXcB/gWisNhQ3iEh3Y8xv13JC5bThc1j6fPp8h28h/BH74lGZ2rHj\nBM89N4fly/8mIiKMTZt64efnTfHiAXaHplSeyUnR06dAO2PMdgARicBKHNeUmRSw4+uMSeKZ3VBC\nxx/ITxISknnvvZUMH/4byckOSpUKYsiQO/H11YpqVfjkJFH4XUwSAMaYHSKiw25dK0cqzHUZia73\nSQjUoS7zk/nz99Knz1z27z8DwP/93618+OE9hIYG2hyZUvbISaLYICITgGnO+cfQTgGvjXFkrLju\nfQoCtSeU/CQu7gLdu8/k5Ml4atUqzYQJ7WnSpILdYSllq5wkil7A88Ag5/xK4HO3RVRQnd0HX7q0\nsG46XJNEPpGa6sDhMPj6elO0qB+jRrUlKuoc/fs3wtdXO/BTKttEISK1garATGPM8LwJqQCKP5kx\nSdzxFtw2MOvtVZ5Zv/4fnn32F+6/P5zXXmsGkDaokFLKkmXNnIi8itV9x2PAIhF5Js+iKkhSEjMW\nNz38KzR+3b54FADnziXxwgvzaNjw36xff4T//ncLycmpdoelVL6U3R3FY0AdY8x5ESkFzAUm501Y\nBUTCKfi2GcRFWfNtpkCFu7PfR7mVMYYZM7bzwgvzOXIkDm9vYcCARrz11t1azKRUFrJLFEnGmPMA\nxpgTIqLPBV6NA/Phx3vT59t9DRHd7ItHERubRJcuM5g3by8At99ejgkTOlCv3g02R6ZU/pZdoqji\nMla2AFVdx842xjzo1sg82antGZPEQ4uhYkv74lEAFC3qR1JSKsWL+/Phh/fQs2cDvLy0C3ClriS7\nRNH5kvkx7gykwDiyBr6+PX3+8XVQpoF98RRyK1b8TdmyRalevSQiwuTJHQkI8KFMmaJ2h6aUx8hu\nzOwleRlIgZB6IWOSeHCeJgmbnDwZz6BBi5gyZRMtW1Zm0aLuiAgVK4bYHZpSHicn7ShUTk1wKevu\nsQ9CqtgXSyHlcBimTt3EwIGLOH06AT8/b+66qwKpqQYfHy1mUupauLWCWkTaisguEdkrIq9ks11n\nETEi4rn9R22ZBIlWlw80eVeThA22bTtO8+ZT6dFjNqdPJ9CyZWW2bn2ON95ojo+PPouh1LXK8R2F\niPgbY5KuYntvYCzQCogC1orIbNd+o5zbBQMvAKtzeux85+x+WNTTmq7QEhoNtTeeQigmJpFGjb4k\nLu4CpUsXYeTI1jz6aG0dr1qpXHDFr1ki0lBEtgJ7nPN1RSQnXXg0xBq7Yr8x5gIwHbg/k+3eAT4C\nEnMedj5iDHzpMnBNqy/si6UQujieSvHiAQwe3IRevRqwc2cfHnusjiYJpXJJTu7HRwMdgFMAxpjN\nQE5ajZUDDrvMRzmXpRGRW4GbjDFzsjuQiPQUkXUisu7EiRM5OHUeGunyEraZokVOeSQ6+hwPPfQd\n06ZtSVs2dOhdjB/fgRIltJdXpXJTThKFlzHm70uWXXdfB84GfCOBl660rTHmC2NMpDEmslSpUtd7\n6tzz19T0af8QqPWUXZEUGikpDkaN+pMaNcbyww87eOONZaSmOgD0DkIpN8lJHcVhEWkIGGe9Qz9g\ndw72iwZucpkv71x2UTBQC1jm/Ae/AZgtIh2NMfl/rNPvW8KhX9Pn+5y2L5ZCYu3aaHr1msOGDUcA\n6NSpBqNHt8XbWyuqlXKnnCSK57CKnyoAx4DFzmVXshaoLiKVsRJEV+DRiyuNMTFA2MV5EVkGvOwR\nSWJKBJzemT7fYx/ot1m3OX/+AoMHL2bcuLUYAxUqFOfzz++lY8dwu0NTqlC4YqIwxhzH+pC/KsaY\nFBHpCywAvIHJxphtIvI2sM4YM/uqo80P1n2SniR8i0K/c5ok3MzHx4vFi/fj5SUMGNCYN95oRpEi\nOsiiUnlFLj41kuUGIpOAyzYyxvR0V1DZiYyMNOvW2XTTEXMA/u1SWf1S9q+dunb79p0mJCSAkiWD\nAKvYKSDAh9q1y9gcmVKeSUTWG2Ouqa1aTgp3FwNLnD+/AaWBHLenKFAm35w+3fesfXEUYElJKbz7\n7gpq1RrP4MGL05bfdls5TRJK2SQnRU/fus6LyH+BVW6LKL/aPxccKdZ01Y7gX9zeeAqgZcsO8txz\nc9i58yRgPeGUmurQymqlbHYtfT1VBgrXV7vTu2Bm+/T5Dt9mva26asePn2fgwEV89dVmAMLDSzJ+\nfHvuvruyzZEppSAHiUJEzpBeR+EFnAay7LepwDmzF6bUSJ9/ehf4BNgXTwFz8mQ8ERFjOX06AX9/\nb4YOvYtBg5rg76/9VSqVX2T73yhWA4e6pLd/cJgr1X4XNJOrp093/Q1Cb856W3XVwsKCuP/+cKKi\nzjFuXHuqVQu1OySl1CWyTRTGGCMic40xtfIqoHxl21fp011WQLk77IulgDh//gJvv72c9u1vpmnT\nigCMG9cef39vbVmtVD6Vk1rCTSJS3+2R5Ddb/g3zn0yfL3+XfbEUED//vIuaNccxfPjv9O49B4fD\nujkNCPDRJKFUPpblHYWI+BhjUoD6WF2E7wPOY42fbYwxt+ZRjHkvJQkW/V/6fN8Y+2IpAA4fjuGF\nF+Yzc6bVULF+/RuYOLGDjletlIfIruhpDXAr0DGPYsk/No9Pn+53DvyC7YvFg6WkOBg9ejWvv76U\n8+eTKVrUj3ffvZs+fRrqQEJKeZDsEoUAGGP25VEs+UNKEizrb003eEmTxHU4dy6JDz5YxfnzyXTu\nHMFnn7WlfPlidoellLpK2SWKUiIyIKuVxpiRbojHfjPuSZ9u8pZ9cXios2cTCQz0wd/fh9DQQCZO\n7IC/vzft2+vTYkp5quzu/72BoljdgWf2U/DsnQ3RzkbnpeuDbxF74/Egxhi+/nor4eFjGD78t7Tl\nDz4YoUlCKQ+X3R3FEWPM23kWid1O74JZLiO1Pr7evlg8zO7dp+jdew5LlhwAYMWKQxhj9EkmpQqI\nK9ZRFAopSRlbXz+xWbsOz4HExBQ++mgV77+/igsXUgkNDeTjj1vx1FP1NEkoVYBklyha5lkUdtvn\nMjRG941Qqo59sXiIo0fjaNp0Cnv2WCP7PfVUPT7+uBVhYUE2R6aUym1ZJgpjTOEZ23O5c9jusFpQ\nup69sXiIMmWKcNNNxfHx8WL8+PY0a1bJ7pCUUm6iPa8d2wixh63p8KseyK/QcDgMkyat5+67K3Pz\nzSUREb7++kFKlAjEz8/b7vCUUm6krZ6m35k+3aC/fXHkY5s3H6VJk8n06jWH3r3ncLFfyDJlimqS\nUKoQKNx3FF83gpR4a7rLSvDV8nVXcXEXePPNZXz22Z+kphpuvDGYXr2uaSRFpZQHK7yJ4tgGOLLa\nmm44BMrfmf32hcxPP+2kX795REWdw8tL6NevIe++24JixfztDk0plccKb6JY+kL69F3v2xdHPhQd\nfY6uXWeQlJRKgwZlmTChA5GRN9odllLKJoUzURxcmN4Cu8m79saSTyQnp+Lj44WIUK5cMd57rwV+\nft707n2bjlmtVCFXOD8BfmiTPt2w8IzqmpXffz9MgwZfMG3alrRlL710B/363a5JQilVCBNFbHT6\n9P2zwKvwPrVz+nQCzz77M02aTGbr1uOMG7eOwjbSrVLqygpf0dNvw9KnqxW+oTbA6sBv2rQtvPTS\nQk6ciMfX14tBg5owdOhd2vWGUuoyhStRpCTBtqnW9J0f2BqKXY4di6Nbtx9YuvQgAM2aVWT8+PZE\nRJSyNzClVL5VuBLFli/Sp2972b44bBQSEsCRI3GEhQUxYkQrnniirt5FKKWyVXgShTGw9Hlrulgl\n8Co8l75o0T5uvbUsJUsG4e/vw/ffP0zZskUpWVIbGCqlrqzwVGYfcxlf4qFF9sWRh44ciaVbtx9o\n3XoagwcvTlteq1ZpTRJKqRwrPF+rVwyyfgeEQolq9sbiZqmpDiZOXM+QIUs4dy6JwEAfwsNL6mBC\nSqlrUjgSxZHVcHipNV32dntjcbMNG47Qq9cvrF37DwDt21dnzJh2VKoUYnNkSilPVTgSxdeN0qfb\nf2NfHG528OBZGjacRGqqoVy5YEaPvpcHHqihdxFKqevi1kQhIm2BUYA38G9jzIeXrB8A/AtIAU4A\nzxhj/s7VIFa79OPUaTb4F8/Vw+cnlSqF8PTT9QgO9uett5oTHKwd+Cmlrp/bKrNFxBsYC9wL1AS6\niUjNSzbbCEQaY+oAM4DhuRpE8nlYNdSaLlYRqt6Xq4e328GDZ7nvvm9Yvvxg2rIvvriPkSPbaJJQ\nSuUad95RNAT2GmP2A4jIdOB+YPvFDYwxS122/xN4PFcjmPVg+nS333P10HZKTk5l5Mg/eOut5SQk\npHDyZDx//NEDQIuZlFK5zp2Johxw2GU+CsiuJrkHMC+zFSLSE+gJUKFChZydPTUZ/l5oTZeuD0UL\nRjfZq1YdolevX9i27QQAXbvWYuTI1jZHpZQqyPJFZbaIPA5EAs0yW2+M+QL4AiAyMjJnvdbN6ZY+\n/cjSrLfzEGfOJDBw4CK+/HIjAFWrlmDcuPa0bl3V5siUUgWdOxNFNHCTy3x557IMROQeYCjQzBiT\nlDtn/h32/GBN3zaoQFRgOxyGWbN24evrxSuv3MmQIXcSGOhrd1hKqULAnYliLVBdRCpjJYiuwKOu\nG4hIfWAi0NYYczxXzmoMTG+SPu/BAxPt3HmSypVD8Pf3oWTJIP73vwepUKE4NWqE2R2aUqoQcdtT\nT8aYFKAvsADYAXxnjNkmIm+LyMX+vT8GigLfi8gmEZl93Sde+mL69CPLwNvzvnXHxyczdOgS6tQZ\nz/Dhv6Utb926qiYJpVSec2sdhTFmLjD3kmWvu0zfk+sn3Tja+l3kBrgp0yqPfG3+/L307j2HAwfO\nAnDyZLzNESmlCrt8UZmda84fS5/uvNC+OK7BP//E8uKL8/n+e+vp4dq1SzNhQgfuuOOmK+yplFLu\nVbASxcoh6dNhteyL4yrt3n2KyMgviI29QFCQL2++2YwXX2yEr2/hHaZVKZV/FKxEcWqb9btqR/Cg\nhmfVq4dy223lKFLEl88/v5eKFbUDP6VU/lFwEoUxcHSNNV2np72xXMG5c0m8/vpSeve+jZtvLomI\nMHt2V4oU8bM7NKWUukzBSRQHXOrMb8ifXYkbY5gxYzsvvDCfI0fi2LnzJPPnW72WaJJQSuVXBSdR\nzHE20QgqDUH57xHS/fvP0LfvXObN2wtAo0bl+eij3H/oSymlclvBSBSOVLhwzppuMcbeWC5x4UIq\nI0b8zjvvrCAxMYWQkAA+/LAl//d/DfDy8px6FKVU4VUwEkVcVPr0zZ3tiyMThw/H8Pbby0lKSuWx\nx2rzySetKVOmqN1hKaVUjhWMRPH34vRpcVtj8xw7cyaBkJAARISqVUMZNaot1aqF0rJlFbtDU0qp\nq2b/p2pu2DTW+l2hpa1hOByGyZM3Uq3a50ybtiVt+bPPRmqSUEp5rIKRKJLPW7+97RvVbdu24zRv\nPpUePWZz+nRCWqW1Ukp5uoJR9HRmt/X7lifz/NTx8cm8885yRoz4g5QUB6VLF+HTT9vQrZvntAxX\nSqnseH6i2PdL+nTZvG0/sXv3Kdq0mcbBg2cRgV69GvD++y0pUSIwT+NQSil38vxEMfcx63dQGShW\nMU9PXbFicQICfKhbtwwTJnSgUaPyeXp+lb8lJycTFRVFYmKi3aGoQiQgIIDy5cvj65t7Qyx4dqJI\njk9vP/Hw4uy3zQUpKQ4mTFhHt261KFkyCH9/H+bPf4xy5Yrh41MwqntU7omKiiI4OJhKlSohHtT3\nmPJcxhhOnTpFVFQUlStXzrXjevan2+7v06fd3FvsmjXRNGw4iX795jF4cHpSqlgxRJOEylRiYiIl\nS5bUJKHyjIhQsmTJXL+L9ew7ivWfWr/dOCZ2TEwiQ4f+yrhxazEGKlQozv33h7vtfKpg0SSh8po7\n3nOemyhio+HEZmu6zZRcP7wxhm+/3Ub//gs4ejQOHx8vBgxoxOuvN9MO/JRShYrnlpl84VJxXK1T\nrh9+8+ZjdOv2A0ePxnHHHTexYUNPPvqolSYJ5VG8vb2pV68etWrV4r777uPs2bNp67Zt20aLFi0I\nDw+nevXqvPPOOxhj0tbPmzePyMhIatasSf369XnppZfsuIRsbdy4kR49etgdRrY++OADqlWrRnh4\nOAsWLMh0m7vuuot69epRr149brzxRjp1sj7TZs2aRZ06dahXrx6RkZGsWrUKgBMnTtC2bds8uwaM\nMR7106BBA2NSLhgzAutn1esmt6SkpGaY799/vpk0ab1JTXXk2jlU4bF9+3a7QzBFihRJm37iiSfM\nu+++a4wxJj4+3lSpUsUsWLDAGGPM+fPnTdu2bc2YMWOMMcZs3brVVKlSxezYscMYY0xKSooZN25c\nrsaWnJx83cd46KGHzKZNm/L0nFdj27Ztpk6dOiYxMdHs37/fVKlSxaSkpGS7z4MPPmj+85//GGOM\niY2NNQ6H9fmzefNmEx4enrbdU089ZVatWpXpMTJ77wHrzDV+7npm0VPU8vTpJm/lyiGXLj1A795z\nmTixA02bWo/ZjhzZJleOrRSfuKmu4iVz5W2cGjduzJYtVtcyX3/9NU2aNKF169YABAUFMWbMGJo3\nb06fPn0YPnw4Q4cOpUaNGoB1Z/Lcc89ddsy4uDj69evHunXrEBHeeOMNOnfuTNGiRYmLiwNgxowZ\n/PLLL0ydOpWnnnqKgIAANm7cSJMmTfjxxx/ZtGkTISHWqI7Vq1dn1apVeHl50atXLw4dOgTAZ599\nRpMmTTKcOzY2li1btlC3bl0A1qxZwwsvvEBiYiKBgYFMmTKF8PBwpk6dyo8//khcXBypqaksX76c\njz/+mO+++46kpCQeeOAB3nrL+hzp1KkThw8fJjExkRdeeIGePa9vELRZs2bRtWtX/P39qVy5MtWq\nVWPNmjU0btw40+3PnTvHr7/+ypQpVnF60aLpHYieP38+Q/1Dp06d+N///nfZ6+IOnpkoDmZ++3Yt\njh8/z8CBi/jqK6u+Y+TIP9IShVIFRWpqKkuWLEkrptm2bRsNGjTIsE3VqlWJi4vj3Llz/PXXXzkq\nanrnnXcoXrw4W7duBeDMmTNX3CcqKorff/8db29vUlNTmTlzJk8//TSrV6+mYsWKlClThkcffZT+\n/ftz5513cujQIdq0acOOHTsyHGfdunXUqpX+tGONGjVYuXIlPj4+LF68mFdffZUffvgBgA0bNrBl\nyxZCQ0NZuHAhe/bsYc2aNRhj6NixIytWrKBp06ZMnjyZ0NBQEhISuO222+jcuTMlS5bMcN7+/fuz\ndOnSy66ra9euvPLKKxmWRUdH06hRo7T58uXLEx0dneVr89NPP9GyZUuKFSuWtmzmzJkMGTKE48eP\nM2fOnLTlkZGRDBs2LLuXOtd4ZqLYMtH6XeW+az6Ew2H48ssNDB68mDNnEvH392bYsKYMHHhHLgWp\nlIur+OafmxISEqhXrx7R0dFERETQqlWrXD3+4sWLmT59etp8iRIlrrjPww8/jLe3NwBdunTh7bff\n5umnn2b69Ol06dIl7bjbt29P2+fcuXPExcVl+IZ95MgRSpUqlTYfExPDk08+yZ49exARkpOT09a1\natWK0NBQABYuXMjChQupX78+YN0V7dmzh6ZNmzJ69GhmzpwJwOHDh9mzZ89lieLTTz/N2YtzDb75\n5hv+9a9/ZVj2wAMP8MADD7BixQpee+01Fi+2Hs8vXbo0//zzj9ticeWZieJCrPX7htuuafcDB87w\n+OMz+f33wwC0bl2VsWPbUa1aaG5FqFS+EBgYyKZNm4iPj6dNmzaMHTuW559/npo1a7JixYoM2+7f\nv5+iRYtSrFgxbrnlFtavX59WrHO1XItILn2mv0iRImnTjRs3Zu/evZw4cYKffvop7Ruyw+Hgzz//\nJCAgINtrcz32a6+9xt13383MmTM5ePAgzZs3z/ScxhiGDBnCs88+m+F4y5YtY/Hixfzxxx8EBQXR\nvHnzTNsjXM0dRbly5Th8+HDafFRUFOXKlcv0ek6ePMmaNWvSEtWlmjZtyv79+zl58iRhYWFpRWx5\nwfOeejKp6dMN+l/TIYoV82f37lPccENRpk/vzPz5j2mSUAVaUFAQo0eP5pNPPiElJYXHHnuMVatW\npX07TUhI4Pnnn2fQoEEADBw4kPfff5/du60ONx0OBxMmTLjsuK1atWLs2LFp8xeLnsqUKcOOHTtw\nOBxZfvCBlVAeeOABBgwYQERERNq399atW/P555+nbbdp06bL9o2IiGDv3vRemmNiYtI+hKdOnZrl\nOdu0acPkyZPT6lCio6M5fvw4MTExlChRgqCgIHbu3Mmff/6Z6f6ffvopmzZtuuzn0iQB0LFjR6ZP\nn05SUhIHDhxgz549NGzYMNPjzpgxgw4dOmRIjnv37k17Em3Dhg0kJSWlvUa7d+/OUPTmTp6XKJLS\nH+/DL+cjxS1YsJekpBQASpYMYvbsruzc2YcuXWppoyhVKNSvX586derwzTffEBgYyKxZs3j33XcJ\nDw+ndu3a3HbbbfTt2xeAOnXq8Nlnn9GtWzciIiKoVasW+/fvv+yYw4YN48yZM9SqVYu6deumfdP+\n8MMP6dChA3fccQdly5bNNq4uXbowbdq0tGIngNGjR7Nu3Trq1KlDzZo1M01SNWrUICYmhthYq4Rh\n0KBBDBkyhPr165OSkpLl+Vq3bs2jjz5K48aNqV27Ng899BCxsbG0bduWlJQUIiIieOWVVzLULVyr\nW265hUceeYSaNWvStm1bxo4dm1bs1q5duwxFR9OnT6dbt24Z9v/hhx+oVasW9erVo0+fPnz77bdp\nn1dLly6lffv21x1jTsjFbOUpIqsUNev6nIei5eHZw1fc/vDhGJ5/fj4//bSTd965m2HDmuZBlErB\njh07iIiIsDuMAu3TTz8lODj4snL9wqBp06bMmjUr03qhzN57IrLeGBN5LefyvDuKi4MU1c6+kU1K\nioORI/8gImIsP/20k6JF/QgN1e6/lSpInnvuOfz97RuwzC4nTpxgwIABOXp4IDd4ZmU2QIV7slz1\n559R9Or1C5s3HwOgc+cIRo1qS7lyxbLcRynleQICAujevbvdYeS5UqVKpbXezguemyjKZd7IZPXq\nKF9Kj9wAAAkmSURBVO6440uMgUqVQhgz5l7at785j4NTymKM0TowlafcUZ3guYkii3++hg3L0aZN\nNerXv4Fhw5oSFJR7g3codTUCAgI4deqUdjWu8oxxjkeR3WPF18IzE4XL2BN79pyif/8FjBzZhptv\ntv4h58x5FC8v/cdU9ipfvjxRUVGcOHHC7lBUIXJxhLvc5JmJIjSCpKQUPvxwFR98sIqkpFQCAnyY\nMeMRAE0SKl/w9fXN1VHGlLKLW596EpG2IrJLRPaKyGWtUUTEX0S+da5fLSKVcnLcJecepk6dCbz5\n5nKSklJ5+ul6TJjQIbfDV0ophRvvKETEGxgLtAKi/r+9+4+1uq7jOP58hRKYDlNm07TQgQgqEpFR\nbhlCjqhgFePHAKVhBkVNzf5o2LIfczWzLSK7UjqgiRQWdUeUOcMoBgolIJIpAjPKpSaxJliJr/74\nfK7ndL2c8703zq9734/tbOf7PZ/v9/s+751zPuf7+Z7z/gBbJbXb3l3WbD5w0PZQSTOBrwMzXru3\nkn0vnMrEOWkXI0YMpq3tg1HEL4QQaqiWZxSXAnts77X9b2A1MLVTm6nAinz/XmCCqlz1O3h4IAMG\n9OOWW65g+/YF0UmEEEKN1eyf2ZKmAZNsX5OX5wLvtL2orM2u3OZAXn4qt3m+076uBToKw18E7KpJ\n0K1nMPB81VZ9Q+SiJHJRErkoGW77lJ5s2BIXs20vA5YBSNrW07+h9zaRi5LIRUnkoiRyUSJpW0+3\nreXQ01+Ac8qWz87rumwj6QRgEPD3GsYUQgihm2rZUWwFhkk6V1J/YCbQ3qlNO3B1vj8N+LVbrUph\nCCH0cjUberL9sqRFwH1AP+Au249J+jJpku924E7gB5L2AC+QOpNqltUq5hYUuSiJXJRELkoiFyU9\nzkXLlRkPIYRQX61XZjyEEEJdRUcRQgihoqbtKGpV/qMVFcjFDZJ2S9op6QFJvfZfiNVyUdbuo5Is\nqdf+NLJILiRNz6+NxyStqneM9VLgPfIWSRskPZLfJ5MbEWetSbpL0rP5P2pdPS5JS3KedkoaU2jH\ntpvuRrr4/RRwHtAf2AGM7NTmk0Bbvj8T+GGj425gLsYDJ+X7C/tyLnK7U4CNwBZgbKPjbuDrYhjw\nCPDGvHxGo+NuYC6WAQvz/ZHA/kbHXaNcvAcYA+w6xuOTgV8AAsYBDxXZb7OeUdSk/EeLqpoL2xts\nH86LW0j/WemNirwuAL5Cqhv2Uj2Dq7Miufg48B3bBwFsP1vnGOulSC4MdExxOQj4ax3jqxvbG0m/\nID2WqcBKJ1uAUyWdWW2/zdpRvBn4c9nygbyuyza2XwYOAafXJbr6KpKLcvNJ3xh6o6q5yKfS59j+\neT0Da4Air4vzgfMlbZK0RdKkukVXX0VycTMwR9IBYD3w6fqE1nS6+3kCtEgJj1CMpDnAWODyRsfS\nCJJeB3wTmNfgUJrFCaThp/eSzjI3SrrY9j8aGlVjzAKW275N0rtI/9+6yPYrjQ6sFTTrGUWU/ygp\nkgskTQQWA1Ns/6tOsdVbtVycQioa+aCk/aQx2PZeekG7yOviANBu+z+29wFPkDqO3qZILuYDPwKw\nvRkYQCoY2NcU+jzprFk7iij/UVI1F5LeBtxB6iR66zg0VMmF7UO2B9seYnsI6XrNFNs9LobWxIq8\nR35KOptA0mDSUNTeegZZJ0Vy8TQwAUDSCFJH0RfnqG0Hrsq/fhoHHLL9TLWNmnLoybUr/9FyCubi\nVuBkYE2+nv+07SkNC7pGCuaiTyiYi/uAKyXtBo4Cn7Pd6866C+bis8D3JF1PurA9rzd+sZR0D+nL\nweB8PeaLwIkAtttI12cmA3uAw8DHCu23F+YqhBDCcdSsQ08hhBCaRHQUIYQQKoqOIoQQQkXRUYQQ\nQqgoOooQQggVRUcRmo6ko5K2l92GVGg75FiVMrt5zAdz9dEdueTF8B7sY4Gkq/L9eZLOKnvs+5JG\nHuc4t0oaXWCb6ySd9P8eO/Rd0VGEZnTE9uiy2/46HXe27UtIxSZv7e7Gtttsr8yL84Czyh67xvbu\n4xJlKc7bKRbndUB0FKHHoqMILSGfOfxW0h/y7d1dtLlQ0sP5LGSnpGF5/Zyy9XdI6lflcBuBoXnb\nCXkOg0dzrf/X5/VfU2kOkG/kdTdLulHSNFLNrbvzMQfmM4Gx+azj1Q/3fOaxtIdxbqasoJuk70ra\npjT3xJfyus+QOqwNkjbkdVdK2pzzuEbSyVWOE/q46ChCMxpYNuy0Nq97Fnif7THADGBJF9stAL5l\nezTpg/pALtcwA7gsrz8KzK5y/A8Bj0oaACwHZti+mFTJYKGk04EPAxfaHgV8tXxj2/cC20jf/Efb\nPlL28I/zth1mAKt7GOckUpmODottjwVGAZdLGmV7Camk9njb43Mpj5uAiTmX24Abqhwn9HFNWcIj\n9HlH8odluROBpXlM/iipblFnm4HFks4GfmL7SUkTgLcDW3N5k4GkTqcrd0s6AuwnlaEeDuyz/UR+\nfAXwKWApaa6LOyWtA9YVfWK2n5O0N9fZeRK4ANiU99udOPuTyraU52m6pGtJ7+szSRP07Oy07bi8\nflM+Tn9S3kI4pugoQqu4HvgbcAnpTPg1kxLZXiXpIeADwHpJnyDN5LXC9ucLHGN2eQFBSad11SjX\nFrqUVGRuGrAIuKIbz2U1MB14HFhr20qf2oXjBH5Puj7xbeAjks4FbgTeYfugpOWkwnedCbjf9qxu\nxBv6uBh6Cq1iEPBMnj9gLqn42/+QdB6wNw+3/Iw0BPMAME3SGbnNaSo+p/ifgCGShublucBv8pj+\nINvrSR3YJV1s+09S2fOurCXNNDaL1GnQ3ThzQbsvAOMkXUCave1F4JCkNwHvP0YsW4DLOp6TpDdI\n6ursLIRXRUcRWsXtwNWSdpCGa17sos10YJek7aR5KVbmXxrdBPxK0k7gftKwTFW2XyJV11wj6VHg\nFaCN9KG7Lu/vd3Q9xr8caOu4mN1pvweBPwJvtf1wXtftOPO1j9tIVWF3kObHfhxYRRrO6rAM+KWk\nDbafI/0i6558nM2kfIZwTFE9NoQQQkVxRhFCCKGi6ChCCCFUFB1FCCGEiqKjCCGEUFF0FCGEECqK\njiKEEEJF0VGEEEKo6L8nFR5tceMzeQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0ee52c64a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7964"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = int(len(x_test)/2)\n",
    "\n",
    "user = pd.DataFrame(x_test.iloc[0:15929]['user_id'])\n",
    "user['y_pred'] = y_predicted\n",
    "user['y_test'] = y_test\n",
    "user = user.sort_values(by=['user_id'])\n",
    "\n",
    "user_f = user.iloc[:h]\n",
    "user_s = user.iloc[h:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XlcVNX7wPHPww6KC6Bmmrsp5ppkmuWaS2q2WF+1sl9l\nCy6ZWaalZVm2mGmae6Vmm5VlWpqmZpqVueWSS+4pZO4iiCAw5/fHHWFAhEGZGQae9+vFi7uce+8z\nl2Geuffcc44YY1BKKaUuxcfTASillCrYNFEopZTKkSYKpZRSOdJEoZRSKkeaKJRSSuVIE4VSSqkc\naaJQeSYi94vIj56Ow9NEpJKIJIiIrxuPWUVEjIj4ueuYriQi20Sk1WVsp+9BNxJtR+HdROQAUA5I\nAxKAxUB/Y0yCJ+MqjOzn+lFjzDIPxlAF2A/4G2NSPRWHPRYD1DTG7HHxcapQQF5zUaVXFIXD7caY\n4kBDoBHwvIfjuSye/JZcWL6h54Web+UsTRSFiDHmP2AJVsIAQEQCRWSMiBwUkSMiMlVEgh3W3yEi\nm0TkjIjsFZGO9uUlReRDETksIrEi8tqFWywi8pCIrLZPTxGRMY5xiMh8ERlkn75aRL4WkWMisl9E\nBjiUe1lE5orIJyJyBngo62uyxzHbvv0/IjJcRHwc4vhVRCaKSJyI7BSRtlm2zek1/Coi40TkBPCy\niFQXkZ9E5ISIHBeRT0WklL38x0Al4Dv77abnst4GEpGfReRV+37jReRHEYlwiOdB+2s4ISIvisgB\nEbk1u7+liASLyDv28nEistrx7wbcb/+bHheRYQ7bNRGR30XktP11TxSRAIf1RkT6ichuYLd92XgR\nOWR/D2wQkVscyvuKyAv290a8ff01IrLKXmSz/Xx0t5fvYn8/nRaR30SkvsO+DojIEBHZApwVET/H\nc2CPfb09jiMiMta+6YVjnbYfq5nje9C+7XUislRETtq3fSG786oukzFGf7z4BzgA3GqfrghsBcY7\nrB8HLADCgFDgO+AN+7omQBzQDutLQwWgtn3dPGAaUAwoC6wFnrCvewhYbZ9uARwi4zZmaeAccLV9\nnxuAl4AAoBqwD+hgL/sykALcaS8bnM3rmw3Mt8deBdgF9HaIIxV4GvAHuttfT5iTryEVeBLwA4KB\nGvZzEQiUwfqAeje7c22frwIYwM8+/zOwF7jWvr+fgTft6+pg3Rq82X4uxthf+62X+LtOsm9fAfAF\nbrLHdeGY79uP0QBIBiLt2zUGmtpfUxVgBzDQYb8GWIr1fgi2L3sACLdv8wzwHxBkXzcY6z1VCxD7\n8cId9lXDYd+NgKPAjfaY/89+zgIdzt8m4BqHY6efU+B3oJd9ujjQNLvznM17MBQ4bI89yD5/o6f/\nNwvTj8cD0J8r/ANa/2gJQLz9n2k5UMq+ToCzQHWH8s2A/fbpacC4bPZZzv7hE+ywrCewwj7t+E8q\nwEGghX3+MeAn+/SNwMEs+34emGmffhlYlcNr8wXOA3Uclj0B/OwQx7/Yk5R92Vqgl5Ov4eCljm0v\ncyfwZ5ZznVuiGO6wvi+w2D79EvC5w7oQ+2u7KFFgJc1zQINs1l04ZsUsr7nHJV7DQGCew7wB2uTy\nuk9dODbwN3DHJcplTRRTgFezlPkbaOlw/h7J5v17IVGsAl4BIi7xmi+VKHo6/p30J/9/9D5h4XCn\nMWaZiLQEPgMigNNY34pDgA0icqGsYH0Ag/XNblE2+6uM9Q39sMN2PlhXDpkYY4yIzMH6Z10F3Ad8\n4rCfq0XktMMmvsAvDvMX7dNBhD2OfxyW/YP1LfuCWGP/tHBYf7WTryHTsUWkHDAeuAXrW6kP1odm\nXvznMJ2I9c0Ye0zpxzPGJNpveWUnAuub8d68HkdErgXGAlFYf3s/rKs6R1lf97NAb3uMBihhjwGs\n90hOcTiqDPyfiDzpsCzAvt9sj51Fb2AksFNE9gOvGGO+d+K4eYlRXQatoyhEjDErgVlYtzUAjmN9\nM73OGFPK/lPSWBXfYP3TVs9mV4ewvo1HOGxXwhhz3SUO/Tlwj4hUxrqK+NphP/sd9lHKGBNqjOnk\nGHYOL+k41u2Zyg7LKgGxDvMVxCET2Nf/6+RryHrs1+3L6hljSmDdkpEcyufFYaxbg4BVB4F1uyc7\nx4Eksv/b5GYKsBPraaQSwAtkfg3g8Drs9RHPAf8DShtjSmHdvruwzaXeI9k5BIzK8vcOMcZ8nt2x\nszLG7DbG9MS6TfgWMFdEiuW0jcNxqzkZo7oMmigKn3eBdiLSwBhjw7qXPU5EygKISAUR6WAv+yHw\nsIi0FREf+7raxpjDwI/AOyJSwr6uuv2K5SLGmD+xPtw+AJYYYy5cQawF4u0VmMH2itG6InKDMy/E\nGJMGfAmMEpFQeyIaRMYVC1gfKgNExF9E7gUigUV5fQ12oVi38eJEpALW/XlHR7j8D6S5wO0icpO9\ncvllLv4AB8D+d5sBjBXrYQBfewVuoBPHCQXOAAkiUhvo40T5VOAY4CciL2FdUVzwAfCqiNQUS30R\nuZDgsp6P94FoEbnRXraYiHQWkVAn4kZEHhCRMvbXf+E9ZLPHZuPS5/57oLyIDBTr4Y1QEbnRmWMq\n52iiKGSMMcewKoBfsi8aAuwB1oj1ZNEyrIpJjDFrgYexKrzjgJVkfHt/EOu2wXas2y9zgfI5HPoz\n4Fb77wuxpAFdsJ7C2k9GMimZh5f0JFY9yz5gtX3/MxzW/wHUtO97FHCPMebCLZ28voZXgOuxzsVC\n4Jss698Ahtuf6Hk2D68BY8w2+2uZg3V1kYBV8Zt8iU2exapEXgecxPqG7cz/67NYt//isT64v8il\n/BKstje7sG7bJZH59tBYrGT9I1YC+hCrEh2sZPeR/Xz8zxizHquOaiLW+d5DNk+y5aAjsE1EErBu\nAfYwxpwzxiRi/W1/tR+rqeNGxph4rIcQbse6JbcbaJ2H46pcaIM75bVE5CGsBnA3ezqWvBKR4ljf\nmmsaY/Z7Oh6lcqJXFEq5iYjcLiIh9vvuY7CuGA54NiqlcqeJQin3uQOrov1frNtlPYxe0isvoLee\nlFJK5UivKJRSSuXI6xrcRUREmCpVqng6DKWU8iobNmw4bowpcznbel2iqFKlCuvXr/d0GEop5VVE\n5J/cS2VPbz0ppZTKkSYKpZRSOdJEoZRSKkeaKJRSSuVIE4VSSqkcaaJQSimVI5clChGZISJHReSv\nS6wXEZkgIntEZIuIXO+qWJRSSl0+V7ajmIXV3fDsS6y/Dau/m5pYg91Msf9WSil1JZJOQdJJOB8P\nKWc5f/q/3LfJgcsShTFmlYhUyaHIHcBse6doa0SklIiUtw84o5RSKquUc5B4BE7usBLBX7MgqBQc\n/gPiD0FQOCRlHmF38Hft+PPfnIZhyZ0nW2ZXIPMAKTH2ZRclChF5HHgcoFKlSm4JTiml3C4lEU7u\nhJ1z4PQeKxmknYejGyHtUmNcOXBMEoElITmOurVDmfBr5Utv4wSv6MLDGDMdmA4QFRWl3d0qpbxf\najL8+xts/wj2fmclhbwICoMqHSDhX2gQDX4hEFGX7f/4snHLKR7o1RCAB42h5dA4qlZ99bJD9WSi\niAWucZivaF+mlFKFz7mTcGwz7P4GNk3MuWxYbQgIhZr3QEhZCL0GgsOh9LXgH5LtJomJKbz22ire\nfvs3fH2Fps0qUaNGGCJClSqlrih0TyaKBUB/EZmDVYkdp/UTSimvl3jMqi84vQeO/wV/vA4m7dLl\nwyLBxw+avwqV2kJA8Twf8ocfdtOv3yL27z8NQO/ejQkPD85lK+e5LFGIyOdAKyBCRGKAEYA/gDFm\nKrAI6IQ1AHsi8LCrYlFKKZcwBrZMh4PLrQrm49m2Bsis9LVQ7Cqo/wTU7gkil3342NgzDBy4hLlz\ntwNQv345pk7tTLNm1+SyZd648qmnnrmsN0A/Vx1fKaXylS0Vjm2FrR/AkfVweu9FTxhlUr4pJMRC\n9a4QURfqPWpdOeSjfv0WMX/+34SE+DNyZCueeqopfn753zzOKyqzlVLKbYyBlAQ4vQ9WvwBnDsCJ\n7blvd8MQKNsIKraA4lf2OGpOUlNt6cngrbduxd/fl3feaU+lSiVddkxNFEqposfY4Pg2SI6D3V9b\nt4z8gmDf97lvG1wGSlaF2j3gqiZQ/sZ8v1LITlxcEsOH/8SuXSdZvPh+RIRatSL46qt7XX5sTRRK\nqcIvPhb++tC6Mji6EU7tdm47H38oVR1uHgXh11n1C1dQp3A5jDF89dV2Bg5czOHDCfj6Cps2/Uej\nRq67aslKE4VSqnAyNvhrJvz46KXL+PhBuRvgzH6rcvnqZtajqSWurIFaftm79yT9+//A4sV7AGjW\nrCJTp3ahfv1ybo1DE4VSqnBISbS6stg2y7qFlF0DtjL1IbKXlQiqtLdaLxdQY8b8xosvriApKZVS\npYJ4661befTR6/Hxce8VDWiiUEp5m7NHIPYX2P4xHPoZzp/JubxvALR7H6570C3h5ZfExBSSklLp\n1as+Y8a0p2zZYh6LRROFUqrgO3sENk+B319xorDA1TdZ3Vpce49VSe0Fjh07y99/n+Dmm63+7IYM\naU6rVlVo0cLzt8E0USilCqaUc7D1fVjxVPbrg8tYCaHqbVC1IxSvCD6+7o0xH9hshhkz/uS555bi\n5+fDzp39CQsLJjDQr0AkCdBEoZQqKE7ugt9fhv0/QPLp7MuERULkfVabBV9/t4bnCn/9dZTo6O/5\n9VerI+127aqRmJhCWFj+db+RHzRRKKU8w9jgyEZY8xrsnZ9z2TsXQPXb3ROXG5w9e56RI1cyduwa\nUlNtlCtXjHff7Uj37tchbn781hmaKJRS7mNLsyqil0bDqb+zL1OxhfWoaqW2UMy9j4G6yz33fMXi\nxXsQgb59oxg1qi2lShXcuhRNFEop1zvyJ+z8HNa/ffE6/2IQXgdajrGSRBEwZEhzjhxJYMqUztx4\nY0VPh5MrTRRKqfyVHAffdrXGXkg5a3Wml50ad8Jts61xFwqx1FQb7733BwcOnGb8+NsAaNWqCuvX\nP+6RNhGXQxOFUip/7JkP8+/MuUy9x6w+kiq1cU9MHrZ2bSxPPPE9mzb9B8DjjzfmuuvKAnhNkgBN\nFEqpK5GWAkufgG0zs19/9yIoXcsaoa0QPKXkrNOnk3jhheVMnboeY6By5ZJMnNgpPUl4G00USqm8\nOXcCVj5rPcaaeOTi9c1fhabD3R9XATFnzl8MHLiYI0fO4ufnwzPPNOPFF1tQrFiAp0O7bJoolFK5\nMwb+/gIW5jAeWfeVRaYyOic//riXI0fO0rz5NUyZ0pl69bz/yS1NFEqpixkb/LMU/ngDYlZmX6b4\n1dB6PFRuV6A713O15ORUYmPjqVatNACjR7fjllsq8X//19Cr6iFyoolCKWVJOw9ftLB6YM1Jk6HW\n+AyS/0NuepufftpPnz4L8fERNm+OJiDAl4iIEB5+uJGnQ8tXmiiUKsoOLIUj62D1sEuXqdgCWr0L\nZRu6fdCegurIkQSefXYpn3yyBYDatSOIiTmTflVR2GiiUKqoOb0XVg2xhgDNTulrodcm8C9Y/Q0V\nBDab4f33NzB06HJOn04iKMiP4cNvYfDg5gQEeF+HhM7SRKFUYWUMnNgGW6ZbQ4D6+MGBJdmXrfco\nlG8KdR/Rq4Yc3HXXFyxYYHU90qFDdSZN6kT16mEejsr1NFEoVZicOwEb34X/1sOBxTmXLdsI2n8I\n5QrX/XRXuvvu2qxdG8v48R259946BbIDP1fQRKFUYeBMq+imL0LZ66FsAyhZ1T1xebkFC/4mJuYM\nffveAMCDDzbg7rsjCQ0N9HBk7qWJQilvYgzEx8D22VYvrHH74dSui8td3Rwq3gI17oLyTdwfp5c7\neDCOAQN+YP78vwkM9KVjxxpUq1YaESlySQI0UShVsBkDp/fAH6Ngx2dgS8m5/I3D4ObX3BNbIZSS\nksaECX8wYsTPnD2bQmhoAK+91obKlYtuOxHQRKFUwWRLhY/qwcmdOZdrNgLKRVlXD0W40Vt+WLMm\nhiee+J4tW6xuSe69tw7jxnWgQoUSHo7M8zRRKFVQpJyFdWOs4UCzU7omRD0H1z0Ivt7bb1BB9eKL\nK9iy5QhVq5Zi4sROdOpU09MhFRiaKJTyJGODnV/AovuyX+8XAo8dgJAybg2rKDDGEB9/nhIlrDqH\niRNvY/bszQwb1oKQkKLT060zNFEo5SlxB+CDbJ4+8g2A5qMgapB2k+Eif/99nL59FyECS5f2QkSo\nVSuCUaPaejq0AkkThVLudmwLzG5w8fKWYyDqGffHU4QkJaXyxhu/8Oabv3L+fBrh4cEcOHCaqlUL\nZ9cb+UUThVLucHof/Pqi1TI66UTmddcPhNbjPBNXEbJ06V769l3Enj0nAXjkkYaMHt2O8PAQD0dW\n8Lk0UYhIR2A84At8YIx5M8v6SsBHQCl7maHGmEWujEkpt4iPha9aW/0qGVv2ZW4eBTe+4N64iiBj\nDL17L2DmzE0A1KlThqlTO3PLLZU9HJn3cFmiEBFfYBLQDogB1onIAmPMdodiw4EvjTFTRKQOsAio\n4qqYlHKZtPOwYiDE7bt0f0oAEXWh8SCo08vqe0m5nIhQpUopgoP9eOmllgwa1KxQd+DnCq58pzYB\n9hhj9gGIyBzgDsAxURjgwkPKJYF/XRiPUvnrzEH47l74b+2ly1TvCs1ehlLVtJ2DG23a9B+HD8dz\n223WI65DhjSnV6/6WhdxmVyZKCoAhxzmY4Abs5R5GfhRRJ4EigG3ZrcjEXkceBygUqVK+R6oUnly\n8m/4ugOc+Sf79S3HQFhtqNpJe2J1s/j4ZEaM+Jnx4/8gPDyYnTv7ExYWTGCgnyaJK+Dpa9+ewCxj\nzDsi0gz4WETqGpP5pq4xZjowHSAqKsp4IE6lIDUZxgddvLx2T7jpFatBnPIIYwzffruTAQMWExNz\nBh8f4b776uHvr48X5wdXJopY4BqH+Yr2ZY56Ax0BjDG/i0gQEAEcdWFcSuXN6X0wu77VctpRpbbQ\n9RsI1C4ePOmff07Tv/8PfP+91TliVNTVTJvWheuvL+/hyAoPVyaKdUBNEamKlSB6AFmbnx4E2gKz\nRCQSCAKOuTAmpZxnDHzaBI6sz7w8qDT0Pa6N4QoAYwzdun3Jhg2HKVEikNdfb0N0dBS+vvq3yU8u\nSxTGmFQR6Q8swXr0dYYxZpuIjATWG2MWAM8A74vI01gV2w8ZY/TWkvIcY+CvmfBj74vX1XkQOswA\nH31ixtNsNoOPjyAijBnTnqlT1zNuXAfKlw/1dGiFknjb53JUVJRZv3597gWVyos9C2D+HdmvC60E\nvfeAr/b/42knTiQydOgyAN5/v6uHo/EuIrLBGBN1Odt6ujJbKc+6VHcaAE2Hw00j9cmlAsAYw+zZ\nm3n22aUcP55IQIAvI0a0omJFrR9yB00UquiJOwC75sIvQ8GkZV7XdhI06KPJoQDZseMYffosZOVK\n63HkVq2qMGVKZ00SbqSJQhUNMb/A7m9g47vZr+8wA+o+7N6YVI6MMbz00greeutXUlJsRESE8M47\n7enVqz6iidytNFGowintPBz9E+bccunhQyu1gdK1rAZy/toxXEEjIsTGxpOSYuOxx67nzTdvJSws\n2NNhFUmaKFThkXQafh4I2z66dJkGfaF2d6jYwn1xKaf9+288x48nUr9+OQBGj25H796NaN5ce2Tw\nJE0Uyvvt/Q6+zeEJmNo9oeNH+tRSAZaWZmPKlPUMG/YTFSqEsmlTNAEBvkREhBARoUnC0zRRKO9l\nbPDTU7BpYublJatBhw/hmlYeCUvlzcaNh3niie9Zv97qE7RFi8qcOZNMRITeDiwonEoUIhIAVDLG\n7HFxPErlLj4GZkZCSkLm5Xcvgqq3eSYmlWdnziTz4os/MXHiOmw2Q8WKJZgwoSN33llbK6sLmFwT\nhYh0BsYCAUBVEWkIjDDG3OXq4JRKZ0uz2jx8esPFj7T6BUHP36FsQ8/EpvLMGEOLFjPZvPkIvr7C\noEFNefnlVoSGBno6NJUNZ64oRmJ1D74CwBizSURquDQqpS44vg2W94WYVRevi7wf2k7WTvm8kIjw\n9NNNmTx5PdOmdaFhw6s8HZLKgTOJIsUYczrLpaB39fuhvEtO3WkUvxrCr4Nui7VTPi9y/nwaY8f+\njq+vMHhwcwAefLABDzxQXzvw8wLOJIodIvI/wMfeE+wAYI1rw1JFUsJhmHZ19usa9oNmIyCkjHtj\nUlfsl1/+ITp6Idu3HyMw0JcHH2xAuXLFERF8fbUuwhs4kyj6Ay8BNuAbrN5gdUR4lX/OHISv2sDp\nvZmXt/8AIh8AP71v7Y2OH0/kueeWMnPmJgBq1gxj8uTOlCtX3MORqbxyJlF0MMYMAYZcWCAid2Ml\nDaUuj7HByZ0w67qL1zUdDs1fdX9MKl8YY5g1axODBy/lxIlzBAT48vzzNzN06M0EBekT+d7Imb/a\ncC5OCsOyWaaUc/6cBD/1v3h5RF34388QHO72kFT++uSTrZw4cY42baoyeXInatWK8HRI6gpcMlGI\nSAesYUoriMhYh1UlsG5DKeU8Y4NNk+GnJy9ed3Vz6Lna/TGpfJOYmEJcXBLly4ciIkye3Il16/7l\n/vvraZuIQiCnK4qjwF9AErDNYXk8MNSVQalCZnZDOLb54uX3rYHyN7o/HpWvfvhhN/36LaJatdIs\nXdoLEaFWrQi9iihELpkojDF/An+KyKfGmCQ3xqQKC1sqjMumf6Xb58K13dwfj8pXsbFnGDhwCXPn\nbgcgNDSQEyfOadcbhZAzdRQVRGQUUAcIurDQGHOty6JS3u/7HvD3F5mXDbLpgECFQFqajUmT1jF8\n+E/Ex5+nWDF/Ro5szYABN+Lnp20iCiNnEsUs4DVgDHAb8DDa4E7lJGuS8PGHp897Lh6Vb2w2Q8uW\ns/j110MA3HlnbcaP70ilSiU9HJlyJWcSRYgxZomIjDHG7AWGi8h64EUXx6a8TWoSfN0RYlZmLBuU\npi2oCxEfH6F9++ocPBjHxImd6Nq1lqdDUm7gTKJIFhEfYK+IRAOxQKhrw1JeZ9tsWPx/mZdpkvB6\nxhi+/HIbfn4+dOtWB4AhQ5ozaFAzihcP8HB0yl2cSRRPA8Wwuu4YBZQEHnFlUMpL2NJgXDZvoXKN\n4d6fNEl4ub17T9K37yJ+/HEvZcqE0KZNVUqXDiYw0I9AbSxfpOSaKIwxf9gn44FeACJSwZVBKS+Q\nHAcTS128vOs3UFN7oPdmycmpvP32b4wa9QtJSamULh3EqFFtKFkyKPeNVaGUY6IQkRuACsBqY8xx\nEbkOqyuPNkBFN8SnCpp/lsHcdhcvH3hehxotBH7++QB9+ixk587jAPTqVZ8xY9pTtmwxD0emPCmn\nltlvAN2AzVgV2N8DfYG3gGj3hKcKlHeyebS1Ulu4d5n7Y1H5Li3NRt++VpKoVSucKVM607p1VU+H\npQqAnK4o7gAaGGPOiUgYcAioZ4zZ557QVIGRXZuIm16BG4Zoz65ezmYzJCWlEhLij6+vD1OmdGbV\nqn947rnmBAZqB37KktM7IckYcw7AGHNSRHZpkihiUs7ChGy6hNaGc4XC1q1HiI5eSO3a4Xz4oTVQ\nVMuWVWjZsopnA1MFTk6JopqIXOghVrDGy07vMdYYc7dLI1OedXIXzMzyjHz/OB12tBA4e/Y8I0eu\nZOzYNaSm2ti//xSnTp2jdOlgT4emCqicEkXWzngmujIQVUDE7YcPqmVeVudBuO0jz8Sj8tV33/1N\n//4/cPBgHCLQt28Uo0a1pVQpfaJJXVpOnQIud2cgqgBY+gRsmZ55WdvJ0LCPZ+JR+SY11Ub37nP5\n5psdADRseBXTpnWhSRN90l3lTmurFBgDX7SE2F8yltV7FG6dCj6+notL5Rs/Px9KlgykePEAXn21\nNf37N9EO/JTTxBjX9e8nIh2B8YAv8IEx5s1syvwPeBmro8HNxpj7ctpnVFSUWb9+vQuiLcI+bw7/\n/pYxr3URhcIff8QAcOONVpOnEycSOXculYoV9W9bFInIBmNM1OVs6/QVhYgEGmOS81DeF5gEtANi\ngHUissAYs92hTE3geaC5MeaUiJR1PnSVLxbco0mikDl9Oonnn1/GtGkbqF07gk2bogkI8CU8XMeJ\nUJcn12tPEWkiIluB3fb5BiLynhP7bgLsMcbsM8acB+Zgtc1w9BgwyRhzCsAYczRP0avLt+5tqwHd\n7q8zlg08r0nCixlj+OyzrdSuPZGpUzfg6+tD1661SEvTkYvVlXHmimIC0AX4FsAYs1lEWjuxXQWs\nRnoXxABZx728FkBEfsW6PfWyMWaxE/tWl8uWCp/eCEc3Zl7e97h2weHFdu8+Qd++i1i2zGrq1Lz5\nNUyd2oW6dfUiXV05ZxKFjzHmnywDpKfl4/FrAq2w+o5aJSL1jDGnHQuJyOPA4wCVKlXKp0MXQaf2\nwIyamZc9shtK1/BMPCpfpKSk0abNbGJizhAWFszo0bfy8MON8PHRRpEqfziTKA6JSBPA2OsdngR2\nObFdLHCNw3xF+zJHMcAfxpgUYL+I7MJKHOscCxljpgPTwarMduLYKqukU5mTRPGK8Nh+8NEH37yV\nMQYRwd/fl1Gj2rBixQFGj76VMmW0Az+Vv5x5Pq4PMAioBBwBmtqX5WYdUFNEqopIANADWJClzLdY\nVxOISATWrSjtJsQVJoVlTN/0CjxxSJOElzpyJIFevebx2mur0pc9+GADZs68Q5OEcglnPilSjTE9\n8rpjY0yqiPQHlmDVP8wwxmwTkZHAemPMAvu69iKyHet21mBjzIm8HkvlIOFfmObQqKpUDWj2kufi\nUZfNZjO8//4Ghg5dzunTSZQqFcTAgU0JDdWOGZVr5dqOQkT2An8DXwDfGGPi3RHYpWg7ijxISYQJ\nWb5hDkwGXx3C0tts3vwf0dELWbPGahvRsWMNJk3qRLVqpT0cmfIWV9KOItdbT8aY6sBrQGNgq4h8\nKyJ5vsJQHjDDoVO/ur2tXl81SXiVlJQ0nn32Rxo3ns6aNTGUL1+cL7+8h0WL7tMkodzGqTb8xpjf\njDEDgOv3F54dAAAgAElEQVSBM8CnLo1KXbl9CyHB+vZJmfrQ4QPtGtwL+fn58Oef/2GzGZ58sgk7\ndvTj3nuvQ/Rvqdwo1zoKESmO1VCuBxAJzAducnFc6nKlpcCyaPhrRsay//3ssXBU3h08GEdamo2q\nVUsjIkyd2pm4uGSioq72dGiqiHKmMvsv4DtgtDHml9wKKw+a1xX2fZd5WesJEKS3KLxBSkoa48f/\nwYgRP9OsWUWWLu2FiFCzZrinQ1NFnDOJopoxRvsAKOg+bnxxa+v7/oDyTTwTj8qT338/RHT0QrZs\nOQJAWFgwiYkpFCumdUrK8y6ZKETkHWPMM8DXInLRo1E6wl0BYQx8UBXO/JOxbEAC+Ovz9N7g1Klz\nDB26jOnTrSRftWopJk3qxG231cxlS6XcJ6crii/sv3Vku4Js3ejMSaLPMU0SXiI5OZWGDadx8GAc\n/v4+DB58E8OGtSAkRPvcUgVLTiPcrbVPRhpjMiULe0M6HQHP034eBBvGZcw/GQ8BxT0Xj8qTwEA/\nevduxPLl+5kypTN16pTxdEhKZcuZBncbjTHXZ1n2pzGmkUsjuwRtcGe3cQKseCpj/t6foJIznfoq\nT0lKSuWNN36hVq0I7ruvHmANUerrK/q4q3I5lwxcJCLdsR6JrSoi3zisCgVOZ7+VcousI9I9uAXK\n1PNcPCpXS5fupW/fRezZc5KyZYtx1121CQ721+FIlVfIqY5iLXACq9fXSQ7L44E/XRmUysG4ALCl\nZMx3X6VJogD7778EBg1awuef/wXAddeVYerULgQHaz2E8h451VHsB/YDy9wXjsrRxvGZk8TTqeDj\n67l41CWlpdmYNm0DL7ywnLi4ZIKD/RgxoiVPP92MgAD9mynvktOtp5XGmJYicgpwrMgQwBhjwi6x\nqXKFpNOwYmDG/DM6LEdBlpZmeO+9tcTFJdOpU00mTryNqlW14aPyTjnderpQMxrhjkBUDv6aBUse\nzpjv9InHQlGXFh+fTFqaoVSpIAICfHn//ds5ciSBu++O1Mpq5dVyuvV0oTX2NcC/xpjzInIzUB/4\nBKtzQOVKxsDYLJWdYbUh8n7PxKOyZYxh3rydDBjwAx06VOfDD+8A4OabddheVTg488jFt1jDoFYH\nZmINVfqZS6NSsPOLi5NEp0/h4R2eiUdl68CB03TtOodu3b4kNjaev/46RlJSqqfDUipfOdPXk80Y\nkyIidwPvGWMmiIg+9eQqaefh3WxGLNM6iQIlJSWNsWN/55VXVnLuXColSgTy+uttiI6OwtdXH3lV\nhYtTQ6GKyL1AL+BO+zJ9ts8Vfn8VfssyTOltsyHyAc/Eo7KVmJhC06YfsHXrUQB69KjL2LHtKV8+\n1MORKeUaziSKR4C+WN2M7xORqsDnrg2rCJrdCI5typgPrQSP7QfRb6cFTUiIP1FRV5OYmMLkyZ1p\n3766p0NSyqVy7cIDQET8gBr22T3GGI/dhC10XXgkx8HEUpmXPboPSlb1TDzqIsYYZs/eTPXqYekV\n1HFxSQQE+GrDOeU1XNKFh8PObwE+BmKx2lBcJSK9jDG/Xs4BlYOUxIuTxFNJ4JdNHYXyiB07jtGn\nz0JWrvyHyMgINm2KJiDAl5IlgzwdmlJu48ytp3FAJ2PMdgARicRKHJeVmZSdLQ0mOHQHXqkt3KuN\n4AuKc+dSGDXqF0aP/pWUFBtlyoTw/PM34++vtwJV0eNMogi4kCQAjDE7RESH3bpS4xxOfeg1miQK\nkMWL99Cv3yL27TsFwGOPXc+bb95KWFiwhyNTyjOcSRQbRWQqViM7gPvRTgEvn7HBWIe+fgJLweMH\nPRePyiQh4Ty9es3j+PFE6tYty9SpnWneXBvOqaLNmUQRDQwAnrPP/wK857KICruxWTqE63/KM3Go\ndGlpNmw2g7+/L8WLBzB+fEdiYs7w9NNN8ffXDvyUyjFRiEg9oDowzxgz2j0hFWLvZOnvZ5At+3LK\nbTZs+JcnnvieO+6oxYsvtgRIH1RIKWW5ZM2ciLyA1X3H/cBSEXnEbVEVRltnZJ4fZAPtKM5jzpxJ\n5qmnfqBJkw/YsOEwH3+8hZSUNE+HpVSBlNMVxf1AfWPMWREpAywCZuRQXl3K8b/gx94Z89odh8cY\nY5g7dztPPbWYw4cT8PUVBg1qyiuvtNbbTEpdQk6JItkYcxbAGHNMRJsI55kx8F4JSEnIWNZGq3c8\nJT4+me7d5/LDD3sAuPHGCkyd2oWGDa/ycGRKFWw5JYpqDmNlC1DdcexsY8zdLo3M22XXRXjrCdCo\nv2fiURQvHkBycholSwby5pu38vjjjfHx0dt/SuUmp0TRLcv8RFcGUugs65N5fuB58NXuHtxt1ap/\nKF++ODVrhiMizJjRlaAgP8qVK+7p0JTyGjkNXLTcnYEUKkmnYMu0jHmtk3C748cTee65pcycuYm2\nbauydGkvRITKlUvlvrFSKhNn2lGovJrkMJz4fX94Lo4iyGYzzJq1icGDl3Ly5DkCAny55ZZKpKUZ\n/Pz0NpNSl8OlFdQi0lFE/haRPSIyNIdy3UTEiIj39x91anfGdL3HoHwTz8VSxGzbdpRWrWbRu/cC\nTp48R9u2Vdm6tQ8jRrTCz0+fxVDqcjl9RSEigcaY5DyU9wUmAe2AGGCdiCxw7DfKXi4UeArw/q/e\naedhxrUZ8+2ney6WIiYuLommTT8kIeE8ZcsWY+zY9tx3Xz1E26oodcVy/ZolIk1EZCuw2z7fQESc\necazCdbYFfuMMeeBOcAd2ZR7FXgLSHI+7AJox2eZhzBt8rznYilCLoynUrJkEEOGNCc6ujE7d/bj\n/vvra5JQKp84cz0+AegCnAAwxmwGWjuxXQXgkMN8jH1ZOhG5HrjGGLMwpx2JyOMisl5E1h87dsyJ\nQ7vZpsmw6P6M+WJXwS2vey6eIiA29gz33PMln3yyJX3ZsGG3MGVKF0qX1l5elcpPziQKH2PMP1mW\nXXFfB/YGfGOBZ3Ira4yZboyJMsZElSlT5koPnb+MgeX9MuY7zITow56Lp5BLTbUxfvwaateexNdf\n72DEiJ9JS7P6zNIrCKVcw5k6ikMi0gQw9nqHJ4FdTmwXC1zjMF/RvuyCUKAu8LP9H/wqYIGIdDXG\neMdYp4dWwpetMubbvQ91H/JUNIXeunWxREcvZONGKxHfeWdtJkzoiK+vVlQr5UrOJIo+WLefKgFH\ngGX2ZblZB9QUkapYCaIHcN+FlcaYOCDiwryI/Aw86zVJAjInCYD6j3okjMLu7NnzDBmyjMmT12EM\nVKpUkvfeu42uXWt5OjSlioRcE4Ux5ijWh3yeGGNSRaQ/sATwBWYYY7aJyEhgvTFmQZ6jLShsaZlH\nqLtzAVS/3XPxFHJ+fj4sW7YPHx9h0KBmjBjRkmLFdJBFpdwl10QhIu8DFzUtNsY8ntu2xphFWL3O\nOi576RJlW+W2vwLjpwGZ5zVJ5Lu9e09SqlQQ4eEhBAb68fHHdxEU5Ee9euU8HZpSRY4zN3eXAcvt\nP78CZQGn21MUOufjYfPkjPmB5z0XSyGUnJzKa6+tom7dKQwZkjGO+A03VNAkoZSHOHPr6QvHeRH5\nGFjtsogKuvdKZEzfNls7+stHP/98gD59FrJz53HAesIpLc2mldVKedjl9PVUFSh6X+2MLfN41xH1\nIPIBz8VTiBw9epbBg5cye/ZmAGrVCmfKlM60bl3Vw5EppcC5OopTZNRR+AAngUv221Rojc0y+tn/\nbcm+nMqT48cTiYycxMmT5wgM9GXYsFt47rnmBAZqf5VKFRQ5/jeK1cChARntH2zmQp8JRcnZIxnT\nQaWh7wnPxVLIRESEcMcdtYiJOcPkyZ2pUSMs942UUm6VY6IwxhgRWWSMqeuugAqkqQ5DZfY76bk4\nCoGzZ88zcuRKOne+lhYtKgMweXJnAgN9tWW1UgWUM7WEm0SkkcsjKahif/V0BIXGd9/9TZ06kxk9\n+jf69l2IzWZdnAYF+WmSUKoAu+QVhYj4GWNSgUZYXYTvBc5ijZ9tjDHXuylGz5pzc8Z03+Oei8OL\nHToUx1NPLWbevJ0ANGp0FdOmddHxqpXyEjndeloLXA90dVMsBYsxMNbhguu2jyE43HPxeKHUVBsT\nJvzBSy+t4OzZFIoXD+C111rTr18THUhIKS+SU6IQAGPMXjfFUnCcOQTvV8q8rI4+CptXZ84k88Yb\nqzl7NoVu3SJ5992OVKxYIvcNlVIFSk6JooyIDLrUSmPMWBfEUzBkTRLPFL0HvS7X6dNJBAf7ERjo\nR1hYMNOmdSEw0JfOna/NfWOlVIGU0/W/L1Acqzvw7H4Kp7P/ZUyXqKxJwknGGD77bCu1ak1k9OiM\nBwDuvjtSk4RSXi6nK4rDxpiRboukIDA2mFo+Y/6R3Z6LxYvs2nWCvn0Xsnz5fgBWrTqIMUafZFKq\nkMi1jqLIMCZz6+sWo7Ufp1wkJaXy1luref311Zw/n0ZYWDBvv92Ohx5qqElCqUIkp0TR1m1RFASL\nH8o8f8Ngj4ThLf77L4EWLWaye7fVAPGhhxry9tvtiIgI8XBkSqn8dslEYYwpOk2Q//0dts/OmB90\nxUOCF3rlyhXjmmtK4ufnw5QpnWnZsoqnQ1JKuYj2vAbwZeuM6Ud2g+gz/lnZbIb3399A69ZVufba\ncESEzz67m9KlgwkI8M19B0opr6WfiOfjIc0+DlOToVC6hmfjKYA2b/6P5s1nEB29kL59F3KhX8hy\n5YprklCqCNArCseBiJpmO0prkZWQcJ6XX/6Zd99dQ1qa4eqrQ4mOjvJ0WEopNyvaiSJuf8Z0zW7g\nH+y5WAqYb7/dyZNP/kBMzBl8fIQnn2zCa6+1oUSJQE+HppRys6KdKNa+lTHdda7n4ihgYmPP0KPH\nXJKT02jcuDxTp3YhKupqT4ellPKQopso4mNhyzRPR1FgpKSk4efng4hQoUIJRo1qQ0CAL3373qBj\nVitVxBXNT4AjG2F6xYz57is9F0sB8Ntvh2jceDqffJIxvOszz9zEk0/eqElCKVVEE8UnjTOm238I\nFVt4LhYPOnnyHE888R3Nm89g69ajTJ68nqI40q1SKmdF79bT4oczpjt+BNc96LlYPMQYwyefbOGZ\nZ37k2LFE/P19eO655gwbdot2vaGUukjRShSH/4BtszLmi2CSOHIkgZ49v2bFigMAtGxZmSlTOhMZ\nWcazgSmlCqyilSg+a5oxPeCs5+LwoFKlgjh8OIGIiBDGjGnHgw820KsIpVSOik6iSE3KmG41DvyL\nTud1S5fu5frryxMeHkJgoB9ffXUv5csXJzy86JwDpdTlKzqV2ZNKZ0xf/5Tn4nCjw4fj6dnza9q3\n/4QhQ5alL69bt6wmCaWU04rGFcXpfRlXFOWbQSG/1ZKWZmPatA08//xyzpxJJjjYj1q1wnUwIaXU\nZSn8icKWBh9Wz5jv+eulyxYCGzceJjr6e9at+xeAzp1rMnFiJ6pUKeXhyJRS3qpwJ4qURJhQLGO+\nWpdCfTVx4MBpmjR5n7Q0Q4UKoUyYcBt33VVbryKUUlfEpYlCRDoC4wFf4ANjzJtZ1g8CHgVSgWPA\nI8aYf/ItgD3fZkyXrgl3fZdvuy6IqlQpxcMPNyQ0NJBXXmlFaKh24KeUunIuq8wWEV9gEnAbUAfo\nKSJ1shT7E4gyxtQH5gKj8zWI5f2s38ER8MiufN11QXDgwGluv/1zVq48kL5s+vTbGTu2gyYJpVS+\nceUVRRNgjzFmH4CIzAHuALZfKGCMWeFQfg3wQL4d/fReSD5tTUc9m2+7LQhSUtIYO/Z3XnllJefO\npXL8eCK//94bQG8zKaXynSsTRQXgkMN8DHBjDuV7Az9kt0JEHgceB6hUqZJzR/+ofsZ0owHObeMF\nVq8+SHT092zbdgyAHj3qMnZsew9HpZQqzApEZbaIPABEAS2zW2+MmQ5MB4iKinKu17rUROu3X1Ch\nGJDo1KlzDB68lA8//BOA6tVLM3lyZ9q3r57LlkopdWVcmShigWsc5ival2UiIrcCw4CWxpjkfI+i\n67x836Un2GyG+fP/xt/fh6FDb+b5528mONjf02EppYoAVyaKdUBNEamKlSB6APc5FhCRRsA0oKMx\n5mi+HXnfwozpCs3zbbfutnPncapWLUVgoB/h4SF8+undVKpUktq1IzwdmlKqCHHZU0/GmFSgP7AE\n2AF8aYzZJiIjRaSrvdjbQHHgKxHZJCIL8uXg87pkTAeE5ssu3SkxMYVhw5ZTv/4URo/OaCDYvn11\nTRJKKbdzaR2FMWYRsCjLspccpm/N94M6dv7XdnK+797VFi/eQ9++C9m/33pi6/jxRA9HpJQq6gpE\nZXa+2vVVxnT9xz0XRx79+288Awcu5quvrKeH69Ury9SpXbjppmty2VIppVyr8CWKHxwGI/Lx9Vwc\nebBr1wmioqYTH3+ekBB/Xn65JQMHNsXf3zviV0oVboUrUTiO93zdQx4LI69q1gzjhhsqUKyYP++9\ndxuVK2sHfkqpgqNwJYp/f8uYbv++5+LIxZkzybz00gr69r2Ba68NR0RYsKAHxYoFeDo0pZS6SOFK\nFDErM6Z9Ct5LM8Ywd+52nnpqMYcPJ7Bz53EWL7Z6LdEkoZQqqArep+mVWD3M+l0+p55CPGPfvlP0\n77+IH37YA0DTphV56638f+hLKaXyW+FJFNs/zpiuXHD6Pjp/Po0xY37j1VdXkZSUSqlSQbz5Zlse\ne6wxPj7agZ9SquArHInClpb5aadmIzwXSxaHDsUxcuRKkpPTuP/+erzzTnvKlSvu6bCUUspphSNR\nnNyRMX3vco8/Fnvq1DlKlQpCRKhePYzx4ztSo0YYbdtW82hcSil1OVzWhYdbnbEPihcUDpXaeCwM\nm80wY8af1KjxHp98siV9+RNPRGmSUEp5rcKRKH619wqS6rnuLrZtO0qrVrPo3XsBJ0+eS6+0Vkop\nb+f9t55saXB0ozXdsL/bD5+YmMKrr65kzJjfSU21UbZsMcaN60DPnnXdHotSSrmC9yeKL1tnTN/w\nnFsPvWvXCTp0+IQDB04jAtHRjXn99baULu39AyUppdQF3p0oDq+F2F+sab9gCHFvF9yVK5ckKMiP\nBg3KMXVqF5o2rejW46uCLSUlhZiYGJKSknIvrFQ+CQoKomLFivj759/AZt6dKL7tmjHd97jLD5ea\namPq1PX07FmX8PAQAgP9WLz4fipUKIGfX+Go7lH5JyYmhtDQUKpUqYKItplRrmeM4cSJE8TExFC1\natV826/3froZGyQesaYr3Qr+IS493Nq1sTRp8j5PPvkDQ4YsS19euXIpTRIqW0lJSYSHh2uSUG4j\nIoSHh+f7Vaz3XlEc2Zgx3fVrlx0mLi6JYcN+YvLkdRgDlSqV5I47arnseKpw0SSh3M0V7znvTRTL\nojOmA0vk++6NMXzxxTaefnoJ//2XgJ+fD4MGNeWll1pqB35KqSLFe++ZHNng0t1v3nyEnj2/5r//\nErjppmvYuPFx3nqrnSYJ5VV8fX1p2LAhdevW5fbbb+f06dPp67Zt20abNm2oVasWNWvW5NVXX8U4\njOnyww8/EBUVRZ06dWjUqBHPPPOMJ15Cjv7880969+7t6TBy9MYbb1CjRg1q1arFkiVLsi1zyy23\n0LBhQxo2bMjVV1/NnXfeCcD8+fOpX78+DRs2JCoqitWrVwNw7NgxOnbs6LbXgDHGq34aN25sTNJp\nY8Zg/cSsNvklNTUt0/zTTy8277+/waSl2fLtGKro2L59u6dDMMWKFUuffvDBB81rr71mjDEmMTHR\nVKtWzSxZssQYY8zZs2dNx44dzcSJE40xxmzdutVUq1bN7NixwxhjTGpqqpk8eXK+xpaSknLF+7jn\nnnvMpk2b3HrMvNi2bZupX7++SUpKMvv27TPVqlUzqampOW5z9913m48++sgYY0x8fLyx2azPn82b\nN5tatWqll3vooYfM6tXZf/5l994D1pvL/Nz1zltPF7rsAKjQPF92uWLFfvr2XcS0aV1o0aIyAGPH\ndsiXfSvFOy6qq3jG5F7GrlmzZmzZYnUt89lnn9G8eXPat7d6Wg4JCWHixIm0atWKfv36MXr0aIYN\nG0bt2rUB68qkT58+F+0zISGBJ598kvXr1yMijBgxgm7dulG8eHESEhIAmDt3Lt9//z2zZs3ioYce\nIigoiD///JPmzZvzzTffsGnTJkqVskZ1rFmzJqtXr8bHx4fo6GgOHjwIwLvvvkvz5pn/1+Pj49my\nZQsNGjQAYO3atTz11FMkJSURHBzMzJkzqVWrFrNmzeKbb74hISGBtLQ0Vq5cydtvv82XX35JcnIy\nd911F6+88goAd955J4cOHSIpKYmnnnqKxx9/3Onzm5358+fTo0cPAgMDqVq1KjVq1GDt2rU0a9Ys\n2/Jnzpzhp59+YubMmQAUL57RgejZs2cz1T/ceeedfPrppxedF1fwzkTxVf6N43D06FkGD17K7Nmb\nARg79vf0RKFUYZGWlsby5cvTb9Ns27aNxo0bZypTvXp1EhISOHPmDH/99ZdTt5peffVVSpYsydat\nWwE4depUrtvExMTw22+/4evrS1paGvPmzePhhx/mjz/+oHLlypQrV4777ruPp59+mptvvpmDBw/S\noUMHduzYkWk/69evp27djB4QateuzS+//IKfnx/Lli3jhRde4OuvrQddNm7cyJYtWwgLC+PHH39k\n9+7drF27FmMMXbt2ZdWqVbRo0YIZM2YQFhbGuXPnuOGGG+jWrRvh4eGZjvv000+zYsWKi15Xjx49\nGDp0aKZlsbGxNG3aNH2+YsWKxMbGXvLcfPvtt7Rt25YSJTLqXefNm8fzzz/P0aNHWbhwYfryqKgo\nhg8fntOpzjfemSjygc1m+PDDjQwZsoxTp5IIDPRl+PAWDB58k6dDU4VRHr7556dz587RsGFDYmNj\niYyMpF27dvm6/2XLljFnzpz0+dKlS+e6zb333ouvr9XDc/fu3Rk5ciQPP/wwc+bMoXv37un73b59\ne/o2Z86cISEhIdM37MOHD1OmTJn0+bi4OP7v//6P3bt3IyKkpKSkr2vXrh1hYWEA/Pjjj/z44480\natQIsK6Kdu/eTYsWLZgwYQLz5s0D4NChQ+zevfuiRDFu3DjnTs5l+Pzzz3n00UczLbvrrru46667\nWLVqFS+++CLLllmP55ctW5Z///3XZbE48s5Ece6Y9fsyH4vdv/8UDzwwj99+OwRA+/bVmTSpEzVq\nhOVXhEoVCMHBwWzatInExEQ6dOjApEmTGDBgAHXq1GHVqlWZyu7bt4/ixYtTokQJrrvuOjZs2JB+\nWyevHG+RZH2mv1ixYunTzZo1Y8+ePRw7doxvv/02/RuyzWZjzZo1BAUF5fjaHPf94osv0rp1a+bN\nm8eBAwdo1apVtsc0xvD888/zxBNPZNrfzz//zLJly/j9998JCQmhVatW2bZHyMsVRYUKFTh06FD6\nfExMDBUqVMj29Rw/fpy1a9emJ6qsWrRowb59+zh+/DgRERHpt9jcwfueejK2jOkyDS9rFyVKBLJr\n1wmuuqo4c+Z0Y/Hi+zVJqEItJCSECRMm8M4775Camsr999/P6tWr07+dnjt3jgEDBvDcc1Z/aYMH\nD+b1119n165dgPXBPXXq1Iv2265dOyZNmpQ+f+HWU7ly5dixYwc2m+2SH3xgJZS77rqLQYMGERkZ\nmf7tvX379rz33nvp5TZt2nTRtpGRkezZk9FLc1xcXPqH8KxZsy55zA4dOjBjxoz0OpTY2FiOHj1K\nXFwcpUuXJiQkhJ07d7JmzZpstx83bhybNm266CdrkgDo2rUrc+bMITk5mf3797N7926aNGmS7X7n\nzp1Lly5dMiXHPXv2pD+JtnHjRpKTk9PP0a5duzLdenMl70sUKQkZ0yWdb6K+ZMkekpNTAQgPD2HB\ngh7s3NmP7t3raqMoVSQ0atSI+vXr8/nnnxMcHMz8+fN57bXXqFWrFvXq1eOGG26gf3+rB+b69evz\n7rvv0rNnTyIjI6lbty779u27aJ/Dhw/n1KlT1K1blwYNGqR/037zzTfp0qULN910E+XLl88xru7d\nu/PJJ5+k33YCmDBhAuvXr6d+/frUqVMn2yRVu3Zt4uLiiI+PB+C5557j+eefp1GjRqSmpl7yeO3b\nt+e+++6jWbNm1KtXj3vuuYf4+Hg6duxIamoqkZGRDB06NFPdwuW67rrr+N///kedOnXo2LEjkyZN\nSr/t1qlTp0y3jubMmUPPnj0zbf/1119Tt25dGjZsSL9+/fjiiy/SP69WrFhB586drzhGZ8iFbOUt\noq4tY9Y/cRxCK8Hj/+Ra/tChOAYMWMy33+7k1VdbM3x4CzdEqRTs2LGDyMhIT4dRqI0bN47Q0NCL\n7usXBS1atGD+/PnZ1gtl994TkQ3GmKjLOZb3XVHY7N8UcunbKTXVxtixvxMZOYlvv91J8eIBhIVp\n999KFSZ9+vQhMDDQ02G43bFjxxg0aJBTDw/kB++rzE62tyyteOkrgzVrYoiO/p7Nm61OA7t1i2T8\n+I5UqJD/XX0opTwnKCiIXr16eToMtytTpkx662138L5EccG192a7+I8/Yrjppg8xBqpUKcXEibfR\nufO1bg5OKYsxRuvAlFu5ojrBexNF5ewb3TVpUoEOHWrQqNFVDB/egpCQ/Bu8Q6m8CAoK4sSJE9rV\nuHIbYx+PIqfHii+HdyaKqp3SJ3fvPsHTTy9h7NgOXHut9Q+5cOF9+PjoP6byrIoVKxITE8OxY8c8\nHYoqQi6McJefvDNRlGtMcnIqb765mjfeWE1ychpBQX7Mnfs/AE0SqkDw9/fP11HGlPIUlz71JCId\nReRvEdkjIhe1RhGRQBH5wr7+DxGp4sx+l++uSv36U3n55ZUkJ6fx8MMNmTq1S36Hr5RSChdeUYiI\nLzAJaAfEAOtEZIExZrtDsd7AKWNMDRHpAbwFdL94bxn2nyzFrQ9bPUpGRkYwdWoX7cRPKaVcyJVX\nFOr0+qwAAAgISURBVE2APcaYfcaY88Ac4I4sZe4APrJPzwXaSi61fqcSgwkK8uX119uwaVO0Jgml\nlHIxl7XMFpF7gI7GmEft872AG40x/R3K/GUvE2Of32svczzLvh4HLnQMXxf4yyVBe58I4HiupYoG\nPRcZ9Fxk0HORoZYxJvRyNvSKymxjzHRgOoCIrL/cZuiFjZ6LDHouMui5yKDnIoOIrL/cbV156ykW\nuMZhvqJ9WbZlRMQPKAmccGFMSiml8siViWIdUFNEqopIANADWJClzALg/+zT9wA/GW/rpVAppQo5\nl916Msakikh/YAngC8wwxmwTkZFYg3wvAD4EPhaRPcBJrGSSm+muitkL6bnIoOcig56LDHouMlz2\nufC6bsaVUkq5l/d1M66UUsqtNFEopZTKUYFNFK7q/sMbOXEuBonIdhHZIiLLRaTQtkLM7Vw4lOsm\nIkZE/r+9e42VqjrDOP5/tKJYDVWJjZe2R+MVFZCiQU20FCWWppgaAhLQYtRWbG28fmiw0ab9YGJt\nUkr1eGuQRLSiRQnSqjFHseQcFS+AtVZbJJZoxA9IjKJp8emHtY5MT+fM7HMKe27vL5lkZs++vPNm\nZq9Za8+8q21/GlkkF5Jm5vfGXyQtLTvGshT4jHxVUo+kl/PnZFq1/bQ6Sb+TtCX/R63a85K0MOdp\nvaQJhXZsu+lupIvf/wCOBEYA64AxA9a5AujO9y8Aft/ouBuYi8nAvvn+/E7ORV5vf2A10AdMbHTc\nDXxfHA28DByQHx/c6LgbmIs7gfn5/hhgU6Pj3k25OBOYALw6yPPTgD8CAiYBzxXZb7P2KHZL+Y8W\nVTcXtntsf5wf9pH+s9KOirwvAH5Oqhv2SZnBlaxILi4Dfmt7K4DtLSXHWJYiuTDQP8XlKOCdEuMr\nje3VpF+QDuY8YImTPuBLkg6pt99mbSgOA/5Z8XhzXlZ1Hdv/BrYBB5USXbmK5KLSJaRvDO2obi5y\nV/orth8rM7AGKPK+OAY4RtIaSX2Szi0tunIVycVNwFxJm4FVwJXlhNZ0hno+AVqkhEcoRtJcYCJw\nVqNjaQRJewC/AuY1OJRm8QXS8NM3SL3M1ZJOsv1BQ6NqjNnAYtu3SjqN9P+tE21/1ujAWkGz9iii\n/MdORXKBpLOBBcB025+WFFvZ6uVif1LRyKclbSKNwa5o0wvaRd4Xm4EVtv9l+y3gDVLD0W6K5OIS\n4EEA273APqSCgZ2m0PlkoGZtKKL8x051cyHpZOAOUiPRruPQUCcXtrfZHm27y3YX6XrNdNvDLobW\nxIp8Rh4h9SaQNJo0FLWxzCBLUiQXbwNTACQdT2ooOnGO2hXARfnXT5OAbbbfrbdRUw49efeV/2g5\nBXNxC7AfsCxfz3/b9vSGBb2bFMxFRyiYi8eBqZJeA3YA19tuu153wVxcC9wl6WrShe157fjFUtL9\npC8Ho/P1mBuBvQBsd5Ouz0wD/g58DFxcaL9tmKsQQgi7ULMOPYUQQmgS0VCEEEKoKRqKEEIINUVD\nEUIIoaZoKEIIIdQUDUVoOpJ2SHql4tZVY92uwSplDvGYT+fqo+tyyYtjh7GPyyVdlO/Pk3RoxXN3\nSxqzi+N8QdL4AttcJWnf//fYoXNFQxGa0Xbb4ytum0o67hzb40jFJm8Z6sa2u20vyQ/nAYdWPHep\n7dd2SZQ747yNYnFeBURDEYYtGorQEnLP4VlJL+Xb6VXWOUHS87kXsl7S0Xn53Irld0jas87hVgNH\n5W2n5DkMNuRa/3vn5Tdr5xwgv8zLbpJ0naQZpJpb9+Vjjsw9gYm51/H5yT33PBYNM85eKgq6Sbpd\n0lqluSd+lpf9mNRg9UjqycumSurNeVwmab86xwkdLhqK0IxGVgw7Lc/LtgDn2J4AzAIWVtnucuDX\ntseTTtSbc7mGWcAZefkOYE6d438H2CBpH2AxMMv2SaRKBvMlHQR8FzjB9ljgF5Ub234IWEv65j/e\n9vaKpx/O2/abBTwwzDjPJZXp6LfA9kRgLHCWpLG2F5JKak+2PTmX8rgBODvnci1wTZ3jhA7XlCU8\nQsfbnk+WlfYCFuUx+R2kukUD9QILJB0O/MH2m5KmAF8HXsjlTUaSGp1q7pO0HdhEKkN9LPCW7Tfy\n8/cCPwQWkea6uEfSSmBl0Rdm+31JG3OdnTeB44A1eb9DiXMEqWxLZZ5mSvo+6XN9CGmCnvUDtp2U\nl6/JxxlBylsIg4qGIrSKq4H3gHGknvD/TEpke6mk54BvA6sk/YA0k9e9tn9S4BhzKgsISjqw2kq5\nttCppCJzM4AfAd8cwmt5AJgJvA4st22ls3bhOIEXSdcnfgOcL+kI4DrgFNtbJS0mFb4bSMCTtmcP\nId7Q4WLoKbSKUcC7ef6AC0nF3/6LpCOBjXm45VHSEMxTwAxJB+d1DlTxOcX/BnRJOio/vhB4Jo/p\nj7K9itSAjauy7YeksufVLCfNNDab1Ggw1DhzQbufApMkHUeave0jYJukLwPfGiSWPuCM/tck6YuS\nqvXOQvhcNBShVdwGfE/SOtJwzUdV1pkJvCrpFdK8FEvyL41uAJ6QtB54kjQsU5ftT0jVNZdJ2gB8\nBnSTTror8/7+TPUx/sVAd//F7AH73Qr8Ffia7efzsiHHma993EqqCruOND/268BS0nBWvzuBP0nq\nsf0+6RdZ9+fj9JLyGcKgonpsCCGEmqJHEUIIoaZoKEIIIdQUDUUIIYSaoqEIIYRQUzQUIYQQaoqG\nIoQQQk3RUIQQQqjpP3mVudm2b2SOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0edf3ed978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, _ = roc_curve(user_f['y_test'], user_f['y_pred'])\n",
    "roc_auc = auc(fpr, tpr)\n",
    "\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFX3wPHvSU9oIQkggqETetGAFKU3ARXEQhFfeX1F\nQEBFERFsIBZEEKSKoj9FRUURlCZNEBUhIKAU6ULoNSSEhJT7+2OWZAMhLJDNZJPzeZ48mbnTzmw2\ne3bunblXjDEopZRSV+JldwBKKaVyN00USimlsqSJQimlVJY0USillMqSJgqllFJZ0kShlFIqS5oo\n1DUTkR4i8pPdcdhNRMJFJE5EvHPwmGVFxIiIT04d051EZIuINLuO7fQ9mINEn6PwbCKyDygBpABx\nwCKgvzEmzs648iLHa/0/Y8xSG2MoC+wFfI0xyXbF4YjFAJWMMbvcfJyy5JJzzq/0iiJvuNsYUxCo\nA9QFhtocz3Wx81tyXvmGfi309Vau0kSRhxhjjgCLsRIGACLiLyJjRGS/iBwVkakiEui0/F4R2Sgi\nZ0Vkt4i0c5QXEZGPROSwiBwUkdcvVrGIyKMistoxPUVExjjHISJzRWSQY/pmEflWRI6LyF4RGei0\n3qsiMltEZorIWeDRS8/JEcenju3/FZHhIuLlFMevIjJRRGJEZLuItLxk26zO4VcRGSciJ4FXRaSC\niCwXkZMickJEPheRYMf6nwHhwA+O6qbnL60GEpGfRWSkY7+xIvKTiIQ5xfOI4xxOishLIrJPRFpl\n9rcUkUARedexfoyIrHb+uwE9HH/TEyIyzGm7+iLyu4iccZz3RBHxc1puRORJEdkJ7HSUjReRA473\nwHoRudNpfW8RedHx3oh1LL9FRFY5VtnkeD0ecqzf0fF+OiMiv4lILad97RORISKyGTgnIj7Or4Ej\n9ihHHEdFZKxj04vHOuM4VkPn96Bj2+oiskRETjm2fTGz11VdJ2OM/njwD7APaOWYLg38BYx3Wj4O\nmAeEAIWAH4A3HcvqAzFAa6wvDaWAKo5lc4BpQAGgOLAWeMKx7FFgtWO6CXCA9GrMosB54GbHPtcD\nLwN+QHlgD9DWse6rQBLQybFuYCbn9ykw1xF7WWAH8JhTHMnAM4Av8JDjfEJcPIdkYADgAwQCFR2v\nhT9QDOsD6r3MXmvHfFnAAD6O+Z+B3UBlx/5+Bt5yLKuGVTV4h+O1GOM491ZX+LtOcmxfCvAGGjni\nunjM6Y5j1AYSgaqO7W4DGjjOqSywDXjaab8GWIL1fgh0lD0MhDq2eRY4AgQ4lg3Gek9FAOI4XqjT\nvio67bsucAy43RHzfxyvmb/T67cRuMXp2GmvKfA70NMxXRBokNnrnMl7sBBw2BF7gGP+drv/N/PS\nj+0B6M8N/gGtf7Q4INbxz7QMCHYsE+AcUMFp/YbAXsf0NGBcJvss4fjwCXQq6wascEw7/5MKsB9o\n4ph/HFjumL4d2H/JvocCHzumXwVWZXFu3sAFoJpT2RPAz05xHMKRpBxla4GeLp7D/isd27FOJ+DP\nS17rqyWK4U7L+wGLHNMvA186LQtynNtliQIraZ4Hamey7OIxS19yzl2vcA5PA3Oc5g3Q4irnffri\nsYF/gHuvsN6liWIKMPKSdf4Bmjq9fv/N5P17MVGsAl4Dwq5wzldKFN2c/076k/0/Wk+YN3QyxiwV\nkabAF0AYcAbrW3EQsF5ELq4rWB/AYH2zW5DJ/spgfUM/7LSdF9aVQwbGGCMis7D+WVcB3YGZTvu5\nWUTOOG3iDfziNH/ZPp2EOeL416nsX6xv2RcdNI5PC6flN7t4DhmOLSIlgPHAnVjfSr2wPjSvxRGn\n6Xisb8Y4Yko7njEm3lHllZkwrG/Gu6/1OCJSGRgLRGL97X2wruqcXXrezwGPOWI0QGFHDGC9R7KK\nw1kZ4D8iMsCpzM+x30yPfYnHgBHAdhHZC7xmjPnRheNeS4zqOmgbRR5ijFkJfIJVrQFwAuubaXVj\nTLDjp4ixGr7B+qetkMmuDmB9Gw9z2q6wMab6FQ79JXC/iJTBuor41mk/e532EWyMKWSMae8cdhan\ndAKreqaMU1k4cNBpvpQ4ZQLH8kMunsOlx37DUVbTGFMYq0pGslj/WhzGqhoErDYIrOqezJwAEsj8\nb3M1U4DtWHcjFQZeJOM5gNN5ONojngceBIoaY4Kxqu8ubnOl90hmDgCjLvl7Bxljvszs2Jcyxuw0\nxnTDqiZ8G5gtIgWy2sbpuOVdjFFdB00Uec97QGsRqW2MScWqyx4nIsUBRKSUiLR1rPsR0EtEWoqI\nl2NZFWPMYeAn4F0RKexYVsFxxXIZY8yfWB9uHwKLjTEXryDWArGOBsxAR8NoDRGp58qJGGNSgK+B\nUSJSyJGIBpF+xQLWh8pAEfEVkQeAqsCCaz0Hh0JY1XgxIlIKq37e2VGu/wNpNnC3iDRyNC6/yuUf\n4AA4/m4zgLFi3Qzg7WjA9XfhOIWAs0CciFQB+rqwfjJwHPARkZexrigu+hAYKSKVxFJLRC4muEtf\nj+lAHxG53bFuARHpICKFXIgbEXlYRIo5zv/ieyjVEVsqV37tfwRKisjTYt28UUhEbnflmMo1mijy\nGGPMcawG4JcdRUOAXcAase4sWorVMIkxZi3QC6vBOwZYSfq390ewqg22YlW/zAZKZnHoL4BWjt8X\nY0kBOmLdhbWX9GRS5BpOaQBWO8seYLVj/zOclv8BVHLsexRwvzHmYpXOtZ7Da8CtWK/FfOC7S5a/\nCQx33NHz3DWcA8aYLY5zmYV1dRGH1fCbeIVNnsNqRF4HnML6hu3K/+tzWNV/sVgf3F9dZf3FWM/e\n7MCqtksgY/XQWKxk/RNWAvoIqxEdrGT3f47X40FjTBRWG9VErNd7F5ncyZaFdsAWEYnDqgLsaow5\nb4yJx/rb/uo4VgPnjYwxsVg3IdyNVSW3E2h+DcdVV6EP3CmPJSKPYj0Ad4fdsVwrESmI9a25kjFm\nr93xKJUVvaJQKoeIyN0iEuSodx+DdcWwz96olLo6TRRK5Zx7sRraD2FVl3U1ekmvPIBWPSmllMqS\nXlEopZTKksc9cBcWFmbKli1rdxhKKeVR1q9ff8IYU+x6tvW4RFG2bFmioqLsDkMppTyKiPx79bUy\np1VPSimlsqSJQimlVJY0USillMqSJgqllFJZ0kShlFIqS5oolFJKZcltiUJEZojIMRH5+wrLRUQm\niMguEdksIre6KxallFLXz53PUXyC1d3wp1dYfhdWfzeVsAa7meL4rZRS6kYYAwmnIPEMnDvKhXOx\nN7Q7tyUKY8wqESmbxSr3Ap86OkVbIyLBIlLSMeCMUkqpzCSdh6Q4KxFciIXYA5BwGk7+DYkxsGc+\nXIiB5AQABv/Qmj8PZTUMy9XZ+WR2KTIOkBLtKLssUYhIb6A3QHh4eI4Ep5RStrkQB2d2wZG1cGQd\nnNltzackQvwx1/dTsBQ1bj7JhF8b3lA4HtGFhzHmA+ADgMjISO3uVimVtxgD+5fB76/BwdVXX98n\nAAJCIe4glLoDYvZC8bpQtDJbz0eyYacPD/e5C/wK8khvQ9M3YihXbsR1h2dnojgI3OI0X9pRppRS\neVPCGdj+hXWFkBwP0b+Ab5B11ZCZoOJQtLKVBG6qD0UrQeGyUKDEZavGxyfx+uureOed3/D2Fhq0\nb0nFiiAilC0bfENh25ko5gH9RWQWViN2jLZPKKXyBGPg9A7r6mDrZxAXbSWHqykRCeXugirdIbSK\ny4dbuHAnTz65gL17zwDw2GO3ERoaeJWtXOe2RCEiXwLNgDARiQZeAXwBjDFTgQVAe6wB2OOBXu6K\nRSml3CI1GQ6shOMbISUJ9i+1GpdP78h6u0pdoHRT8CsICIRVt5KEyDUd/uDBszz99GJmz94KQK1a\nJZg6tQMNG95ylS2vjTvveup2leUGeNJdx1dKqWyXnAD7FsO+n6wqpMQzWa8fEGpVF9V8HEIiILQa\nBBTNtnCefHIBc+f+Q1CQLyNGNOOppxrg45P9j8d5RGO2UkrlqJQkuHAWTv1jXR2c+AvWj73y+uXu\nguTzVjuCtz+UuhPKtATJ/g/t5OTUtGTw9tut8PX15t132xAeXiTbj3WRJgqllALrWYTDa2Dt2xC9\nMut1C94MtZ6A8JZwc6NrrjK6HjExCQwfvpwdO06xaFEPRISIiDC++eYBtx9bE4VSKn9Jioe9C60r\nhqPr4dR269bUK/EtAOXaQ3BFKylU6JhzsQLGGL75ZitPP72Iw4fj8PYWNm48Qt26N/YQ3bXQRKGU\nypsSzsDJLdZTzCe3wbaZVmK4mtJNofIDUKdfjlwpZGX37lP077+QRYt2AdCwYWmmTu1IrVqX3x7r\nTpoolFJ5Q8IZ2POjdbWw/Yurrx9SFW5pbj3PEFodqv8Hgoq5P04XjRnzGy+9tIKEhGSCgwN4++1W\n/O9/t+LllfPJSxOFUspzJSfC3gUw774rr+MfDKWbWM8xVO0BNXpBgZtyLsbrFB+fREJCMj171mLM\nmDYUL17Atlg0USilPEfSedj6KawZAXGHLl/uH2zdeVS+g1V9VDDn6vFv1PHj5/jnn5PccYfVn92Q\nIY1p1qwsTZqUsTkyTRRKqdzMGDi2AQ7/AUeiYMvHma9X+QFo+Ir14JqHSU01zJjxJ88/vwQfHy+2\nb+9PSEgg/v4+uSJJgCYKpVRuE3sQNn8Af3+Y+VUDQKX7oFgdqP4oFM7ep5Bz0t9/H6NPnx/59Ver\nI+3WrcsTH59ESEj2db+RHTRRKKXsk3LBeqht7ZtwfBOc3Jr5ejfVg6IRUK0nhLcAL8/+6Dp37gIj\nRqxk7Ng1JCenUqJEAd57rx0PPVQdsflOq8x49qutlPI8qSkQNcZ60jmrsRUCi0HzcVYDdB5z//3f\nsGjRLkSgX79IRo1qSXBwgN1hXZEmCqWU+8Ufh9UvWh3m7Vt8+fICN8Htw+GWZlbfSN5+OR5iThoy\npDFHj8YxZUoHbr+9tN3hXJUmCqWUe5zZbfWs+tP/gEzGG/MJhPuXQqlGOR5aTkpOTuX99/9g374z\njB9/FwDNmpUlKqq3Lc9EXA9NFEqp7GMMbJ8Fi3tZw3ZeqsK9UHcA3NLU49sZXLF27UGeeOJHNm48\nAkDv3rdRvXpxAI9JEqCJQil1o+IOw98fWcNx/j0j47KCpaw7lCK65vkrB2dnziTw4ovLmDo1CmOg\nTJkiTJzYPi1JeBpNFEqpa5N03nqeYfuXWY/v3GUxlG2Tc3HlErNm/c3TTy/i6NFz+Ph48eyzDXnp\npSYUKOC57S6aKJRSrjGpsOxJ2DT18mU+QRBcAW4fBpU65/nG6Kz89NNujh49R+PGtzBlSgdq1szZ\nDvzcQROFUipr547CnA6X97wa3hIqdobK90MBz/8wvF6JickcPBhL+fLWyHWjR7fmzjvD+c9/6nhU\nO0RWNFEopS5nDPz1ISzpffmywmWgRxQEheV8XLnM8uV76dt3Pl5ewqZNffDz8yYsLIheveraHVq2\n0kShlMpo7yKY19kaH9rZzY2gwyyP7jIjuxw9Gsdzzy1h5szNAFSpEkZ09Nm0q4q8RhOFUgpSk2HH\nbFj0aMbbWv2DoekYqPmYbaHlJqmphunT1/PCC8s4cyaBgAAfhg+/k8GDG+Pn5213eG6jiUKp/Cxm\nL2yebvW15Cy4AnT6AUKr2hNXLtW581fMm/cPAG3bVmDSpPZUqBBic1Tup4lCqfxm70LYPQ/2LIDY\n/RmXle8IDYZDydvtiS2Xu+++Kqxde5Dx49vxwAPVcmUHfu6giUKp/CApHrb8Hyzrl/nyhq9Ard5Q\n8OacjSuXmzfvH6Kjz9KvXz0AHnmkNvfdV5VChfxtjixnaaJQKi+6EAen/7GuHjZPv/zKofL91tWD\nf1Eodxd4+9oTZy61f38MAwcuZO7cf/D396Zdu4qUL18UEcl3SQI0USiVNxxYCWteh2N/gkmGxJjM\n1ytaGZq/ZyUHdZmkpBQmTPiDV175mXPnkihUyI/XX29BmTJF7A7NVpoolPJEp3bA8v7g5Qt7F2S+\nTuEycHa/dfVQ7i6o0h188t+3YVetWRPNE0/8yObNRwF44IFqjBvXllKlCtscmf00USjlCfYthqix\nELPb6r77ShoMh+q9rCThlXdv13SHl15awebNRylXLpiJE9vTvn0lu0PKNTRRKJUbJSfC+ndh53eX\nd53h7LZnoXIXKFYLfAvkXHx5gDGG2NgLFC5sXWVNnHgXn366iWHDmhAUpG02zjRRKJWb/LsUtn4K\nWz/LfHntPlClG9xUH3xy79CZud0//5ygX78FiMCSJT0RESIiwhg1qqXdoeVKmiiUyg0uxMKcjhC9\nKmN5yduhliM5aPvCDUtISObNN3/hrbd+5cKFFEJDA9m37wzlyuXNrjeyiyYKpewUdxhmt4KTWzOW\nNxtrtTUEBNsTVx60ZMlu+vVbwK5dpwD473/rMHp0a0JDg2yOLPdza6IQkXbAeMAb+NAY89Yly8OB\n/wOCHeu8YIy5wi0cSuUhp/6Bz26F5PiM5XX6Q4sJkE+e+M0Jxhgee2weH3+8EYBq1YoxdWoH7ryz\njM2ReQ63JQoR8QYmAa2BaGCdiMwzxjh/dRoOfG2MmSIi1YAFQFl3xaSU7TZ/AEueuLz8tmfhjlFa\nveQGIkLZssEEBvrw8stNGTSoYZ7uwM8d3HlFUR/YZYzZAyAis4B7AedEYYCLNykXAQ65MR6l7LVm\nFPw6PGNZk9EQ+ZxeQWSzjRuPcPhwLHfdZd3iOmRIY3r2rKVtEdfJnYmiFHDAaT4auLSnsVeBn0Rk\nAFAAaJXZjkSkN9AbIDw8PNsDVcqtzuyG/6uVsZqp669QqpF9MeVRsbGJvPLKz4wf/wehoYFs396f\nkJBA/P19NEncALsbs7sBnxhj3hWRhsBnIlLDGJPqvJIx5gPgA4DIyEhjQ5xKXbuosbDy2cvLnzgE\nBUvmfDx5mDGG77/fzsCBi4iOPouXl9C9e018fb3sDi1PcGeiOAg4D4VV2lHm7DGgHYAx5ncRCQDC\ngGNujEsp90lOgHXvwP6ll9/qWrsPtJpiT1x52L//nqF//4X8+OMOACIjb2batI7ceqsm4+zizkSx\nDqgkIuWwEkRXoPsl6+wHWgKfiEhVIAA47saYlHIPY6xuvBf3unzZI5usJ6dVtjPG0KXL16xff5jC\nhf15440W9OkTibe3XklkJ7clCmNMsoj0BxZj3fo6wxizRURGAFHGmHnAs8B0EXkGq2H7UWOMVi0p\nz7F5utVAHX/JRXBwRejwJdwUaU9ceVxqqsHLSxARxoxpw9SpUYwb15aSJQvZHVqeJJ72uRwZGWmi\noqLsDkPlZ8mJ8PeMywcB8g+GoOLQZjqUbmJPbHncyZPxvPDCUgCmT7/H5mg8i4isN8Zc1zcXuxuz\nlfIMsQdh+xew6vnLl4W3stoegivoba5uYozh00838dxzSzhxIh4/P29eeaUZpUtrF+A5QROFUlk5\nux+mX+EJ3nLt4Y43oHjtnI0pn9m27Th9+85n5cp/AWjWrCxTpnTQJJGDNFEodaljm+DXYVY3G2d2\nZVxWpg3UGwxlMn3kR2UjYwwvv7yCt9/+laSkVMLCgnj33Tb07FkL0Su3HKWJQqmLUlNg5m1wfFPG\n8gI3QZuPoHx7e+LKp0SEgwdjSUpK5fHHb+Wtt1oREhJod1j5kiYKpQC2fQELemQsq/8CVH4QStS1\nJ6Z86NChWE6ciKdWrRIAjB7dmsceq0vjxtojg500Uaj8LTEGphSHlAvpZRFdof1n4KX/HjklJSWV\nKVOiGDZsOaVKFWLjxj74+XkTFhZEWJgmCbvpf4LKn1KS4M8JsPK5jOUPLIfw5vbElE9t2HCYJ574\nkagoq0/QJk3KcPZsImFhOk5EbuFSohARPyDcGLPrqisrldv98w38+GDGspaToU5fe+LJp86eTeSl\nl5YzceI6UlMNpUsXZsKEdnTqVEUbq3OZqyYKEekAjAX8gHIiUgd4xRjT2d3BKZUt4g5B7AGrk779\nyyDhZPqyopXg7tnaxUYOM8bQpMnHbNp0FG9vYdCgBrz6ajMKFdLxOHIjV64oRmB1D74CwBizUUQq\nujUqpW7UuSMwrwsc+u3K63RfY41JrXKciPDMMw2YPDmKadM6UqfOTXaHpLLgSqJIMsacueRS0LP6\n/VD5g0mFJX3g1HY4+EvGZd5+cFN9uKkeVO0BJW6zJ8Z86sKFFMaO/R1vb2Hw4MYAPPJIbR5+uJZ2\n4OcBXEkU20TkQcDL0RPsQGCNe8NS6hot7WsNM5pxKBNoOQlqPQFeOvSlXX755V/69JnP1q3H8ff3\n5pFHalOiREFEBG9vbYvwBK4kiv7Ay0Aq8B1Wb7AvujMopVyScgFWDrbuXnIWUhU6fgVhNbTvJRud\nOBHP888v4eOPNwJQqVIIkyd3oESJgjZHpq6VK4mirTFmCDDkYoGI3IeVNJTKWcbAtpmwYQIcvaQX\n4SLl4T9/g68+vWsnYwyffLKRwYOXcPLkefz8vBk69A5eeOEOAgL0jnxP5MpfbTiXJ4VhmZQp5T7J\nCbB8IPw1/fJlEV2h+Tirqw2VK8yc+RcnT56nRYtyTJ7cnoiIMLtDUjfgiolCRNpiDVNaSkTGOi0q\njFUNpZT7nf0XFv8X9i/PWF7zf1b33mVaQWCoPbGpNPHxScTEJFCyZCFEhMmT27Nu3SF69Kipz0Tk\nAVldURwD/gYSgC1O5bHAC+4MSin+eMN6MO74xozldZ60uvb21y6mc4uFC3fy5JMLKF++KEuW9ERE\niIgI06uIPOSKicIY8yfwp4h8boxJyMGYVH62bwl82+by8vBW0GWh9r+Uixw8eJann17M7NlbAShU\nyJ+TJ89r1xt5kCv/daVEZBRQDQi4WGiMqey2qFT+s28x/PAgXDibXubtB08c0qqlXCYlJZVJk9Yx\nfPhyYmMvUKCALyNGNGfgwNvx8dFnIvIiVxLFJ8DrwBjgLqAX+sCdyi7GwKe14MTf6WVFykPTd6DS\nffbFpTKVmmpo2vQTfv31AACdOlVh/Ph2hIcXsTky5U6uJIogY8xiERljjNkNDBeRKOAlN8em8rK4\nQ7B5Ovz5fsa+l7osgrJt7YtLZcnLS2jTpgL798cwcWJ77rknwu6QVA5wJVEkiogXsFtE+gAHgULu\nDUvlOf98DecOw845EL3y8uXNxsFtT+d8XCpLxhi+/noLPj5edOlSDYAhQxozaFBDChb0szk6lVNc\nSRTPAAWwuu4YBRQB/uvOoFQesuF9WDHwystvbmw9RV2oVM7FpFyye/cp+vVbwE8/7aZYsSBatChH\n0aKB+Pv74K+dvOYrV00Uxpg/HJOxQE8AEdH/apW58yfh50Gw8ztIirt8ef2hcEtzuLkh+GlXDrlR\nYmIy77zzG6NG/UJCQjJFiwYwalQLihQJuPrGKk/KMlGISD2gFLDaGHNCRKpjdeXRAiidA/EpT2EM\nfFQBYvZevsyvkDVy3E2ROR+XuiY//7yPvn3ns337CQB69qzFmDFtKF68gM2RKTtl9WT2m0AXYBNW\nA/aPQD/gbaBPzoSncr2E09ZtrfuXZiyPeAjq9IcSt4Kv3lfvCVJSUunXz0oSERGhTJnSgebNy9kd\nlsoFsrqiuBeobYw5LyIhwAGgpjFmT86EpnK18ydhzeuw4b2M5VW6QYcv7IlJXbPUVENCQjJBQb54\ne3sxZUoHVq36l+efb4y/vz7cqCxZvRMSjDHnAYwxp0RkhyYJxcmt8H81Lx/3IawG9IgCH23l9BR/\n/XWUPn3mU6VKKB99dC8ATZuWpWnTsvYGpnKdrBJFeRG52EOsYI2XndZjrDFGn4bKT1KSYPkA2Dwt\nY3nFTtBsLBTRKgpPce7cBUaMWMnYsWtITk5l797TnD59nqJFtXt2lbmsEkWXS+YnujMQlUvFH4fv\n7oKj6zOWd/oBKnS0JyZ13X744R/691/I/v0xiEC/fpGMGtWS4GC9o0ldWVadAi7LyUBULnNyO6x9\nA7Z+ll4m3lCuHdz9rVYxeZjk5FQeemg23323DYA6dW5i2rSO1K+vd7qrq9PWKpXR2X9hetnLy0vd\nAQ+tBNFO3zyRj48XRYr4U7CgHyNHNqd///ragZ9ymRjjvv79RKQdMB7wBj40xryVyToPAq9idTS4\nyRjTPat9RkZGmqioqKxWUdfj5Fb4pPrl5bc9C+XbQ3iLnI9J3ZA//ogG4PbbrUeeTp6M5/z5ZEqX\n1rE88iMRWW+Mua6HmVy+ohARf2NM4jWs7w1MAloD0cA6EZlnjNnqtE4lYCjQ2BhzWkSKux66yhZJ\n52BBT9g1J2N5y8lQp689MakbcuZMAkOHLmXatPVUqRLGxo198PPzJjRUn2dR1+eqiUJE6gMfYfXx\nFC4itYH/GWMGXGXT+sCui7fUisgsrGcztjqt8zgwyRhzGsAYc+zaT0Fdt0O/w5eNMpbd8QbUG6wD\nBHkgYwxffvk3gwYt5ujRc/j4eHHPPRGkpKRiXdQrdX1c+TSYAHQEvgcwxmwSkeYubFcK6yG9i6KB\n2y9ZpzKAiPyK9U5+1RizyIV9qxsRexDmdoKjTlV4NR+HVpM1QXionTtP0q/fApYutR51atz4FqZO\n7UiNGnqRrm6cK58KXsaYfy8ZID0lG49fCWiG1XfUKhGpaYw547ySiPQGegOEh4dn06HzqSNR8Hm9\njGWP/wuF9XX1VElJKbRo8SnR0WcJCQlk9OhW9OpVFy8vufrGSrnAlURxwFH9ZBztDgOAHS5sdxC4\nxWm+tKPMWTTwhzEmCdgrIjuwEsc655WMMR8AH4DVmO3CsVVmDq2BLxumzzcaAZHPga8+aOWJjDGI\nCL6+3owa1YIVK/YxenQrihXTDvxU9rrqXU+OBuYJQCtH0VKgvzHmxFW288FKKC2xEsQ6oLsxZovT\nOu2AbsaY/4hIGPAnUMcYczKzfYLe9XTdjv9lDTl6Ue8DUEg7APZER4/G8dxzS6hcOYSXXmpqdzjK\nQ7j7rqdkY0zXa92xMSZZRPoDi7HaH2YYY7aIyAggyhgzz7GsjYhsxarOGpxVklDX6Y+3YPXQ9PmH\nN2iS8EDgGhyuAAAgAElEQVSpqYbp09fzwgvLOHMmgeDgAJ5+ugGFCunDj8q9XEkU60TkH+Ar4Dtj\nTKyrOzfGLAAWXFL2stO0AQY5fpQ7fNMS9i9Pn7/rUyhR17541HXZtOkIffrMZ80a69mIdu0qMmlS\ne00SKke4MsJdBRFpBHQFXhORjcAsY8wst0enrk9KEkSNgdUvppeVbAjdVuuT1R4mKSmFoUOX8d57\na0hJMZQsWZDx49tx//3VuOQGE6XcxqV7IY0xvwG/icirwHvA54AmitzGpMLYTO6XD28FDyzJ+XjU\nDfPx8eLPP4+QmmoYMKA+I0c21yFJVY5z5YG7glgPynUFqgJzgUZZbqRyXvwxmFIiY1nJhtBpLgQV\nsycmdV32748hJSWVcuWKIiJMndqBmJhEIiNvtjs0lU+5ckXxN/ADMNoY84ub41HX6t9lMO8+uHA2\nvazm49B6GmjVhEdJSkph/Pg/eOWVn2nYsDRLlvRERKhUKdTu0FQ+50qiKG/MpcOZqVwhaiysfDZj\nWcevIeIBe+JR1+333w/Qp898Nm8+CkBISCDx8UkUKOBnc2RKZZEoRORdY8yzwLcictnDFjrCnc3i\nDmdMEv/dAUUr2RePui6nT5/nhReW8sEHGwAoVy6YSZPac9dd+rdUuUdWVxRfOX7ryHa5TdI5mOZU\nX907GgrpADSeJjExmTp1prF/fwy+vl4MHtyIYcOaEBTka3doSmWQ1Qh3ax2TVY0xGZKF40E6HQEv\npxkDPw+CDe+ll3X4UpOEh/L39+Gxx+qybNlepkzpQLVqetOByp1c6cJjgzHm1kvK/jTG2PLUVr7t\nwiP+BEy55IOky2Io28aeeNQ1S0hI5s03fyEiIozu3WsC1hCl3t6iz0Qot3NLFx4i8hDWLbHlROQ7\np0WFgDOZb6XcwpiMScLbDwbEgbdWUXiKJUt206/fAnbtOkXx4gXo3LkKgYG+Ohyp8ghZtVGsBU5i\n9fo6yak8FqvzPpVTJhZJn249HWr9z75Y1DU5ciSOQYMW8+WXfwNQvXoxpk7tSGCgJnnlObJqo9gL\n7MXqLVbZITUZPq0DFxzdazUeqUnCQ6SkpDJt2npefHEZMTGJBAb68MorTXnmmYb4+eloc8qzZFX1\ntNIY01RETgPODRmC1Z9fiNujy8/iDsE0p0bqQuHQYLh98ahrkpJieP/9tcTEJNK+fSUmTryLcuWK\n2h2WUtclq6qni8OdhuVEIMpJakrGJBFcEf77j33xKJfExiaSkmIIDg7Az8+b6dPv5ujROO67r6o2\nViuPdsWWNKensW8BvI0xKUBD4AlAh9BylyPrYJxT/r5/CTy2U3t9zcWMMXz33TaqVp3Es88uTiu/\n445wunTRXl6V53Pl0+d7rGFQKwAfYw1V+oVbo8qvYvbB5/XT528fBmVaXXF1Zb99+85wzz2z6NLl\naw4ejOXvv4+TkJBsd1hKZStX+npKNcYkich9wPvGmAkionc9ZbfkRPiwXPr8fQug3F32xaOylJSU\nwtixv/Paays5fz6ZwoX9eeONFvTpE4m3t179qbzFpaFQReQBoCfQyVGm9/Zlp72L4DunpPDAcghv\nfuX1la3i45No0OBD/vrrGABdu9Zg7Ng2lCxZyObIlHIPVxLFf4F+WN2M7xGRcsCX7g0rn0hNgZm3\nwvHN6WWtp2mSyOWCgnyJjLyZ+PgkJk/uQJs2FewOSSm3umoXHgAi4gNUdMzuMsbYVgmbp7rwmBQK\nCafS5x9eDyVuvfL6yhbGGD79dBMVKoRwxx3hAMTEJODn560PzimP4ZYuPJx2fifwGXAQ6xmKm0Sk\npzHm1+s5oHJYPjA9SQSVgL5H7I1HZWrbtuP07TuflSv/pWrVMDZu7IOfn7cOR6ryFVeqnsYB7Y0x\nWwFEpCpW4riuzJTvpSTBe5cMRqNJItc5fz6JUaN+YfToX0lKSqVYsSCGDr0DX19tqFb5jyuJwu9i\nkgAwxmwTER1263r9MiR9ulgdeHidfbGoTC1atIsnn1zAnj2nAXj88Vt5661WhIQE2hyZUvZwJVFs\nEJGpwEzHfA+0U8Drc2wTrB9nTZeI1CSRC8XFXaBnzzmcOBFPjRrFmTq1A40bh9sdllK2ciVR9AEG\nAs875n8B3ndbRHnZZ3XSp7sssi8OlUFKSiqpqQZfX28KFvRj/Ph2REef5ZlnGuDrqx34KZVlohCR\nmkAFYI4xZnTOhJRHHV2fPv3gCggMtS8WlWb9+kM88cSP3HtvBC+91BQgbVAhpZTlii1zIvIiVvcd\nPYAlIvLfHIsqrzl7AD6/PX3+lma2haIsZ88m8tRTC6lf/0PWrz/MZ59tJikpxe6wlMqVsrqi6AHU\nMsacE5FiwAJgRs6ElYcc/ws+rZU+r1VOtjLGMHv2Vp56ahGHD8fh7S0MGtSA115rrtVMSl1BVoki\n0RhzDsAYc1xEuy+9ZilJGZNEpx+gbFv74snnYmMTeeih2SxcuAuA228vxdSpHalT5yabI1Mqd8sq\nUZR3GitbgArOY2cbY+5za2R5wcTC6dNdFmmSsFnBgn4kJqZQpIg/b73Vit69b8PLS7sAV+pqskoU\nXS6Zn+jOQPKcnd9DcoI1Xae/JgmbrFr1LyVLFqRSpVBEhBkz7iEgwIcSJQraHZpSHiOrMbOX5WQg\neYYx8FEFiNmbXtZS7ybOaSdOxPP880v4+OONtGxZjiVLeiIilCkTbHdoSnkcV56jUK46sQW+uhMS\nTqeX9dpuXzz5UGqq4ZNPNjJ48BJOnTqPn583d94ZTkqKwcdHq5mUuh5uTRQi0g4YD3gDHxpj3rrC\nel2A2UA9Y4xndg17/hT8X430+eq9oO1HoMNg5pgtW47Rt+98fvllPwAtW5Zj8uQOVK6sz6wodSNc\nThQi4m+MSbyG9b2BSUBrIBpYJyLznPuNcqxXCHgK+MPVfec6KRdgstOHUddfoVQj++LJh2JiEmjQ\n4CPi4i5QvHgBxo5tQ/fuNXW8aqWywVVveRWR+iLyF7DTMV9bRFypdK+PNXbFHmPMBWAWcG8m640E\n3gYSXA87l1nQI3367m80SeSgi+OpFCkSwJAhjenT5za2b3+SHj1qaZJQKpu48mzEBKAjcBLAGLMJ\ncGUItlLAAaf5aEdZGhG5FbjFGDM/qx2JSG8RiRKRqOPHj7tw6BySdB7eFdgx25qv/CBUvt/emPKJ\ngwfPcv/9XzNzZvrogMOG3cmUKR0pWlR7eVUqO7mSKLyMMf9eUnbDfR04HuAbCzx7tXWNMR8YYyKN\nMZHFihW70UNnj9QUmFEpY1mHL+yJJR9JTk5l/Pg1VKkyiW+/3cYrr/xMSkoqgF5BKOUmriSKAyJS\nHzAi4i0iTwM7XNjuIHCL03xpR9lFhYAawM8isg9oAMwTEc8YEGnrZxDnOJ1qPeFZA17aBYQ7rVt3\nkNtv/5Cnn15MXNwFOnWqwsqVj+LtrZ0GKOVOrjRm98WqfgoHjgJLHWVXsw6oJCLlsBJEV6D7xYXG\nmBgg7OK8iPwMPOcRdz0ZA4t7WdPFb4W7PrU3njzu3LkLDBmylMmT12EMhIcX4f337+KeeyLsDk2p\nfOGqicIYcwzrQ/6aGGOSRaQ/sBjr9tgZxpgtIjICiDLGzLvmaHOL5QPTp9t9bF8c+YSPjxdLl+7B\ny0sYNKghr7zSlAIFdJBFpXLKVROFiEwHzKXlxpjeV9vWGLMAq9dZ57KXr7Bus6vtL1f49RXY6OjN\npEQkFKuV9frquuzefYrg4ABCQ4Pw9/fhs886ExDgQ82aJewOTal8x5XK3aXAMsfPr0BxwOXnKfKU\no+thzYj0+R5r7Yslj0pMTOb111dRo8YUhgxZmlZer14pTRJK2cSVqqevnOdF5DNgtdsiyq2SE2Gm\nUzv7k6f1qets9vPP++jbdz7bt58ArDucUlJStbFaKZtdTxce5YD899Xu83rp0z03QoB2Lpddjh07\nx+DBS/j0000ARESEMmVKB5o3L2dzZEopcK2N4jTpbRRewCngBXcGlevERsOJv6zpxq9D8dr2xpOH\nnDgRT9Wqkzh16jz+/t4MG3Ynzz/fGH9/7a9Sqdwiy/9GsZ5gqk368w+p5mKfCfnJiqfTpxsMsy+O\nPCgsLIh7740gOvoskyd3oGLFELtDUkpdIstEYYwxIrLAGFMjq/XytPgTsPNba7q2K4+PqKycO3eB\nESNW0qFDZZo0KQPA5Mkd8Pf31ierlcqlXGkl3Cgidd0eSW6UmgxTnLoMaaEDEN2IH374h2rVJjN6\n9G/06zef1FTr4jQgwEeThFK52BWvKETExxiTDNTF6iJ8N3AOa/xsY4y5NYditM+00unTtftoFx3X\n6cCBGJ56ahFz5liDONWtexPTpnXU8aqV8hBZVT2tBW4F7smhWHKP1GT4qBLEH7XmC5SElpPtjckD\nJSenMmHCH7z88grOnUuiYEE/Xn+9OU8+WR8fH73lVSlPkVWiEABjzO4ciiV3MKkwzjd9PiAE+hyy\nLx4PdvZsIm++uZpz55Lo0qUq773XjtKlC9sdllLqGmWVKIqJyKArLTTGjHVDPPb73mlspcr3WwMR\nKZedOZNAYKAP/v4+hIQEMm1aR/z9venQobLdoSmlrlNW1//eQEGs7sAz+8l7zu6HPT9a04FhmiSu\ngTGGL774i4iIiYwe/Wta+X33VdUkoZSHy+qK4rAxZkQWy/Oe5QPSpx/fZ1sYnmbHjpP06zefZcv2\nArBq1X6MMXonk1J5xFXbKPIFY+CT6nBqmzVf/VHwLWBrSJ4gISGZt99ezRtvrObChRRCQgJ5553W\nPPpoHU0SSuUhWSWKljkWhd1+ezk9SZRtC20/sjceD3DkSBxNmnzMzp2nAHj00Tq8805rwsKCbI5M\nKZXdrpgojDGncjIQ2yQnwJrX0+e7LLIvFg9SokQBbrmlCD4+XkyZ0oGmTcvaHZJSyk2057UN49On\nB56zL45cLjXVMH36epo3L0flyqGICF98cR9Fiwbi56cPIiqVl+Xvp552/wC/ODrCLd0EfLXaJDOb\nNh2hceMZ9Okzn3795nOxX8gSJQpqklAqH8i/VxQmFb53eui81TT7Ysml4uIu8OqrP/Pee2tISTHc\nfHMh+vSJvPqGSqk8Jf8min2L06d7rIPQKvbFkgt9//12BgxYSHT0Wby8hAED6vP66y0oXNjf7tCU\nUjks/yaKde9YvwuXgZv0W7KzgwfP0rXrbBITU7jttpJMndqRyMib7Q5LKWWT/JkoEk7DgRXWdOUH\n7Y0ll0hKSsHHxwsRoVSpwowa1QI/P2/69aunY1Yrlc/lz0+AKU5Dftd73r44conffjvAbbd9wMyZ\nm9PKnn22EQMG3K5JQimVDxPFv8sgNcmarjsAgsLsjcdGp06d54knfqBx4xn89dcxJk+OIj+OdKuU\nylr+qno6ugFmt0qfbzHBvlhsZIxh5szNPPvsTxw/Ho+vrxfPP9+YYcPu1K43lFKXyV+J4k+nxNBr\nu31x2Ojo0Ti6dfuWFSv2AdC0aRmmTOlA1arFst5QKZVv5a9EsXeh9fuOURASYW8sNgkODuDw4TjC\nwoIYM6Y1jzxSW68ilFJZyj+JYuf3EH/Mmg5vlfW6ecySJbu59daShIYG4e/vwzffPEDJkgUJDdUn\n0ZVSV5d/GrPndU6fzifPTRw+HEu3bt/Sps1MhgxZmlZeo0ZxTRJKKZfljyuK6F/SpzvPB8nb+TEl\nJZVp09YzdOgyzp5NJDDQh4iIUB1MSCl1XfJHonDuIbZ8e/viyAEbNhymT58fWbfuEAAdOlRi4sT2\nlC0bbHNkSilPlfcThTGw81trusFL9sbiZvv2naF+/emkpBhKlSrEhAl30blzFb2KUErdELcmChFp\nB4wHvIEPjTFvXbJ8EPA/IBk4DvzXGPNvtgbxy9D06VufztZd5zZlywbTq1cdChXy57XXmlGokHbg\np5S6cW6rrBcRb2AScBdQDegmItUuWe1PINIYUwuYDYzO9kD2L7N+F60MgSHZvns77dt3hrvv/pKV\nK/ellX3wwd2MHdtWk4RSKtu484qiPrDLGLMHQERmAfcCWy+uYIxZ4bT+GuDhbI/iaJT1+97vs33X\ndklKSmHs2N957bWVnD+fzIkT8fz++2MAWs2klMp27kwUpYADTvPRwO1ZrP8YsDCzBSLSG+gNEB4e\n7noESefTp4NKXHk9D7J69X769PmRLVuOA9C1aw3Gjm1jc1RKqbwsVzRmi8jDQCTQNLPlxpgPgA8A\nIiMjXe+1btvn6dMeXu10+vR5Bg9ewkcf/QlAhQpFmTy5A23aVLA5MqVUXufORHEQuMVpvrSjLAMR\naQUMA5oaYxKzNYIlj1u/y3fI1t3aITXVMHfuP/j6evHCC3cwdOgdBAb62h2WUiofcGeiWAdUEpFy\nWAmiK9DdeQURqQtMA9oZY45l69Hjj6dPNxqZrbvOKdu3n6BcuWD8/X0IDQ3i88/vIzy8CFWq5N+u\n0ZVSOc9tdz0ZY5KB/sBiYBvwtTFmi4iMEJF7HKu9AxQEvhGRjSIyL9sC2D4rfbpE3WzbbU6Ij09i\n2LBl1Ko1hdGjf00rb9OmgiYJpVSOc2sbhTFmAbDgkrKXnabd1zvfioHW7wr3uu0Q7rBo0S769ZvP\n3r1nADhxIt7miJRS+V2uaMzOdv98kz7d2DOqnQ4diuXppxfxzTfW3cM1axZn6tSONGp0y1W2VEop\n98p7iSLlAvz4YPp8sZr2xeKiHTtOEhn5AbGxFwgK8uXVV5vy9NMN8PX1tjs0pZTKg4lij1NNV7ff\n7YvjGlSqFEK9eqUoUMCX99+/izJltAM/pVTukbcSxYW49HEn/ArBzQ3sjecKzp5N5OWXV9CvXz0q\nVw5FRJg3rysFCvjZHZpSSl0mbyWKnx5Pn75/6ZXXs4kxhtmzt/LUU4s4fDiO7dtPsGiR1WuJJgml\nVG6VdxLFts/hH8ctsdUegZL17Y3nEnv2nKZ//wUsXLgLgAYNSvP22/lrSFallGfKO4li1fPp0y0m\n2BfHJS5cSGHMmN8YOXIVCQnJBAcH8NZbLXn88dvw8tIO/JRSuV/eSBTRv0CcNaIbfY+BfxF743Fy\n4EAMI0asJDExhR49avLuu20oUaKg3WEppZTL8kai+Pvj9OmgYvbF4XD69HmCgwMQESpUCGH8+HZU\nrBhCy5bl7Q5NKaWumdu68MhRWxyJoubjWa/nZqmphhkz/qRixfeZOXNzWvkTT0RqklBKeSzPTxTG\nqdfxaj1tC2PLlmM0a/YJjz02j1Onzqc1WiullKfz/Kqn0zvTp0vdkeOHj49PYuTIlYwZ8zvJyakU\nL16AcePa0q1bjRyPRSml3MHzE8W8+9Knc3gY0B07TtK27Uz27TuDCPTpcxtvvNGSokUDczQOpZRy\nJ89OFMbAyS3WdOPXc/zwZcoUISDAh9q1SzB1akcaNCid4zGo3CspKYno6GgSEhLsDkXlIwEBAZQu\nXRpf3+wb2MyzE0XMnvTp+i+4/XDJyalMnRpFt241CA0Nwt/fh0WLelCqVGF8fDy/uUdlr+joaAoV\nKkTZsmWRHL7aVfmTMYaTJ08SHR1NuXLlsm2/nv3pduBn67eXL3i5t6fVtWsPUr/+dAYMWMiQIend\ng5QpE6xJQmUqISGB0NBQTRIqx4gIoaGh2X4V69lXFKlJ1u+QKm47RExMAsOGLWfy5HUYA+HhRbj3\n3gi3HU/lLZokVE5zx3vOsxNF8nnr9y3Ns33Xxhi++moLzzyzmCNH4vDx8WLQoAa8/HJT7cBPKZWv\neHadyf7l1m+f7L/LaNOmo3Tr9i1HjsTRqNEtbNjQm7ffbq1JQnkUb29v6tSpQ40aNbj77rs5c+ZM\n2rItW7bQokULIiIiqFSpEiNHjsQ4PZe0cOFCIiMjqVatGnXr1uXZZ5+14xSy9Oeff/LYY4/ZHUaW\n3nzzTSpWrEhERASLFy/OdJ0777yTOnXqUKdOHW6++WY6deoEwNy5c6lVqxZ16tQhMjKS1atXA3D8\n+HHatWuXY+eAMcajfm677TaTZgzWz+rhJjskJ6dkmH/mmUVm+vT1JiUlNVv2r/KXrVu32h2CKVCg\nQNr0I488Yl5//XVjjDHx8fGmfPnyZvHixcYYY86dO2fatWtnJk6caIwx5q+//jLly5c327ZtM8YY\nk5ycbCZPnpytsSUlJd3wPu6//36zcePGHD3mtdiyZYupVauWSUhIMHv27DHly5c3ycnJWW5z3333\nmf/7v/8zxhgTGxtrUlOtz59NmzaZiIiItPUeffRRs3r16kz3kdl7D4gy1/m567lVT87jYld75IZ3\nt2LFXvr1W8C0aR1p0qQMAGPHtr3h/SoFwLtuaqt41lx9HYeGDRuyebPVtcwXX3xB48aNadOmDQBB\nQUFMnDiRZs2a8eSTTzJ69GiGDRtGlSpW+5+3tzd9+/a9bJ9xcXEMGDCAqKgoRIRXXnmFLl26ULBg\nQeLi4gCYPXs2P/74I5988gmPPvooAQEB/PnnnzRu3JjvvvuOjRs3EhxsjepYqVIlVq9ejZeXF336\n9GH//v0AvPfeezRu3DjDsWNjY9m8eTO1a9cGYO3atTz11FMkJCQQGBjIxx9/TEREBJ988gnfffcd\ncXFxpKSksHLlSt555x2+/vprEhMT6dy5M6+99hoAnTp14sCBAyQkJPDUU0/Ru3dvl1/fzMydO5eu\nXbvi7+9PuXLlqFixImvXrqVhw4aZrn/27FmWL1/Oxx9b3RIVLJjegei5c+cytD906tSJzz///LLX\nxR08N1GsGWn9LngzFK103bs5duwcgwcv4dNPNwEwduzvaYlCqbwiJSWFZcuWpVXTbNmyhdtuuy3D\nOhUqVCAuLo6zZ8/y999/u1TVNHLkSIoUKcJff/0FwOnTp6+6TXR0NL/99hve3t6kpKQwZ84cevXq\nxR9//EGZMmUoUaIE3bt355lnnuGOO+5g//79tG3blm3btmXYT1RUFDVqpPeAUKVKFX755Rd8fHxY\nunQpL774It9++y0AGzZsYPPmzYSEhPDTTz+xc+dO1q5dizGGe+65h1WrVtGkSRNmzJhBSEgI58+f\np169enTp0oXQ0NAMx33mmWdYsWLFZefVtWtXXngh4236Bw8epEGD9JE2S5cuzcGDB6/42nz//fe0\nbNmSwoULp5XNmTOHoUOHcuzYMebPn59WHhkZyfDhw7N6qbONZyaKlAtwwnpjUm/Ide0iNdXw0Ucb\nGDJkKadPJ+Dv783w4U0YPLhRNgaqlMM1fPPPTufPn6dOnTocPHiQqlWr0rp162zd/9KlS5k1a1ba\nfNGiRa+6zQMPPIC3t3U7+0MPPcSIESPo1asXs2bN4qGHHkrb79atW9O2OXv2LHFxcRm+YR8+fJhi\nxdJ7i46JieE///kPO3fuRERISkpKW9a6dWtCQkIA+Omnn/jpp5+oW7cuYF0V7dy5kyZNmjBhwgTm\nzJkDwIEDB9i5c+dliWLcuHGuvTjX4csvv+R///tfhrLOnTvTuXNnVq1axUsvvcTSpdbt+cWLF+fQ\noUNui8WZZyaKi1cTALWu/dJw797TPPzwHH777QAAbdpUYNKk9lSsGJJdESqVKwQGBrJx40bi4+Np\n27YtkyZNYuDAgVSrVo1Vq1ZlWHfPnj0ULFiQwoULU716ddavX59WrXOtnKtILr2nv0CBAmnTDRs2\nZNeuXRw/fpzvv/8+7Rtyamoqa9asISAgIMtzc973Sy+9RPPmzZkzZw779u2jWbNmmR7TGMPQoUN5\n4oknMuzv559/ZunSpfz+++8EBQXRrFmzTJ9HuJYrilKlSnHgwIG0+ejoaEqVKpXp+Zw4cYK1a9em\nJapLNWnShD179nDixAnCwsLSqthygufd9WRSYY2ju47wVuBz5TfSlRQu7M+OHSe56aaCzJrVhUWL\nemiSUHlaUFAQEyZM4N133yU5OZkePXqwevXqtG+n58+fZ+DAgTz/vDVS5ODBg3njjTfYsWMHYH1w\nT5069bL9tm7dmkmTJqXNX6x6KlGiBNu2bSM1NfWKH3xgJZTOnTszaNAgqlatmvbtvU2bNrz//vtp\n623cuPGybatWrcquXem9NMfExKR9CH/yySdXPGbbtm2ZMWNGWhvKwYMHOXbsGDExMRQtWpSgoCC2\nb9/OmjVrMt1+3LhxbNy48bKfS5MEwD333MOsWbNITExk79697Ny5k/r1Mx+mefbs2XTs2DFDcty1\na1fanWgbNmwgMTEx7TXasWNHhqo3d/K8RHHx2QmATnNd3mzx4l0kJiYDEBoaxLx5Xdm+/UkeeqiG\nPhSl8oW6detSq1YtvvzySwIDA5k7dy6vv/46ERER1KxZk3r16tG/f38AatWqxXvvvUe3bt2oWrUq\nNWrUYM+ePZftc/jw4Zw+fZoaNWpQu3bttG/ab731Fh07dqRRo0aULFkyy7geeughZs6cmVbtBDBh\nwgSioqKoVasW1apVyzRJValShZiYGGJjYwF4/vnnGTp0KHXr1iU5OfmKx2vTpg3du3enYcOG1KxZ\nk/vvv5/Y2FjatWtHcnIyVatW5YUXXsjQtnC9qlevzoMPPki1atVo164dkyZNSqt2a9++fYaqo1mz\nZtGtW7cM23/77bfUqFGDOnXq8OSTT/LVV1+lfV6tWLGCDh063HCMrpCL2cpTRNasZKIe3QUlG0L3\n3666/oEDMQwcuIjvv9/OyJHNGT68SQ5EqRRs27aNqlWr2h1GnjZu3DgKFSp0Wb1+ftCkSRPmzp2b\nabtQZu89EVlvjIm8nmN53hVFYoz1279wlqslJ6cyduzvVK06ie+/307Bgn6EhGj330rlJX379sXf\n39/uMHLc8ePHGTRokEs3D2QHz2vMNo5LSq8rPyG9Zk00ffr8yKZNRwHo0qUq48e3o1SprJOLUsqz\nBAQE0LOnfSNb2qVYsWJpT2/nBM9LFAmO+7QjHsx08R9/RNOo0UcYA2XLBjNx4l106FA5BwNUKp0x\nRoYtp0wAAAkTSURBVNvAVI5yR3OC5yWKi0pn3tZQv34p2ratSN26NzF8eBOCgrJv8A6lrkVAQAAn\nT57UrsZVjjGO8Siyuq34enhuoigcDsDOnSd55pnFjB3blsqVrX/I+fO74+Wl/5jKXqVLlyY6Oprj\nx4/bHYrKRy6OcJedPDNRVOxEYmIyb721mjffXE1iYgoBAT7Mnm1VR2mSULmBr69vto4yppRd3HrX\nk4i0E5F/RGSXiFz2NIqI+IvIV47lf4hIWVf2u2xPVWrVmsqrr64kMTGFXr3qMHVqx+wOXymlFG68\nohARb2AS/H979x9zZVnHcfz9CUUwHabMpmmhExFUICKj3DIFnVnBKsaPKUrTDIqakv3RtGU/5mpm\nW6T2SOmQJlJY1DOzzBlGMVAoAZHMH8jsKZe/iDXBSvz0x3U9ntPj4Zz7eeL8fL6v7Wznvs913/f3\nfHfOuc593ed8L84FeoCNkrptby9rdimwy/ZJkuYA3wRmv3FvJU+/dATTFh8CvMjYsSPp6vpwFPEL\nIYQ6qucZxRnAk7Z32P43sBKY0afNDOD2fP8uYKpqXPXbtWc4w4YN4brrzmHz5gXRSYQQQp3V7Z/Z\nkmYC59u+LC/PA95je1FZm225TU9efiq3eaHPvi4Heqv/nQZsq0vQ7Wck8ELNVoND5KIkclESuSgZ\nY/vwgWzYFhezbS8FlgJI2jTQv6F3mshFSeSiJHJRErkokbRpoNvWc+jpr8DxZcvH5XUV20g6CBgB\nvFjHmEIIIfRTPTuKjcBoSSdIGgrMAbr7tOkGLsn3ZwK/cbtVKQwhhA5Xt6En269KWgTcCwwBbrP9\nqKSvkib57gZuBX4o6UngJVJnUsvSesXchiIXJZGLkshFSeSiZMC5aLsy4yGEEBqr/cqMhxBCaKjo\nKEIIIVTVsh1Fvcp/tKMCuVgsabukrZLul9Sx/0KslYuydh+XZEkd+9PIIrmQNCu/Nh6VtKLRMTZK\ngffI2yWtkfRwfp9c0Iw4603SbZKey/9Rq/S4JC3JedoqaVKhHdtuuRvp4vdTwInAUGALMK5Pm08D\nXfn+HOBHzY67ibk4Gzg03184mHOR2x0OrAU2AJObHXcTXxejgYeBt+Tlo5sddxNzsRRYmO+PA3Y2\nO+465eL9wCRg234evwD4JSBgCvBgkf226hlFXcp/tKmaubC9xvaevLiB9J+VTlTkdQHwNVLdsFca\nGVyDFcnFJ4GbbO8CsP1cg2NslCK5MNA7xeUI4G8NjK9hbK8l/YJ0f2YAy51sAI6QdEyt/bZqR/E2\n4C9lyz15XcU2tl8FdgNHNSS6xiqSi3KXkr4xdKKaucin0sfb/kUjA2uCIq+Lk4GTJa2TtEHS+Q2L\nrrGK5OJa4CJJPcA9wGcbE1rL6e/nCdAmJTxCMZIuAiYDZzU7lmaQ9Cbg28D8JofSKg4iDT99gHSW\nuVbS6bb/0dSommMusMz2DZLeS/r/1mm2X2t2YO2gVc8oovxHSZFcIGkacDUw3fa/GhRbo9XKxeGk\nopEPSNpJGoPt7tAL2kVeFz1At+3/2H4aeJzUcXSaIrm4FPgxgO31wDBSwcDBptDnSV+t2lFE+Y+S\nmrmQ9E7gFlIn0anj0FAjF7Z32x5pe5TtUaTrNdNtD7gYWgsr8h75GelsAkkjSUNROxoZZIMUycUz\nwFQASWNJHcVgnKO2G7g4//ppCrDb9rO1NmrJoSfXr/xH2ymYi+uBw4BV+Xr+M7anNy3oOimYi0Gh\nYC7uBc6TtB3YB3zBdseddRfMxeeB70u6knRhe34nfrGUdCfpy8HIfD3my8DBALa7SNdnLgCeBPYA\nnyi03w7MVQghhAOoVYeeQgghtIjoKEIIIVQVHUUIIYSqoqMIIYRQVXQUIYQQqoqOIrQcSfskbS67\njarSdtT+KmX285gP5OqjW3LJizED2McCSRfn+/MlHVv22A8kjTvAcW6UNLHANldIOvT/PXYYvKKj\nCK1or+2JZbedDTruhbYnkIpNXt/fjW132V6eF+cDx5Y9dpnt7QckylKcN1MsziuA6CjCgEVHEdpC\nPnP4naQ/5tv7KrQ5VdJD+Sxkq6TRef1FZetvkTSkxuHWAiflbafmOQweybX+D8nrv6HSHCDfyuuu\nlXSVpJmkmlt35GMOz2cCk/NZx+sf7vnM48YBxrmesoJukr4naZPS3BNfyes+R+qw1khak9edJ2l9\nzuMqSYfVOE4Y5KKjCK1oeNmw0+q87jngXNuTgNnAkgrbLQC+Y3si6YO6J5drmA2cmdfvAy6scfyP\nAI9IGgYsA2bbPp1UyWChpKOAjwKn2h4PfL18Y9t3AZtI3/wn2t5b9vBP8ra9ZgMrBxjn+aQyHb2u\ntj0ZGA+cJWm87SWkktpn2z47l/K4BpiWc7kJWFzjOGGQa8kSHmHQ25s/LMsdDNyYx+T3keoW9bUe\nuFrSccBPbT8haSrwLmBjLm8ynNTpVHKHpL3ATlIZ6jHA07Yfz4/fDnwGuJE018Wtku4G7i76xGw/\nL2lHrrPzBHAKsC7vtz9xDiWVbSnP0yxJl5Pe18eQJujZ2mfbKXn9unycoaS8hbBf0VGEdnEl8Hdg\nAulM+A2TEtleIelB4EPAPZI+RZrJ63bbXyxwjAvLCwhKOrJSo1xb6AxSkbmZwCLgnH48l5XALOAx\nYLVtK31qF44T+APp+sR3gY9JOgG4Cni37V2SlpEK3/Ul4D7bc/sRbxjkYugptIsRwLN5/oB5pOJv\n/0PSicCOPNzyc9IQzP3ATElH5zZHqvic4n8GRkk6KS/PA36bx/RH2L6H1IFNqLDtP0llzytZTZpp\nbC6p06C/ceaCdl8Cpkg6hTR728vAbklvBT64n1g2AGf2PidJb5ZU6ewshNdFRxHaxc3AJZK2kIZr\nXq7QZhawTdJm0rwUy/Mvja4Bfi1pK3AfaVimJtuvkKprrpL0CPAa0EX60L077+/3VB7jXwZ09V7M\n7rPfXcCfgHfYfiiv63ec+drHDaSqsFtI82M/BqwgDWf1Wgr8StIa28+TfpF1Zz7OelI+Q9ivqB4b\nQgihqjijCCGEUFV0FCGEEKqKjiKEEEJV0VGEEEKoKjqKEEIIVUVHEUIIoaroKEIIIVT1X0r1NJY5\n0XuUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0edc0d3668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, _ = roc_curve(user_s['y_test'], user_s['y_pred'])\n",
    "roc_auc = auc(fpr, tpr)\n",
    "\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import model_lgbm as mlgbm\n",
    "\n",
    "path_to_train = './data/features/fat_train.csv'\n",
    "path_to_test = './data/features/fat_test.csv'\n",
    "path_to_sample = './data/sample_submission_kaggle.csv'\n",
    "path_to_submission = './data/submission2.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8')\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8')\n",
    "\n",
    "fs, pred, model = mlgbm.create_submission(train, test, path_to_sample, params, cols, path_to_submission, iterations=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('context_type', 1682),\n",
       " ('song_listened', 1533),\n",
       " ('context_type_cumulative_mean', 1361),\n",
       " ('ts_listen_hour_cumulative_mean', 1253),\n",
       " ('user_id', 1205),\n",
       " ('media_duration', 1183),\n",
       " ('ts_listen_dayofweek_cumulative_mean', 1072),\n",
       " ('platform_name_cumulative_mean', 1026),\n",
       " ('listen_type_cumulative_mean', 939),\n",
       " ('ExplicitLyrics_cumulative_mean', 803),\n",
       " ('release_date_year_cumulative_mean', 734),\n",
       " ('album_id_cumulative_mean', 726),\n",
       " ('genre_id_cumulative_mean', 720),\n",
       " ('release_date_decade_cumulative_mean', 713),\n",
       " ('Isrc_Country_cumulative_mean', 691),\n",
       " ('artist_id_cumulative_mean', 685),\n",
       " ('cumsum_is_listened_1', 654),\n",
       " ('bpm_min', 626),\n",
       " ('media_duration_min', 617),\n",
       " ('media_duration_std', 591),\n",
       " ('media_duration_max', 585),\n",
       " ('Isrc_Label_cumulative_mean', 579),\n",
       " ('bpm_std', 565),\n",
       " ('bpm_max', 563),\n",
       " ('gain_std', 559),\n",
       " ('gain_min', 546),\n",
       " ('gain_max', 546),\n",
       " ('rank_std', 535),\n",
       " ('media_duration_mean', 526),\n",
       " ('rank_mean', 516),\n",
       " ('rank_max', 506),\n",
       " ('rank_min', 499),\n",
       " ('bpm_mean', 488),\n",
       " ('cumsum_listen_type_1', 485),\n",
       " ('gain_mean', 476),\n",
       " ('cumsum_is_listened', 475),\n",
       " ('user_age', 455),\n",
       " ('release_date_listened_this_year', 394),\n",
       " ('was_is_listened_before', 383),\n",
       " ('cumsum_listen_type', 257),\n",
       " ('release_date_listened_this_decade', 201),\n",
       " ('country_percent', 200),\n",
       " ('genre_listened', 197),\n",
       " ('percent_listened', 185),\n",
       " ('explicit_lyrics_percent', 143),\n",
       " ('artist_listened', 125),\n",
       " ('album_listened', 101),\n",
       " ('user_gender', 96)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlgbm.feature_score(fs, cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.61458974417470558, 0.0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_submission_correlation = './data/vitalik_submission.csv'\n",
    "previous_submission = pd.read_csv(path_to_submission_correlation, encoding='utf-8')['is_listened']\n",
    "\n",
    "from scipy.stats.stats import pearsonr\n",
    "pearsonr(pred,previous_submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9344586787287571, 0.0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_submission_correlation = './data/submission.csv'\n",
    "previous_submission = pd.read_csv(path_to_submission_correlation, encoding='utf-8')['is_listened']\n",
    "\n",
    "from scipy.stats.stats import pearsonr\n",
    "pearsonr(pred,previous_submission)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}