AS_feature_track_API_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tsdaemon/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (10) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>TitleShort</th>\n",
       "      <th>TitleVersion</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>Link</th>\n",
       "      <th>Duration</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Preview</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>311270</td>\n",
       "      <td>Les hommes de la Paix</td>\n",
       "      <td>Les hommes de la Paix</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FR6V80627011</td>\n",
       "      <td>https://www.deezer.com/track/311270</td>\n",
       "      <td>291.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>343460.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-e.deezer.com/stream/e4312...</td>\n",
       "      <td>142.1</td>\n",
       "      <td>-14.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>315013</td>\n",
       "      <td>Le Recyclé</td>\n",
       "      <td>Le Recyclé</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FRP4P0600100</td>\n",
       "      <td>https://www.deezer.com/track/315013</td>\n",
       "      <td>230.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>276157.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-c.deezer.com/stream/cc7db...</td>\n",
       "      <td>97.0</td>\n",
       "      <td>-9.6</td>\n",
       "      <td>FR</td>\n",
       "      <td>P4P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>237198</td>\n",
       "      <td>Caressé moin</td>\n",
       "      <td>Caressé moin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FR6V80402545</td>\n",
       "      <td>https://www.deezer.com/track/237198</td>\n",
       "      <td>335.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>359138.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-b.deezer.com/stream/b1d13...</td>\n",
       "      <td>180.5</td>\n",
       "      <td>-9.6</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>518120</td>\n",
       "      <td>Station and other place</td>\n",
       "      <td>Station and other place</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FR6V80800863</td>\n",
       "      <td>https://www.deezer.com/track/518120</td>\n",
       "      <td>219.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>308744.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-d.deezer.com/stream/d9d65...</td>\n",
       "      <td>136.0</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>597708</td>\n",
       "      <td>Le sucre et le sel</td>\n",
       "      <td>Le sucre et le sel</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CAC220300117</td>\n",
       "      <td>https://www.deezer.com/track/597708</td>\n",
       "      <td>233.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292680.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-b.deezer.com/stream/b44d8...</td>\n",
       "      <td>166.0</td>\n",
       "      <td>-10.8</td>\n",
       "      <td>CA</td>\n",
       "      <td>C22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Id                    Title               TitleShort TitleVersion  \\\n",
       "0  311270    Les hommes de la Paix    Les hommes de la Paix          NaN   \n",
       "1  315013               Le Recyclé               Le Recyclé          NaN   \n",
       "2  237198             Caressé moin             Caressé moin          NaN   \n",
       "3  518120  Station and other place  Station and other place          NaN   \n",
       "4  597708       Le sucre et le sel       Le sucre et le sel          NaN   \n",
       "\n",
       "           Isrc                                 Link  Duration  TrackPosition  \\\n",
       "0  FR6V80627011  https://www.deezer.com/track/311270     291.0            7.0   \n",
       "1  FRP4P0600100  https://www.deezer.com/track/315013     230.0           10.0   \n",
       "2  FR6V80402545  https://www.deezer.com/track/237198     335.0            5.0   \n",
       "3  FR6V80800863  https://www.deezer.com/track/518120     219.0            1.0   \n",
       "4  CAC220300117  https://www.deezer.com/track/597708     233.0            9.0   \n",
       "\n",
       "   DiskNumber      Rank ExplicitLyrics  \\\n",
       "0         1.0  343460.0          False   \n",
       "1         1.0  276157.0          False   \n",
       "2         1.0  359138.0          False   \n",
       "3         1.0  308744.0          False   \n",
       "4         1.0  292680.0          False   \n",
       "\n",
       "                                             Preview    Bpm  Gain  \\\n",
       "0  http://e-cdn-preview-e.deezer.com/stream/e4312...  142.1 -14.9   \n",
       "1  http://e-cdn-preview-c.deezer.com/stream/cc7db...   97.0  -9.6   \n",
       "2  http://e-cdn-preview-b.deezer.com/stream/b1d13...  180.5  -9.6   \n",
       "3  http://e-cdn-preview-d.deezer.com/stream/d9d65...  136.0  -9.9   \n",
       "4  http://e-cdn-preview-b.deezer.com/stream/b44d8...  166.0 -10.8   \n",
       "\n",
       "  Isrc_Country Isrc_Label  \n",
       "0           FR        6V8  \n",
       "1           FR        P4P  \n",
       "2           FR        6V8  \n",
       "3           FR        6V8  \n",
       "4           CA        C22  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_columns', None)\n",
    "vals = pd.read_csv(\"./data/features/all_media_info.csv\", encoding='utf8')\n",
    "vals.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>TitleShort</th>\n",
       "      <th>TitleVersion</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>Link</th>\n",
       "      <th>Duration</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Preview</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>311270</td>\n",
       "      <td>Les hommes de la Paix</td>\n",
       "      <td>Les hommes de la Paix</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FR6V80627011</td>\n",
       "      <td>https://www.deezer.com/track/311270</td>\n",
       "      <td>291.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>343460.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-e.deezer.com/stream/e4312...</td>\n",
       "      <td>142.1</td>\n",
       "      <td>-14.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>315013</td>\n",
       "      <td>Le Recyclé</td>\n",
       "      <td>Le Recyclé</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FRP4P0600100</td>\n",
       "      <td>https://www.deezer.com/track/315013</td>\n",
       "      <td>230.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>276157.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-c.deezer.com/stream/cc7db...</td>\n",
       "      <td>97.0</td>\n",
       "      <td>-9.6</td>\n",
       "      <td>FR</td>\n",
       "      <td>P4P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>237198</td>\n",
       "      <td>Caressé moin</td>\n",
       "      <td>Caressé moin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FR6V80402545</td>\n",
       "      <td>https://www.deezer.com/track/237198</td>\n",
       "      <td>335.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>359138.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-b.deezer.com/stream/b1d13...</td>\n",
       "      <td>180.5</td>\n",
       "      <td>-9.6</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>518120</td>\n",
       "      <td>Station and other place</td>\n",
       "      <td>Station and other place</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FR6V80800863</td>\n",
       "      <td>https://www.deezer.com/track/518120</td>\n",
       "      <td>219.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>308744.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-d.deezer.com/stream/d9d65...</td>\n",
       "      <td>136.0</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>597708</td>\n",
       "      <td>Le sucre et le sel</td>\n",
       "      <td>Le sucre et le sel</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CAC220300117</td>\n",
       "      <td>https://www.deezer.com/track/597708</td>\n",
       "      <td>233.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292680.0</td>\n",
       "      <td>False</td>\n",
       "      <td>http://e-cdn-preview-b.deezer.com/stream/b44d8...</td>\n",
       "      <td>166.0</td>\n",
       "      <td>-10.8</td>\n",
       "      <td>CA</td>\n",
       "      <td>C22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Id                    Title               TitleShort TitleVersion  \\\n",
       "0  311270    Les hommes de la Paix    Les hommes de la Paix          NaN   \n",
       "1  315013               Le Recyclé               Le Recyclé          NaN   \n",
       "2  237198             Caressé moin             Caressé moin          NaN   \n",
       "3  518120  Station and other place  Station and other place          NaN   \n",
       "4  597708       Le sucre et le sel       Le sucre et le sel          NaN   \n",
       "\n",
       "           Isrc                                 Link  Duration  TrackPosition  \\\n",
       "0  FR6V80627011  https://www.deezer.com/track/311270     291.0            7.0   \n",
       "1  FRP4P0600100  https://www.deezer.com/track/315013     230.0           10.0   \n",
       "2  FR6V80402545  https://www.deezer.com/track/237198     335.0            5.0   \n",
       "3  FR6V80800863  https://www.deezer.com/track/518120     219.0            1.0   \n",
       "4  CAC220300117  https://www.deezer.com/track/597708     233.0            9.0   \n",
       "\n",
       "   DiskNumber      Rank ExplicitLyrics  \\\n",
       "0         1.0  343460.0          False   \n",
       "1         1.0  276157.0          False   \n",
       "2         1.0  359138.0          False   \n",
       "3         1.0  308744.0          False   \n",
       "4         1.0  292680.0          False   \n",
       "\n",
       "                                             Preview    Bpm  Gain  \\\n",
       "0  http://e-cdn-preview-e.deezer.com/stream/e4312...  142.1 -14.9   \n",
       "1  http://e-cdn-preview-c.deezer.com/stream/cc7db...   97.0  -9.6   \n",
       "2  http://e-cdn-preview-b.deezer.com/stream/b1d13...  180.5  -9.6   \n",
       "3  http://e-cdn-preview-d.deezer.com/stream/d9d65...  136.0  -9.9   \n",
       "4  http://e-cdn-preview-b.deezer.com/stream/b44d8...  166.0 -10.8   \n",
       "\n",
       "  Isrc_Country Isrc_Label  \n",
       "0           FR        6V8  \n",
       "1           FR        P4P  \n",
       "2           FR        6V8  \n",
       "3           FR        6V8  \n",
       "4           CA        C22  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vals['Isrc_Country'] = vals['Isrc'].str.slice(0,2)\n",
    "vals['Isrc_Label'] = vals['Isrc'].str.slice(2,5)\n",
    "vals.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vals.to_csv(\"./data/features/all_media_info.csv\", encoding='utf8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7578747</th>\n",
       "      <td>133549164</td>\n",
       "      <td>12641</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578748</th>\n",
       "      <td>133549164</td>\n",
       "      <td>10055</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578749</th>\n",
       "      <td>136334560</td>\n",
       "      <td>1029</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578750</th>\n",
       "      <td>136334560</td>\n",
       "      <td>4630</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578751</th>\n",
       "      <td>136334560</td>\n",
       "      <td>6467</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          media_id  user_id  is_listened\n",
       "7578747  133549164    12641            0\n",
       "7578748  133549164    10055            0\n",
       "7578749  136334560     1029            0\n",
       "7578750  136334560     4630            0\n",
       "7578751  136334560     6467            0"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8', usecols=['user_id', 'ts_listen', 'media_id', 'is_listened'])\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8', usecols=['user_id', 'ts_listen', 'media_id'])\n",
    "test['is_listened'] = 0\n",
    "\n",
    "df = pd.concat([train, test], ignore_index=True)\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anatoliy\\Anaconda2\\envs\\py35\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2717: DtypeWarning: Columns (10) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>311270</td>\n",
       "      <td>FR6V80627011</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>343460.0</td>\n",
       "      <td>False</td>\n",
       "      <td>142.1</td>\n",
       "      <td>-14.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>315013</td>\n",
       "      <td>FRP4P0600100</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>276157.0</td>\n",
       "      <td>False</td>\n",
       "      <td>97.0</td>\n",
       "      <td>-9.6</td>\n",
       "      <td>FR</td>\n",
       "      <td>P4P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>237198</td>\n",
       "      <td>FR6V80402545</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>359138.0</td>\n",
       "      <td>False</td>\n",
       "      <td>180.5</td>\n",
       "      <td>-9.6</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>518120</td>\n",
       "      <td>FR6V80800863</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>308744.0</td>\n",
       "      <td>False</td>\n",
       "      <td>136.0</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>597708</td>\n",
       "      <td>CAC220300117</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292680.0</td>\n",
       "      <td>False</td>\n",
       "      <td>166.0</td>\n",
       "      <td>-10.8</td>\n",
       "      <td>CA</td>\n",
       "      <td>C22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Id          Isrc  TrackPosition  DiskNumber      Rank ExplicitLyrics  \\\n",
       "0  311270  FR6V80627011            7.0         1.0  343460.0          False   \n",
       "1  315013  FRP4P0600100           10.0         1.0  276157.0          False   \n",
       "2  237198  FR6V80402545            5.0         1.0  359138.0          False   \n",
       "3  518120  FR6V80800863            1.0         1.0  308744.0          False   \n",
       "4  597708  CAC220300117            9.0         1.0  292680.0          False   \n",
       "\n",
       "     Bpm  Gain Isrc_Country Isrc_Label  \n",
       "0  142.1 -14.9           FR        6V8  \n",
       "1   97.0  -9.6           FR        P4P  \n",
       "2  180.5  -9.6           FR        6V8  \n",
       "3  136.0  -9.9           FR        6V8  \n",
       "4  166.0 -10.8           CA        C22  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_track_data = './data/features/all_media_info.csv'\n",
    "track_info = pd.read_csv(path_to_track_data, encoding='utf-8', usecols=['Id', 'TrackPosition', 'DiskNumber', 'Rank', 'Rank', 'ExplicitLyrics', 'Bpm', 'Gain', 'Isrc', 'Isrc_Country', 'Isrc_Label'])\n",
    "track_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "      <td>FR6V80400860</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>False</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "      <td>FR6V80405714</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>False</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "      <td>FR6V88900849</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>False</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>FR</td>\n",
       "      <td>6V8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>NLA200000110</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>False</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>NL</td>\n",
       "      <td>A20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "      <td>USSM16501030</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>False</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>US</td>\n",
       "      <td>SM1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   media_id  user_id  is_listened          Isrc  TrackPosition  DiskNumber  \\\n",
       "0    222606     9241            0  FR6V80400860           10.0         1.0   \n",
       "1    250467    16547            1  FR6V80405714            4.0         1.0   \n",
       "2    305197     7665            1  FR6V88900849            1.0         1.0   \n",
       "3    900502     1580            0  NLA200000110           11.0         1.0   \n",
       "4    542335     1812            1  USSM16501030            1.0         1.0   \n",
       "\n",
       "       Rank ExplicitLyrics    Bpm  Gain Isrc_Country Isrc_Label  \n",
       "0  292784.0          False  100.1 -10.3           FR        6V8  \n",
       "1  317348.0          False  160.2  -9.9           FR        6V8  \n",
       "2  309034.0          False   90.1 -17.7           FR        6V8  \n",
       "3  291331.0          False  124.9  -8.7           NL        A20  \n",
       "4  406872.0          False  120.0 -12.5           US        SM1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_joined = df.merge(track_info, how='left', left_on='media_id', right_on='Id')\n",
    "del df_joined['Id']\n",
    "df_joined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_joined[df_joined['Bpm'].isnull()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anatoliy\\Anaconda2\\envs\\py35\\lib\\site-packages\\ipykernel\\__main__.py:1: FutureWarning: convert_objects is deprecated.  Use the data-type specific converters pd.to_datetime, pd.to_timedelta and pd.to_numeric.\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\Anatoliy\\Anaconda2\\envs\\py35\\lib\\site-packages\\ipykernel\\__main__.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [media_id, user_id, is_listened, Isrc, TrackPosition, DiskNumber, Rank, ExplicitLyrics, Bpm, Gain, Isrc_Country, Isrc_Label]\n",
       "Index: []"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_joined[df_joined['Gain'].notnull()][df_joined['Gain'].convert_objects(convert_numeric=True).isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [media_id, user_id, is_listened, Isrc, TrackPosition, DiskNumber, Rank, ExplicitLyrics, Bpm, Gain, Isrc_Country, Isrc_Label]\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_joined['TrackPosition'] = df_joined['TrackPosition'].fillna(df_joined['TrackPosition'].mean())\n",
    "df_joined['DiskNumber'] = df_joined['DiskNumber'].fillna(1)\n",
    "df_joined['Rank'] = df_joined['Rank'].fillna(df_joined['Rank'].mean())\n",
    "df_joined['ExplicitLyrics'] = df_joined['ExplicitLyrics'].fillna(False)\n",
    "df_joined['Bpm'] = df_joined['Bpm'].fillna(df_joined['Bpm'].mean())\n",
    "df_joined['Gain'] = df_joined['Gain'].fillna(df_joined['Gain'].mean())\n",
    "df_joined['Isrc_Country'] = df_joined['Isrc_Country'].fillna(\"unk\").map(str)\n",
    "df_joined['Isrc_Label'] = df_joined['Isrc_Label'].fillna(\"unk\").map(str)\n",
    "df_joined[df_joined['Isrc_Country'].isnull()].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>Isrc</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "      <td>FR6V80400860</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "      <td>FR6V80405714</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "      <td>FR6V88900849</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>NLA200000110</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "      <td>USSM16501030</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   media_id  user_id  is_listened          Isrc  TrackPosition  DiskNumber  \\\n",
       "0    222606     9241            0  FR6V80400860           10.0         1.0   \n",
       "1    250467    16547            1  FR6V80405714            4.0         1.0   \n",
       "2    305197     7665            1  FR6V88900849            1.0         1.0   \n",
       "3    900502     1580            0  NLA200000110           11.0         1.0   \n",
       "4    542335     1812            1  USSM16501030            1.0         1.0   \n",
       "\n",
       "       Rank  ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  \n",
       "0  292784.0               0  100.1 -10.3            54        2091  \n",
       "1  317348.0               0  160.2  -9.9            54        2091  \n",
       "2  309034.0               0   90.1 -17.7            54        2091  \n",
       "3  291331.0               0  124.9  -8.7           106        2925  \n",
       "4  406872.0               0  120.0 -12.5           145        8228  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "le_country = preprocessing.LabelEncoder()\n",
    "le_country.fit(df_joined['Isrc_Country'])\n",
    "df_joined['Isrc_Country'] = le_country.transform(df_joined['Isrc_Country'])\n",
    "\n",
    "le_lyrics = preprocessing.LabelEncoder()\n",
    "le_lyrics.fit(df_joined['ExplicitLyrics'])\n",
    "df_joined['ExplicitLyrics'] = le_lyrics.transform(df_joined['ExplicitLyrics'])\n",
    "\n",
    "le_label = preprocessing.LabelEncoder()\n",
    "le_label.fit(df_joined['Isrc_Label'])\n",
    "df_joined['Isrc_Label'] = le_label.transform(df_joined['Isrc_Label'])\n",
    "df_joined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del df_joined['Isrc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>118.968509</td>\n",
       "      <td>-10.009290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>119.150188</td>\n",
       "      <td>-10.479189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>123.398457</td>\n",
       "      <td>-9.960308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>127.367065</td>\n",
       "      <td>-11.012104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>117.064893</td>\n",
       "      <td>-8.140530</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id         Bpm       Gain\n",
       "0        0  118.968509 -10.009290\n",
       "1        1  119.150188 -10.479189\n",
       "2        2  123.398457  -9.960308\n",
       "3        3  127.367065 -11.012104\n",
       "4        4  117.064893  -8.140530"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_user_mean = df_joined[df_joined['is_listened']==1].groupby('user_id', as_index=False).agg({'Bpm':'mean', 'Gain':'mean'})\n",
    "by_user_mean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19716, 19918)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(by_user_mean), len(df_joined['user_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "      <th>BpmByUserMean</th>\n",
       "      <th>GainByUserMean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.356478</td>\n",
       "      <td>-11.257516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.514286</td>\n",
       "      <td>-8.946429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>124.553876</td>\n",
       "      <td>-11.481208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "      <td>121.802594</td>\n",
       "      <td>-8.284311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "      <td>123.736146</td>\n",
       "      <td>-9.279614</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   media_id  user_id  is_listened  TrackPosition  DiskNumber      Rank  \\\n",
       "0    222606     9241            0           10.0         1.0  292784.0   \n",
       "1    250467    16547            1            4.0         1.0  317348.0   \n",
       "2    305197     7665            1            1.0         1.0  309034.0   \n",
       "3    900502     1580            0           11.0         1.0  291331.0   \n",
       "4    542335     1812            1            1.0         1.0  406872.0   \n",
       "\n",
       "   ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  BpmByUserMean  \\\n",
       "0               0  100.1 -10.3            54        2091     122.356478   \n",
       "1               0  160.2  -9.9            54        2091     122.514286   \n",
       "2               0   90.1 -17.7            54        2091     124.553876   \n",
       "3               0  124.9  -8.7           106        2925     121.802594   \n",
       "4               0  120.0 -12.5           145        8228     123.736146   \n",
       "\n",
       "   GainByUserMean  \n",
       "0      -11.257516  \n",
       "1       -8.946429  \n",
       "2      -11.481208  \n",
       "3       -8.284311  \n",
       "4       -9.279614  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_joined = df_joined.join(by_user_mean, how='left', on='user_id', rsuffix='ByUserMean')\n",
    "del df_joined['user_idByUserMean']\n",
    "df_joined['GainByUserMean'].fillna(df_joined['Gain'].mean())\n",
    "df_joined['BpmByUserMean'].fillna(df_joined['Bpm'].mean())\n",
    "df_joined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "      <th>BpmByUserMean</th>\n",
       "      <th>GainByUserMean</th>\n",
       "      <th>GainCenter</th>\n",
       "      <th>BpmCenter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.356478</td>\n",
       "      <td>-11.257516</td>\n",
       "      <td>0.957516</td>\n",
       "      <td>-22.256478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.514286</td>\n",
       "      <td>-8.946429</td>\n",
       "      <td>-0.953571</td>\n",
       "      <td>37.685714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>124.553876</td>\n",
       "      <td>-11.481208</td>\n",
       "      <td>-6.218792</td>\n",
       "      <td>-34.453876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "      <td>121.802594</td>\n",
       "      <td>-8.284311</td>\n",
       "      <td>-0.415689</td>\n",
       "      <td>3.097406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "      <td>123.736146</td>\n",
       "      <td>-9.279614</td>\n",
       "      <td>-3.220386</td>\n",
       "      <td>-3.736146</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   media_id  user_id  is_listened  TrackPosition  DiskNumber      Rank  \\\n",
       "0    222606     9241            0           10.0         1.0  292784.0   \n",
       "1    250467    16547            1            4.0         1.0  317348.0   \n",
       "2    305197     7665            1            1.0         1.0  309034.0   \n",
       "3    900502     1580            0           11.0         1.0  291331.0   \n",
       "4    542335     1812            1            1.0         1.0  406872.0   \n",
       "\n",
       "   ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  BpmByUserMean  \\\n",
       "0               0  100.1 -10.3            54        2091     122.356478   \n",
       "1               0  160.2  -9.9            54        2091     122.514286   \n",
       "2               0   90.1 -17.7            54        2091     124.553876   \n",
       "3               0  124.9  -8.7           106        2925     121.802594   \n",
       "4               0  120.0 -12.5           145        8228     123.736146   \n",
       "\n",
       "   GainByUserMean  GainCenter  BpmCenter  \n",
       "0      -11.257516    0.957516 -22.256478  \n",
       "1       -8.946429   -0.953571  37.685714  \n",
       "2      -11.481208   -6.218792 -34.453876  \n",
       "3       -8.284311   -0.415689   3.097406  \n",
       "4       -9.279614   -3.220386  -3.736146  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_joined['GainCenter'] = df_joined['Gain'] - df_joined['GainByUserMean']\n",
    "df_joined['BpmCenter'] = df_joined['Bpm'] - df_joined['BpmByUserMean']\n",
    "df_joined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Country_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  Isrc_Country  Isrc_Country_count\n",
       "0        0            13                  15\n",
       "1        0            15                 120\n",
       "2        0            16                  55\n",
       "3        0            20                  60\n",
       "4        0            24                   7"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_user_country = df_joined[df_joined['is_listened']==1].groupby(['user_id', 'Isrc_Country'], as_index=False).agg({'media_id':'count'})\n",
    "by_user_country['Isrc_Country_count'] = by_user_country['media_id']\n",
    "del by_user_country['media_id']\n",
    "by_user_country.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "      <th>BpmByUserMean</th>\n",
       "      <th>GainByUserMean</th>\n",
       "      <th>GainCenter</th>\n",
       "      <th>BpmCenter</th>\n",
       "      <th>Isrc_Country_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.356478</td>\n",
       "      <td>-11.257516</td>\n",
       "      <td>0.957516</td>\n",
       "      <td>-22.256478</td>\n",
       "      <td>89.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.514286</td>\n",
       "      <td>-8.946429</td>\n",
       "      <td>-0.953571</td>\n",
       "      <td>37.685714</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>124.553876</td>\n",
       "      <td>-11.481208</td>\n",
       "      <td>-6.218792</td>\n",
       "      <td>-34.453876</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "      <td>121.802594</td>\n",
       "      <td>-8.284311</td>\n",
       "      <td>-0.415689</td>\n",
       "      <td>3.097406</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "      <td>123.736146</td>\n",
       "      <td>-9.279614</td>\n",
       "      <td>-3.220386</td>\n",
       "      <td>-3.736146</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   media_id  user_id  is_listened  TrackPosition  DiskNumber      Rank  \\\n",
       "0    222606     9241            0           10.0         1.0  292784.0   \n",
       "1    250467    16547            1            4.0         1.0  317348.0   \n",
       "2    305197     7665            1            1.0         1.0  309034.0   \n",
       "3    900502     1580            0           11.0         1.0  291331.0   \n",
       "4    542335     1812            1            1.0         1.0  406872.0   \n",
       "\n",
       "   ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  BpmByUserMean  \\\n",
       "0               0  100.1 -10.3            54        2091     122.356478   \n",
       "1               0  160.2  -9.9            54        2091     122.514286   \n",
       "2               0   90.1 -17.7            54        2091     124.553876   \n",
       "3               0  124.9  -8.7           106        2925     121.802594   \n",
       "4               0  120.0 -12.5           145        8228     123.736146   \n",
       "\n",
       "   GainByUserMean  GainCenter  BpmCenter  Isrc_Country_count  \n",
       "0      -11.257516    0.957516 -22.256478                89.0  \n",
       "1       -8.946429   -0.953571  37.685714                18.0  \n",
       "2      -11.481208   -6.218792 -34.453876                32.0  \n",
       "3       -8.284311   -0.415689   3.097406                 2.0  \n",
       "4       -9.279614   -3.220386  -3.736146               349.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_joined2 = df_joined.merge(by_user_country, how='left', on=['user_id', 'Isrc_Country'])\n",
    "df_joined2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "      <th>BpmByUserMean</th>\n",
       "      <th>GainByUserMean</th>\n",
       "      <th>GainCenter</th>\n",
       "      <th>BpmCenter</th>\n",
       "      <th>Isrc_Country_count</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.356478</td>\n",
       "      <td>-11.257516</td>\n",
       "      <td>0.957516</td>\n",
       "      <td>-22.256478</td>\n",
       "      <td>89.0</td>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.514286</td>\n",
       "      <td>-8.946429</td>\n",
       "      <td>-0.953571</td>\n",
       "      <td>37.685714</td>\n",
       "      <td>18.0</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>124.553876</td>\n",
       "      <td>-11.481208</td>\n",
       "      <td>-6.218792</td>\n",
       "      <td>-34.453876</td>\n",
       "      <td>32.0</td>\n",
       "      <td>298.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "      <td>121.802594</td>\n",
       "      <td>-8.284311</td>\n",
       "      <td>-0.415689</td>\n",
       "      <td>3.097406</td>\n",
       "      <td>2.0</td>\n",
       "      <td>631.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "      <td>123.736146</td>\n",
       "      <td>-9.279614</td>\n",
       "      <td>-3.220386</td>\n",
       "      <td>-3.736146</td>\n",
       "      <td>349.0</td>\n",
       "      <td>932.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   media_id  user_id  is_listened  TrackPosition  DiskNumber      Rank  \\\n",
       "0    222606     9241            0           10.0         1.0  292784.0   \n",
       "1    250467    16547            1            4.0         1.0  317348.0   \n",
       "2    305197     7665            1            1.0         1.0  309034.0   \n",
       "3    900502     1580            0           11.0         1.0  291331.0   \n",
       "4    542335     1812            1            1.0         1.0  406872.0   \n",
       "\n",
       "   ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  BpmByUserMean  \\\n",
       "0               0  100.1 -10.3            54        2091     122.356478   \n",
       "1               0  160.2  -9.9            54        2091     122.514286   \n",
       "2               0   90.1 -17.7            54        2091     124.553876   \n",
       "3               0  124.9  -8.7           106        2925     121.802594   \n",
       "4               0  120.0 -12.5           145        8228     123.736146   \n",
       "\n",
       "   GainByUserMean  GainCenter  BpmCenter  Isrc_Country_count  Count  \n",
       "0      -11.257516    0.957516 -22.256478                89.0  153.0  \n",
       "1       -8.946429   -0.953571  37.685714                18.0   23.0  \n",
       "2      -11.481208   -6.218792 -34.453876                32.0  298.0  \n",
       "3       -8.284311   -0.415689   3.097406                 2.0  631.0  \n",
       "4       -9.279614   -3.220386  -3.736146               349.0  932.0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_user = df_joined[df_joined['is_listened']==1].groupby('user_id', as_index=False).agg({'media_id':'count'})\n",
    "by_user['Count'] = by_user['media_id']\n",
    "del by_user['media_id']\n",
    "by_user.head()\n",
    "df_joined3 = df_joined2.merge(by_user, how='left', on='user_id')\n",
    "df_joined3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_joined3['Isrc_Country_count'] = df_joined3['Isrc_Country_count']/df_joined3['Count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_api_track_data_train = './data/features/api_data_track.train.txt'\n",
    "path_to_api_track_data_test = './data/features/api_data_track.test.txt'\n",
    "\n",
    "newcols = ['TrackPosition','DiskNumber','Rank','ExplicitLyrics', \n",
    "           'Bpm','Gain', 'Isrc_Country','Isrc_Label','BpmByUserMean',\n",
    "           'GainByUserMean','GainCenter', 'BpmCenter', 'Isrc_Country_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dt = df_joined3.loc[range(len(train))][newcols]\n",
    "train_dt.to_csv(path_to_api_track_data_train, encoding='utf-8', index=False)\n",
    "\n",
    "test_dt = df_joined3.loc[range(len(train), len(train)+len(test))][newcols]\n",
    "test_dt.to_csv(path_to_api_track_data_test, encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------\n",
    "\n",
    "Для Ореста"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1480597215</td>\n",
       "      <td>222606</td>\n",
       "      <td>9241</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1480544735</td>\n",
       "      <td>250467</td>\n",
       "      <td>16547</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1479563953</td>\n",
       "      <td>305197</td>\n",
       "      <td>7665</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1480152098</td>\n",
       "      <td>900502</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1478368974</td>\n",
       "      <td>542335</td>\n",
       "      <td>1812</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ts_listen  media_id  user_id  is_listened\n",
       "0  1480597215    222606     9241            0\n",
       "1  1480544735    250467    16547            1\n",
       "2  1479563953    305197     7665            1\n",
       "3  1480152098    900502     1580            0\n",
       "4  1478368974    542335     1812            1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8', usecols=['user_id', 'ts_listen', 'media_id', 'is_listened'])\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8', usecols=['user_id', 'ts_listen', 'media_id'])\n",
    "test['is_listened'] = 0\n",
    "\n",
    "df = pd.concat([train, test], ignore_index=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>292784.0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>317348.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>309034.0</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>291331.0</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>406872.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   TrackPosition  DiskNumber      Rank  ExplicitLyrics    Bpm  Gain  \\\n",
       "0           10.0         1.0  292784.0               0  100.1 -10.3   \n",
       "1            4.0         1.0  317348.0               0  160.2  -9.9   \n",
       "2            1.0         1.0  309034.0               0   90.1 -17.7   \n",
       "3           11.0         1.0  291331.0               0  124.9  -8.7   \n",
       "4            1.0         1.0  406872.0               0  120.0 -12.5   \n",
       "\n",
       "   Isrc_Country  Isrc_Label  \n",
       "0            54        2091  \n",
       "1            54        2091  \n",
       "2            54        2091  \n",
       "3           106        2925  \n",
       "4           145        8228  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dt = pd.read_csv(path_to_api_track_data_train, encoding='utf-8', \n",
    "                       usecols=['TrackPosition','DiskNumber','Rank','ExplicitLyrics', 'Bpm','Gain', 'Isrc_Country','Isrc_Label'])\n",
    "\n",
    "test_dt = pd.read_csv(path_to_api_track_data_test, encoding='utf-8', \n",
    "                       usecols=['TrackPosition','DiskNumber','Rank','ExplicitLyrics', 'Bpm','Gain', 'Isrc_Country','Isrc_Label'])\n",
    "\n",
    "dt = pd.concat([train_dt, test_dt], ignore_index=True)\n",
    "dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>media_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>TrackPosition</th>\n",
       "      <th>DiskNumber</th>\n",
       "      <th>Rank</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>682493</th>\n",
       "      <td>1477939775</td>\n",
       "      <td>122437314</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>872156.0</td>\n",
       "      <td>0</td>\n",
       "      <td>112.043</td>\n",
       "      <td>-7.3</td>\n",
       "      <td>44</td>\n",
       "      <td>1511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4454939</th>\n",
       "      <td>1477982531</td>\n",
       "      <td>123339574</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>516548.0</td>\n",
       "      <td>0</td>\n",
       "      <td>160.000</td>\n",
       "      <td>-12.7</td>\n",
       "      <td>54</td>\n",
       "      <td>8701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5354523</th>\n",
       "      <td>1477982561</td>\n",
       "      <td>131340580</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>379211.0</td>\n",
       "      <td>0</td>\n",
       "      <td>141.000</td>\n",
       "      <td>-7.8</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5899458</th>\n",
       "      <td>1477982582</td>\n",
       "      <td>131576046</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>440223.0</td>\n",
       "      <td>0</td>\n",
       "      <td>192.000</td>\n",
       "      <td>-7.8</td>\n",
       "      <td>145</td>\n",
       "      <td>7922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4716574</th>\n",
       "      <td>1477982792</td>\n",
       "      <td>129011934</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999260.0</td>\n",
       "      <td>0</td>\n",
       "      <td>186.000</td>\n",
       "      <td>-9.1</td>\n",
       "      <td>119</td>\n",
       "      <td>8760</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ts_listen   media_id  user_id  is_listened  TrackPosition  \\\n",
       "682493   1477939775  122437314        0            1            1.0   \n",
       "4454939  1477982531  123339574        0            0            1.0   \n",
       "5354523  1477982561  131340580        0            0            1.0   \n",
       "5899458  1477982582  131576046        0            1            1.0   \n",
       "4716574  1477982792  129011934        0            1            1.0   \n",
       "\n",
       "         DiskNumber      Rank  ExplicitLyrics      Bpm  Gain  Isrc_Country  \\\n",
       "682493          1.0  872156.0               0  112.043  -7.3            44   \n",
       "4454939         1.0  516548.0               0  160.000 -12.7            54   \n",
       "5354523         1.0  379211.0               0  141.000  -7.8           145   \n",
       "5899458         1.0  440223.0               0  192.000  -7.8           145   \n",
       "4716574         1.0  999260.0               0  186.000  -9.1           119   \n",
       "\n",
       "         Isrc_Label  \n",
       "682493         1511  \n",
       "4454939        8701  \n",
       "5354523        8228  \n",
       "5899458        7922  \n",
       "4716574        8760  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([df, dt], axis=1)\n",
    "df = df.sort_values(by=['user_id', 'ts_listen'], axis=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('./data/features/api_features_concat.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre_id</th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>media_id</th>\n",
       "      <th>album_id</th>\n",
       "      <th>context_type</th>\n",
       "      <th>release_date</th>\n",
       "      <th>platform_name</th>\n",
       "      <th>platform_family</th>\n",
       "      <th>media_duration</th>\n",
       "      <th>listen_type</th>\n",
       "      <th>...</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "      <th>BpmByUserMean</th>\n",
       "      <th>GainByUserMean</th>\n",
       "      <th>GainCenter</th>\n",
       "      <th>BpmCenter</th>\n",
       "      <th>Isrc_Country_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25471</td>\n",
       "      <td>1480597215</td>\n",
       "      <td>222606</td>\n",
       "      <td>41774</td>\n",
       "      <td>12</td>\n",
       "      <td>20040704</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>223</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>100.1</td>\n",
       "      <td>-10.3</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.356478</td>\n",
       "      <td>-11.257516</td>\n",
       "      <td>0.957516</td>\n",
       "      <td>-22.256478</td>\n",
       "      <td>0.581699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25571</td>\n",
       "      <td>1480544735</td>\n",
       "      <td>250467</td>\n",
       "      <td>43941</td>\n",
       "      <td>0</td>\n",
       "      <td>20060301</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>171</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>160.2</td>\n",
       "      <td>-9.9</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>122.514286</td>\n",
       "      <td>-8.946429</td>\n",
       "      <td>-0.953571</td>\n",
       "      <td>37.685714</td>\n",
       "      <td>0.782609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>1479563953</td>\n",
       "      <td>305197</td>\n",
       "      <td>48078</td>\n",
       "      <td>1</td>\n",
       "      <td>20140714</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>149</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>90.1</td>\n",
       "      <td>-17.7</td>\n",
       "      <td>54</td>\n",
       "      <td>2091</td>\n",
       "      <td>124.553876</td>\n",
       "      <td>-11.481208</td>\n",
       "      <td>-6.218792</td>\n",
       "      <td>-34.453876</td>\n",
       "      <td>0.107383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1480152098</td>\n",
       "      <td>900502</td>\n",
       "      <td>71521</td>\n",
       "      <td>0</td>\n",
       "      <td>20001030</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>240</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>124.9</td>\n",
       "      <td>-8.7</td>\n",
       "      <td>106</td>\n",
       "      <td>2925</td>\n",
       "      <td>121.802594</td>\n",
       "      <td>-8.284311</td>\n",
       "      <td>-0.415689</td>\n",
       "      <td>3.097406</td>\n",
       "      <td>0.003170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1478368974</td>\n",
       "      <td>542335</td>\n",
       "      <td>71718</td>\n",
       "      <td>0</td>\n",
       "      <td>20080215</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>-12.5</td>\n",
       "      <td>145</td>\n",
       "      <td>8228</td>\n",
       "      <td>123.736146</td>\n",
       "      <td>-9.279614</td>\n",
       "      <td>-3.220386</td>\n",
       "      <td>-3.736146</td>\n",
       "      <td>0.374464</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   genre_id   ts_listen  media_id  album_id  context_type  release_date  \\\n",
       "0     25471  1480597215    222606     41774            12      20040704   \n",
       "1     25571  1480544735    250467     43941             0      20060301   \n",
       "2        16  1479563953    305197     48078             1      20140714   \n",
       "3         7  1480152098    900502     71521             0      20001030   \n",
       "4         7  1478368974    542335     71718             0      20080215   \n",
       "\n",
       "   platform_name  platform_family  media_duration  listen_type  \\\n",
       "0              1                0             223            0   \n",
       "1              2                1             171            0   \n",
       "2              2                1             149            1   \n",
       "3              0                0             240            0   \n",
       "4              0                0             150            0   \n",
       "\n",
       "          ...          ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  \\\n",
       "0         ...                       0  100.1 -10.3            54        2091   \n",
       "1         ...                       0  160.2  -9.9            54        2091   \n",
       "2         ...                       0   90.1 -17.7            54        2091   \n",
       "3         ...                       0  124.9  -8.7           106        2925   \n",
       "4         ...                       0  120.0 -12.5           145        8228   \n",
       "\n",
       "   BpmByUserMean  GainByUserMean  GainCenter  BpmCenter  Isrc_Country_count  \n",
       "0     122.356478      -11.257516    0.957516 -22.256478            0.581699  \n",
       "1     122.514286       -8.946429   -0.953571  37.685714            0.782609  \n",
       "2     124.553876      -11.481208   -6.218792 -34.453876            0.107383  \n",
       "3     121.802594       -8.284311   -0.415689   3.097406            0.003170  \n",
       "4     123.736146       -9.279614   -3.220386  -3.736146            0.374464  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import validation as v\n",
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8')\n",
    "\n",
    "train = pd.concat([train, pd.read_csv(path_to_api_track_data_train, encoding='utf-8')], axis=1)\n",
    "#train = train[train['is_listened']==1]\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_folds = './data/lightgbm/api'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1\n"
     ]
    }
   ],
   "source": [
    "v.generate_validation_folds(train, path_to_folds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import model_lgbm as mlgbm\n",
    "\n",
    "cols = [\n",
    "        'genre_id', 'media_id', 'album_id', 'context_type',\n",
    "        'release_date', 'platform_name', 'platform_family', 'media_duration',\n",
    "        'listen_type', 'user_gender', 'user_id', 'artist_id', 'user_age', \n",
    "       ] + newcols\n",
    "\n",
    "params = {\n",
    "    'application':'binary',\n",
    "    'num_leaves': 20,\n",
    "    'max_depth': 15,\n",
    "    'feature_fraction': 0.7,\n",
    "    'bagging_fraction': 0.7,\n",
    "    'max_bin': 20,\n",
    "    'metric': 'auc',\n",
    "    'verbose': 1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttraining's auc: 0.673128\tvalid_1's auc: 0.590241\n",
      "[2]\ttraining's auc: 0.681825\tvalid_1's auc: 0.59497\n",
      "[3]\ttraining's auc: 0.680909\tvalid_1's auc: 0.599964\n",
      "[4]\ttraining's auc: 0.683584\tvalid_1's auc: 0.600678\n",
      "[5]\ttraining's auc: 0.684154\tvalid_1's auc: 0.600209\n",
      "[6]\ttraining's auc: 0.685329\tvalid_1's auc: 0.599678\n",
      "[7]\ttraining's auc: 0.685306\tvalid_1's auc: 0.599636\n",
      "[8]\ttraining's auc: 0.685796\tvalid_1's auc: 0.601696\n",
      "[9]\ttraining's auc: 0.686488\tvalid_1's auc: 0.600083\n",
      "[10]\ttraining's auc: 0.688109\tvalid_1's auc: 0.599247\n",
      "[11]\ttraining's auc: 0.688613\tvalid_1's auc: 0.599462\n",
      "[12]\ttraining's auc: 0.689478\tvalid_1's auc: 0.598761\n",
      "[13]\ttraining's auc: 0.689827\tvalid_1's auc: 0.600514\n",
      "[14]\ttraining's auc: 0.690675\tvalid_1's auc: 0.599702\n",
      "[15]\ttraining's auc: 0.691394\tvalid_1's auc: 0.599969\n",
      "[16]\ttraining's auc: 0.691644\tvalid_1's auc: 0.601591\n",
      "[17]\ttraining's auc: 0.691889\tvalid_1's auc: 0.602576\n",
      "[18]\ttraining's auc: 0.692878\tvalid_1's auc: 0.60233\n",
      "[19]\ttraining's auc: 0.693195\tvalid_1's auc: 0.603348\n",
      "[20]\ttraining's auc: 0.693739\tvalid_1's auc: 0.603197\n",
      "[21]\ttraining's auc: 0.694145\tvalid_1's auc: 0.603146\n",
      "[22]\ttraining's auc: 0.694515\tvalid_1's auc: 0.603393\n",
      "[23]\ttraining's auc: 0.695408\tvalid_1's auc: 0.60402\n",
      "[24]\ttraining's auc: 0.695889\tvalid_1's auc: 0.603907\n",
      "[25]\ttraining's auc: 0.696343\tvalid_1's auc: 0.604298\n",
      "[26]\ttraining's auc: 0.697103\tvalid_1's auc: 0.605134\n",
      "[27]\ttraining's auc: 0.697422\tvalid_1's auc: 0.60524\n",
      "[28]\ttraining's auc: 0.698329\tvalid_1's auc: 0.605918\n",
      "[29]\ttraining's auc: 0.698749\tvalid_1's auc: 0.606529\n",
      "[30]\ttraining's auc: 0.699194\tvalid_1's auc: 0.606909\n",
      "[31]\ttraining's auc: 0.699632\tvalid_1's auc: 0.607297\n",
      "[32]\ttraining's auc: 0.699951\tvalid_1's auc: 0.607211\n",
      "[33]\ttraining's auc: 0.701001\tvalid_1's auc: 0.608413\n",
      "[34]\ttraining's auc: 0.701305\tvalid_1's auc: 0.608664\n",
      "[35]\ttraining's auc: 0.70181\tvalid_1's auc: 0.609152\n",
      "[36]\ttraining's auc: 0.702532\tvalid_1's auc: 0.610125\n",
      "[37]\ttraining's auc: 0.702873\tvalid_1's auc: 0.610339\n",
      "[38]\ttraining's auc: 0.703247\tvalid_1's auc: 0.610675\n",
      "[39]\ttraining's auc: 0.703854\tvalid_1's auc: 0.611993\n",
      "[40]\ttraining's auc: 0.704412\tvalid_1's auc: 0.612327\n",
      "[41]\ttraining's auc: 0.704915\tvalid_1's auc: 0.613057\n",
      "[42]\ttraining's auc: 0.705192\tvalid_1's auc: 0.613086\n",
      "[43]\ttraining's auc: 0.705643\tvalid_1's auc: 0.613445\n",
      "[44]\ttraining's auc: 0.706134\tvalid_1's auc: 0.614221\n",
      "[45]\ttraining's auc: 0.706566\tvalid_1's auc: 0.614995\n",
      "[46]\ttraining's auc: 0.706933\tvalid_1's auc: 0.615144\n",
      "[47]\ttraining's auc: 0.707386\tvalid_1's auc: 0.615286\n",
      "[48]\ttraining's auc: 0.707713\tvalid_1's auc: 0.615597\n",
      "[49]\ttraining's auc: 0.708173\tvalid_1's auc: 0.615297\n",
      "[50]\ttraining's auc: 0.70843\tvalid_1's auc: 0.615567\n",
      "[51]\ttraining's auc: 0.708672\tvalid_1's auc: 0.615951\n",
      "[52]\ttraining's auc: 0.709164\tvalid_1's auc: 0.616252\n",
      "[53]\ttraining's auc: 0.709511\tvalid_1's auc: 0.616443\n",
      "[54]\ttraining's auc: 0.709753\tvalid_1's auc: 0.616551\n",
      "[55]\ttraining's auc: 0.709939\tvalid_1's auc: 0.617279\n",
      "[56]\ttraining's auc: 0.710295\tvalid_1's auc: 0.616957\n",
      "[57]\ttraining's auc: 0.710427\tvalid_1's auc: 0.617165\n",
      "[58]\ttraining's auc: 0.710874\tvalid_1's auc: 0.617513\n",
      "[59]\ttraining's auc: 0.711288\tvalid_1's auc: 0.617602\n",
      "[60]\ttraining's auc: 0.711481\tvalid_1's auc: 0.617852\n",
      "[61]\ttraining's auc: 0.711607\tvalid_1's auc: 0.617949\n",
      "[62]\ttraining's auc: 0.711884\tvalid_1's auc: 0.618479\n",
      "[63]\ttraining's auc: 0.712136\tvalid_1's auc: 0.618209\n",
      "[64]\ttraining's auc: 0.712369\tvalid_1's auc: 0.618391\n",
      "[65]\ttraining's auc: 0.712613\tvalid_1's auc: 0.618679\n",
      "[66]\ttraining's auc: 0.712867\tvalid_1's auc: 0.618946\n",
      "[67]\ttraining's auc: 0.713036\tvalid_1's auc: 0.61911\n",
      "[68]\ttraining's auc: 0.713216\tvalid_1's auc: 0.619287\n",
      "[69]\ttraining's auc: 0.713384\tvalid_1's auc: 0.619217\n",
      "[70]\ttraining's auc: 0.713588\tvalid_1's auc: 0.61935\n",
      "[71]\ttraining's auc: 0.713869\tvalid_1's auc: 0.619874\n",
      "[72]\ttraining's auc: 0.714063\tvalid_1's auc: 0.62089\n",
      "[73]\ttraining's auc: 0.714263\tvalid_1's auc: 0.620902\n",
      "[74]\ttraining's auc: 0.714446\tvalid_1's auc: 0.621092\n",
      "[75]\ttraining's auc: 0.714791\tvalid_1's auc: 0.621252\n",
      "[76]\ttraining's auc: 0.714988\tvalid_1's auc: 0.621317\n",
      "[77]\ttraining's auc: 0.715175\tvalid_1's auc: 0.62157\n",
      "[78]\ttraining's auc: 0.715355\tvalid_1's auc: 0.621961\n",
      "[79]\ttraining's auc: 0.715507\tvalid_1's auc: 0.622038\n",
      "[80]\ttraining's auc: 0.715732\tvalid_1's auc: 0.621731\n",
      "[81]\ttraining's auc: 0.715869\tvalid_1's auc: 0.621762\n",
      "[82]\ttraining's auc: 0.715992\tvalid_1's auc: 0.621851\n",
      "[83]\ttraining's auc: 0.7161\tvalid_1's auc: 0.62207\n",
      "[84]\ttraining's auc: 0.716309\tvalid_1's auc: 0.62212\n",
      "[85]\ttraining's auc: 0.716526\tvalid_1's auc: 0.622304\n",
      "[86]\ttraining's auc: 0.716746\tvalid_1's auc: 0.62261\n",
      "[87]\ttraining's auc: 0.716874\tvalid_1's auc: 0.622744\n",
      "[88]\ttraining's auc: 0.717116\tvalid_1's auc: 0.622887\n",
      "[89]\ttraining's auc: 0.717303\tvalid_1's auc: 0.623009\n",
      "[90]\ttraining's auc: 0.717545\tvalid_1's auc: 0.623073\n",
      "[91]\ttraining's auc: 0.717738\tvalid_1's auc: 0.623278\n",
      "[92]\ttraining's auc: 0.717818\tvalid_1's auc: 0.623333\n",
      "[93]\ttraining's auc: 0.717908\tvalid_1's auc: 0.623326\n",
      "[94]\ttraining's auc: 0.71823\tvalid_1's auc: 0.623489\n",
      "[95]\ttraining's auc: 0.718428\tvalid_1's auc: 0.623532\n",
      "[96]\ttraining's auc: 0.718658\tvalid_1's auc: 0.623604\n",
      "[97]\ttraining's auc: 0.71889\tvalid_1's auc: 0.623744\n",
      "[98]\ttraining's auc: 0.719204\tvalid_1's auc: 0.623885\n",
      "[99]\ttraining's auc: 0.719342\tvalid_1's auc: 0.624065\n",
      "[100]\ttraining's auc: 0.719506\tvalid_1's auc: 0.624247\n",
      "[101]\ttraining's auc: 0.719662\tvalid_1's auc: 0.624444\n",
      "[102]\ttraining's auc: 0.719807\tvalid_1's auc: 0.624374\n",
      "[103]\ttraining's auc: 0.719951\tvalid_1's auc: 0.6244\n",
      "[104]\ttraining's auc: 0.720108\tvalid_1's auc: 0.624291\n",
      "[105]\ttraining's auc: 0.720244\tvalid_1's auc: 0.624832\n",
      "[106]\ttraining's auc: 0.7205\tvalid_1's auc: 0.624875\n",
      "[107]\ttraining's auc: 0.720612\tvalid_1's auc: 0.625457\n",
      "[108]\ttraining's auc: 0.720916\tvalid_1's auc: 0.625651\n",
      "[109]\ttraining's auc: 0.721146\tvalid_1's auc: 0.625638\n",
      "[110]\ttraining's auc: 0.721357\tvalid_1's auc: 0.625764\n",
      "[111]\ttraining's auc: 0.721461\tvalid_1's auc: 0.625958\n",
      "[112]\ttraining's auc: 0.721588\tvalid_1's auc: 0.626084\n",
      "[113]\ttraining's auc: 0.721654\tvalid_1's auc: 0.626192\n",
      "[114]\ttraining's auc: 0.72187\tvalid_1's auc: 0.626249\n",
      "[115]\ttraining's auc: 0.721936\tvalid_1's auc: 0.62636\n",
      "[116]\ttraining's auc: 0.72204\tvalid_1's auc: 0.626395\n",
      "[117]\ttraining's auc: 0.722134\tvalid_1's auc: 0.626535\n",
      "[118]\ttraining's auc: 0.722333\tvalid_1's auc: 0.626661\n",
      "[119]\ttraining's auc: 0.72241\tvalid_1's auc: 0.626719\n",
      "[120]\ttraining's auc: 0.722511\tvalid_1's auc: 0.626921\n",
      "[121]\ttraining's auc: 0.72271\tvalid_1's auc: 0.627061\n",
      "[122]\ttraining's auc: 0.722893\tvalid_1's auc: 0.627119\n",
      "[123]\ttraining's auc: 0.723099\tvalid_1's auc: 0.62737\n",
      "[124]\ttraining's auc: 0.723548\tvalid_1's auc: 0.627458\n",
      "[125]\ttraining's auc: 0.723625\tvalid_1's auc: 0.627504\n",
      "[126]\ttraining's auc: 0.723708\tvalid_1's auc: 0.627528\n",
      "[127]\ttraining's auc: 0.723863\tvalid_1's auc: 0.627512\n",
      "[128]\ttraining's auc: 0.724231\tvalid_1's auc: 0.627577\n",
      "[129]\ttraining's auc: 0.724344\tvalid_1's auc: 0.627656\n",
      "[130]\ttraining's auc: 0.724495\tvalid_1's auc: 0.627765\n",
      "[131]\ttraining's auc: 0.724643\tvalid_1's auc: 0.627744\n",
      "[132]\ttraining's auc: 0.724763\tvalid_1's auc: 0.628047\n",
      "[133]\ttraining's auc: 0.724829\tvalid_1's auc: 0.628084\n",
      "[134]\ttraining's auc: 0.724917\tvalid_1's auc: 0.62818\n",
      "[135]\ttraining's auc: 0.725065\tvalid_1's auc: 0.62828\n",
      "[136]\ttraining's auc: 0.725198\tvalid_1's auc: 0.628309\n",
      "[137]\ttraining's auc: 0.725336\tvalid_1's auc: 0.628388\n",
      "[138]\ttraining's auc: 0.725489\tvalid_1's auc: 0.628447\n",
      "[139]\ttraining's auc: 0.72558\tvalid_1's auc: 0.62844\n",
      "[140]\ttraining's auc: 0.72575\tvalid_1's auc: 0.628353\n",
      "[141]\ttraining's auc: 0.726101\tvalid_1's auc: 0.628468\n",
      "[142]\ttraining's auc: 0.726182\tvalid_1's auc: 0.628992\n",
      "[143]\ttraining's auc: 0.726388\tvalid_1's auc: 0.628974\n",
      "[144]\ttraining's auc: 0.726766\tvalid_1's auc: 0.629152\n",
      "[145]\ttraining's auc: 0.727055\tvalid_1's auc: 0.629227\n",
      "[146]\ttraining's auc: 0.727118\tvalid_1's auc: 0.629505\n",
      "[147]\ttraining's auc: 0.727322\tvalid_1's auc: 0.629566\n",
      "[148]\ttraining's auc: 0.727619\tvalid_1's auc: 0.629649\n",
      "[149]\ttraining's auc: 0.72769\tvalid_1's auc: 0.629667\n",
      "[150]\ttraining's auc: 0.727897\tvalid_1's auc: 0.629794\n",
      "[151]\ttraining's auc: 0.727991\tvalid_1's auc: 0.629844\n",
      "[152]\ttraining's auc: 0.728126\tvalid_1's auc: 0.62978\n",
      "[153]\ttraining's auc: 0.728223\tvalid_1's auc: 0.629838\n",
      "[154]\ttraining's auc: 0.728686\tvalid_1's auc: 0.62994\n",
      "[155]\ttraining's auc: 0.728815\tvalid_1's auc: 0.62997\n",
      "[156]\ttraining's auc: 0.728912\tvalid_1's auc: 0.63018\n",
      "[157]\ttraining's auc: 0.729006\tvalid_1's auc: 0.630271\n",
      "[158]\ttraining's auc: 0.729124\tvalid_1's auc: 0.63044\n",
      "[159]\ttraining's auc: 0.729165\tvalid_1's auc: 0.630465\n",
      "[160]\ttraining's auc: 0.72925\tvalid_1's auc: 0.63069\n",
      "[161]\ttraining's auc: 0.729296\tvalid_1's auc: 0.630727\n",
      "[162]\ttraining's auc: 0.729419\tvalid_1's auc: 0.630881\n",
      "[163]\ttraining's auc: 0.729487\tvalid_1's auc: 0.630995\n",
      "[164]\ttraining's auc: 0.72955\tvalid_1's auc: 0.630998\n",
      "[165]\ttraining's auc: 0.729627\tvalid_1's auc: 0.631094\n",
      "[166]\ttraining's auc: 0.729853\tvalid_1's auc: 0.631155\n",
      "[167]\ttraining's auc: 0.729939\tvalid_1's auc: 0.631428\n",
      "[168]\ttraining's auc: 0.730003\tvalid_1's auc: 0.631525\n",
      "[169]\ttraining's auc: 0.730108\tvalid_1's auc: 0.631657\n",
      "[170]\ttraining's auc: 0.730325\tvalid_1's auc: 0.631708\n",
      "[171]\ttraining's auc: 0.73036\tvalid_1's auc: 0.631659\n",
      "[172]\ttraining's auc: 0.730592\tvalid_1's auc: 0.631804\n",
      "[173]\ttraining's auc: 0.731036\tvalid_1's auc: 0.631838\n",
      "[174]\ttraining's auc: 0.731267\tvalid_1's auc: 0.631886\n",
      "[175]\ttraining's auc: 0.731345\tvalid_1's auc: 0.632312\n",
      "[176]\ttraining's auc: 0.731672\tvalid_1's auc: 0.63234\n",
      "[177]\ttraining's auc: 0.731734\tvalid_1's auc: 0.632327\n",
      "[178]\ttraining's auc: 0.731902\tvalid_1's auc: 0.632344\n",
      "[179]\ttraining's auc: 0.732002\tvalid_1's auc: 0.63242\n",
      "[180]\ttraining's auc: 0.732054\tvalid_1's auc: 0.632467\n",
      "[181]\ttraining's auc: 0.732123\tvalid_1's auc: 0.632595\n",
      "[182]\ttraining's auc: 0.732202\tvalid_1's auc: 0.632658\n",
      "[183]\ttraining's auc: 0.732402\tvalid_1's auc: 0.63265\n",
      "[184]\ttraining's auc: 0.732472\tvalid_1's auc: 0.632695\n",
      "[185]\ttraining's auc: 0.73265\tvalid_1's auc: 0.63244\n",
      "[186]\ttraining's auc: 0.732738\tvalid_1's auc: 0.632624\n",
      "[187]\ttraining's auc: 0.732855\tvalid_1's auc: 0.632728\n",
      "[188]\ttraining's auc: 0.733125\tvalid_1's auc: 0.632769\n",
      "[189]\ttraining's auc: 0.733381\tvalid_1's auc: 0.632864\n",
      "[190]\ttraining's auc: 0.73354\tvalid_1's auc: 0.63309\n",
      "[191]\ttraining's auc: 0.733722\tvalid_1's auc: 0.633132\n",
      "[192]\ttraining's auc: 0.733774\tvalid_1's auc: 0.633243\n",
      "[193]\ttraining's auc: 0.734001\tvalid_1's auc: 0.6333\n",
      "[194]\ttraining's auc: 0.734055\tvalid_1's auc: 0.633356\n",
      "[195]\ttraining's auc: 0.734188\tvalid_1's auc: 0.633347\n",
      "[196]\ttraining's auc: 0.734257\tvalid_1's auc: 0.63346\n",
      "[197]\ttraining's auc: 0.7343\tvalid_1's auc: 0.633461\n",
      "[198]\ttraining's auc: 0.734339\tvalid_1's auc: 0.633462\n",
      "[199]\ttraining's auc: 0.734467\tvalid_1's auc: 0.633659\n",
      "[200]\ttraining's auc: 0.734508\tvalid_1's auc: 0.633877\n",
      "[201]\ttraining's auc: 0.734708\tvalid_1's auc: 0.633914\n",
      "[202]\ttraining's auc: 0.734796\tvalid_1's auc: 0.633918\n",
      "[203]\ttraining's auc: 0.734843\tvalid_1's auc: 0.633957\n",
      "[204]\ttraining's auc: 0.734909\tvalid_1's auc: 0.634066\n",
      "[205]\ttraining's auc: 0.734935\tvalid_1's auc: 0.634202\n",
      "[206]\ttraining's auc: 0.735078\tvalid_1's auc: 0.63423\n",
      "[207]\ttraining's auc: 0.735131\tvalid_1's auc: 0.634247\n",
      "[208]\ttraining's auc: 0.735195\tvalid_1's auc: 0.634302\n",
      "[209]\ttraining's auc: 0.735275\tvalid_1's auc: 0.634321\n",
      "[210]\ttraining's auc: 0.735333\tvalid_1's auc: 0.634301\n",
      "[211]\ttraining's auc: 0.73547\tvalid_1's auc: 0.634484\n",
      "[212]\ttraining's auc: 0.735626\tvalid_1's auc: 0.634518\n",
      "[213]\ttraining's auc: 0.735834\tvalid_1's auc: 0.634529\n",
      "[214]\ttraining's auc: 0.736025\tvalid_1's auc: 0.634563\n",
      "[215]\ttraining's auc: 0.736177\tvalid_1's auc: 0.634784\n",
      "[216]\ttraining's auc: 0.736263\tvalid_1's auc: 0.634817\n",
      "[217]\ttraining's auc: 0.736348\tvalid_1's auc: 0.63493\n",
      "[218]\ttraining's auc: 0.736396\tvalid_1's auc: 0.63499\n",
      "[219]\ttraining's auc: 0.73652\tvalid_1's auc: 0.635027\n",
      "[220]\ttraining's auc: 0.736649\tvalid_1's auc: 0.635164\n",
      "[221]\ttraining's auc: 0.736753\tvalid_1's auc: 0.635226\n",
      "[222]\ttraining's auc: 0.736925\tvalid_1's auc: 0.635295\n",
      "[223]\ttraining's auc: 0.737005\tvalid_1's auc: 0.63532\n",
      "[224]\ttraining's auc: 0.737164\tvalid_1's auc: 0.635337\n",
      "[225]\ttraining's auc: 0.737442\tvalid_1's auc: 0.635387\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-ee6324307edb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mevs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmlgbm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrossvalidate_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_folds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidate_on_both\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/media/windows_d/DRIVE/MS CS UCU/DataScienceGame/model_lgbm.py\u001b[0m in \u001b[0;36mcrossvalidate_model\u001b[0;34m(path_to_folds, cols, params, verbose, iterations, validate_on_both)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidate_on_both\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m             \u001b[0mvalidation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlgb_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlgb_test\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mmodel_lgm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlgb_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_sets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"valid_0\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"auc\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tsdaemon/anaconda3/lib/python3.6/site-packages/lightgbm-0.1-py3.6.egg/lightgbm/engine.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, callbacks)\u001b[0m\n\u001b[1;32m    184\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mvalid_sets\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_valid_contain_train\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 186\u001b[0;31m                 \u001b[0mevaluation_result_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    187\u001b[0m             \u001b[0mevaluation_result_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval_valid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tsdaemon/anaconda3/lib/python3.6/site-packages/lightgbm-0.1-py3.6.egg/lightgbm/basic.py\u001b[0m in \u001b[0;36meval_train\u001b[0;34m(self, feval)\u001b[0m\n\u001b[1;32m   1453\u001b[0m             \u001b[0mEvaluation\u001b[0m \u001b[0mresult\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1454\u001b[0m         \"\"\"\n\u001b[0;32m-> 1455\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__inner_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_data_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1457\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0meval_valid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tsdaemon/anaconda3/lib/python3.6/site-packages/lightgbm-0.1-py3.6.egg/lightgbm/basic.py\u001b[0m in \u001b[0;36m__inner_eval\u001b[0;34m(self, data_name, data_idx, feval)\u001b[0m\n\u001b[1;32m   1680\u001b[0m                 \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1681\u001b[0m                 \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbyref\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp_out_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1682\u001b[0;31m                 result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))\n\u001b[0m\u001b[1;32m   1683\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtmp_out_len\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__num_inner_eval\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1684\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Wrong length of eval results\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "evs = mlgbm.crossvalidate_model(path_to_folds,cols,params,validate_on_both=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>genre_id</th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>media_id</th>\n",
       "      <th>album_id</th>\n",
       "      <th>context_type</th>\n",
       "      <th>release_date</th>\n",
       "      <th>platform_name</th>\n",
       "      <th>platform_family</th>\n",
       "      <th>media_duration</th>\n",
       "      <th>...</th>\n",
       "      <th>ExplicitLyrics</th>\n",
       "      <th>Bpm</th>\n",
       "      <th>Gain</th>\n",
       "      <th>Isrc_Country</th>\n",
       "      <th>Isrc_Label</th>\n",
       "      <th>BpmByUserMean</th>\n",
       "      <th>GainByUserMean</th>\n",
       "      <th>GainCenter</th>\n",
       "      <th>BpmCenter</th>\n",
       "      <th>Isrc_Country_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>1478104371</td>\n",
       "      <td>683078</td>\n",
       "      <td>82356</td>\n",
       "      <td>1</td>\n",
       "      <td>20021008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>542</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>-11.2</td>\n",
       "      <td>54</td>\n",
       "      <td>1208</td>\n",
       "      <td>127.640584</td>\n",
       "      <td>-9.052632</td>\n",
       "      <td>-2.147368</td>\n",
       "      <td>53.359416</td>\n",
       "      <td>0.240000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2744</td>\n",
       "      <td>1479317140</td>\n",
       "      <td>876497</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>307</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>-15.1</td>\n",
       "      <td>54</td>\n",
       "      <td>9684</td>\n",
       "      <td>138.024395</td>\n",
       "      <td>-8.166901</td>\n",
       "      <td>-6.933099</td>\n",
       "      <td>-50.024395</td>\n",
       "      <td>0.372263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2744</td>\n",
       "      <td>1479546361</td>\n",
       "      <td>876497</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>307</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>-15.1</td>\n",
       "      <td>54</td>\n",
       "      <td>9684</td>\n",
       "      <td>125.741566</td>\n",
       "      <td>-10.710046</td>\n",
       "      <td>-4.389954</td>\n",
       "      <td>-37.741566</td>\n",
       "      <td>0.639269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2744</td>\n",
       "      <td>1478457729</td>\n",
       "      <td>876500</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>265</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>-16.2</td>\n",
       "      <td>54</td>\n",
       "      <td>9684</td>\n",
       "      <td>130.469657</td>\n",
       "      <td>-9.994161</td>\n",
       "      <td>-6.205839</td>\n",
       "      <td>-23.469657</td>\n",
       "      <td>0.686131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2744</td>\n",
       "      <td>1480448560</td>\n",
       "      <td>876504</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>356</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>-16.3</td>\n",
       "      <td>54</td>\n",
       "      <td>9684</td>\n",
       "      <td>124.394607</td>\n",
       "      <td>-11.375735</td>\n",
       "      <td>-4.924265</td>\n",
       "      <td>-14.394607</td>\n",
       "      <td>0.761029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample_id  genre_id   ts_listen  media_id  album_id  context_type  \\\n",
       "0          0        50  1478104371    683078     82356             1   \n",
       "1          1      2744  1479317140    876497     99692             1   \n",
       "2          2      2744  1479546361    876497     99692             1   \n",
       "3          3      2744  1478457729    876500     99692             1   \n",
       "4          4      2744  1480448560    876504     99692             1   \n",
       "\n",
       "   release_date  platform_name  platform_family  media_duration  \\\n",
       "0      20021008              0                0             542   \n",
       "1      19851231              0                0             307   \n",
       "2      19851231              0                0             307   \n",
       "3      19851231              2                1             265   \n",
       "4      19851231              2                1             356   \n",
       "\n",
       "          ...          ExplicitLyrics    Bpm  Gain  Isrc_Country  Isrc_Label  \\\n",
       "0         ...                       0  181.0 -11.2            54        1208   \n",
       "1         ...                       0   88.0 -15.1            54        9684   \n",
       "2         ...                       0   88.0 -15.1            54        9684   \n",
       "3         ...                       0  107.0 -16.2            54        9684   \n",
       "4         ...                       0  110.0 -16.3            54        9684   \n",
       "\n",
       "   BpmByUserMean  GainByUserMean  GainCenter  BpmCenter  Isrc_Country_count  \n",
       "0     127.640584       -9.052632   -2.147368  53.359416            0.240000  \n",
       "1     138.024395       -8.166901   -6.933099 -50.024395            0.372263  \n",
       "2     125.741566      -10.710046   -4.389954 -37.741566            0.639269  \n",
       "3     130.469657       -9.994161   -6.205839 -23.469657            0.686131  \n",
       "4     124.394607      -11.375735   -4.924265 -14.394607            0.761029  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_sample = './data/sample_submission_kaggle.csv'\n",
    "path_to_submission = './data/submission.csv'\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8')\n",
    "test = pd.concat([test, pd.read_csv(path_to_api_track_data_test, encoding='utf-8')], axis=1)\n",
    "\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import model_lgbm as mlgbm\n",
    "fs = mlgbm.create_submission(train, test, path_to_sample, params, cols, path_to_submission, iterations=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('user_id', 1878),\n",
       " ('BpmByUserMean', 1644),\n",
       " ('GainByUserMean', 1535),\n",
       " ('Isrc_Country_count', 1108),\n",
       " ('context_type', 881),\n",
       " ('user_age', 748),\n",
       " ('Isrc_Country', 466),\n",
       " ('platform_name', 324),\n",
       " ('Rank', 298),\n",
       " ('Isrc_Label', 253),\n",
       " ('artist_id', 251),\n",
       " ('platform_family', 229),\n",
       " ('release_date', 207),\n",
       " ('listen_type', 203),\n",
       " ('user_gender', 181),\n",
       " ('genre_id', 171),\n",
       " ('media_duration', 155),\n",
       " ('album_id', 143),\n",
       " ('media_id', 139),\n",
       " ('GainCenter', 126),\n",
       " ('TrackPosition', 101),\n",
       " ('Bpm', 99),\n",
       " ('Gain', 97),\n",
       " ('BpmCenter', 87),\n",
       " ('ExplicitLyrics', 56),\n",
       " ('DiskNumber', 20)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlgbm.feature_score(fs,cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}