AS_Feature_release_date_expanded.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>release_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7578747</th>\n",
       "      <td>20161014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578748</th>\n",
       "      <td>20161014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578749</th>\n",
       "      <td>19910101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578750</th>\n",
       "      <td>19910101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578751</th>\n",
       "      <td>19910101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         release_date\n",
       "7578747      20161014\n",
       "7578748      20161014\n",
       "7578749      19910101\n",
       "7578750      19910101\n",
       "7578751      19910101"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8', usecols=['release_date'])\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8', usecols=['release_date'])\n",
    "\n",
    "df = pd.concat([train, test], ignore_index=True)\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>release_date</th>\n",
       "      <th>release_date_dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20040704</td>\n",
       "      <td>2004-07-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20060301</td>\n",
       "      <td>2006-03-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20140714</td>\n",
       "      <td>2014-07-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20001030</td>\n",
       "      <td>2000-10-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   release_date release_date_dt\n",
       "0      20040704      2004-07-04\n",
       "1      20060301      2006-03-01\n",
       "2      20140714      2014-07-14\n",
       "3      20001030      2000-10-30\n",
       "4      20080215      2008-02-15"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import date\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "def extract_date(date_int):\n",
    "    date_str = str(date_int)\n",
    "    year = int(date_str[0:4])\n",
    "    month = int(date_str[4:6])\n",
    "    day = int(date_str[6:8])\n",
    "    return date(year, month, day)\n",
    "\n",
    "df['release_date_dt'] = df.apply(lambda row: extract_date(row['release_date']),axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.ix[df['release_date'] > 30000000,'release_date_dt'] = date(2000,1,1)\n",
    "df.ix[df['release_date'] > 30000000,'release_date'] = 20000101"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rl_f = df.apply(lambda row: (row['release_date_dt'].year, row['release_date_dt'].month, row['release_date_dt'].day, int((row['release_date_dt'].year-1900)/10)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>release_date</th>\n",
       "      <th>release_date_dt</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20040704</td>\n",
       "      <td>2004-07-04</td>\n",
       "      <td>2004</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20060301</td>\n",
       "      <td>2006-03-01</td>\n",
       "      <td>2006</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20140714</td>\n",
       "      <td>2014-07-14</td>\n",
       "      <td>2014</td>\n",
       "      <td>7</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20001030</td>\n",
       "      <td>2000-10-30</td>\n",
       "      <td>2000</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>20080215</td>\n",
       "      <td>2008-02-15</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   release_date release_date_dt  release_date_year  release_date_month  \\\n",
       "0      20040704      2004-07-04               2004                   7   \n",
       "1      20060301      2006-03-01               2006                   3   \n",
       "2      20140714      2014-07-14               2014                   7   \n",
       "3      20001030      2000-10-30               2000                  10   \n",
       "4      20080215      2008-02-15               2008                   2   \n",
       "5      20080215      2008-02-15               2008                   2   \n",
       "6      20080215      2008-02-15               2008                   2   \n",
       "7      20080215      2008-02-15               2008                   2   \n",
       "8      20080215      2008-02-15               2008                   2   \n",
       "9      20080215      2008-02-15               2008                   2   \n",
       "\n",
       "   release_date_day  release_date_decade  \n",
       "0                 4                   10  \n",
       "1                 1                   10  \n",
       "2                14                   11  \n",
       "3                30                   10  \n",
       "4                15                   10  \n",
       "5                15                   10  \n",
       "6                15                   10  \n",
       "7                15                   10  \n",
       "8                15                   10  \n",
       "9                15                   10  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ftrs = pd.DataFrame(list(rl_f), columns=['release_date_year', 'release_date_month', 'release_date_day', 'release_date_decade'])\n",
    "df = pd.concat([df, ftrs], axis=1)\n",
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_release_date_train = './data/features/release_date.train.txt'\n",
    "path_to_release_date_test = './data/features/release_date.test.txt'\n",
    "newcols = ['release_date_year','release_date_month','release_date_day','release_date_decade']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2004</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2006</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014</td>\n",
       "      <td>7</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   release_date_year  release_date_month  release_date_day  \\\n",
       "0               2004                   7                 4   \n",
       "1               2006                   3                 1   \n",
       "2               2014                   7                14   \n",
       "3               2000                  10                30   \n",
       "4               2008                   2                15   \n",
       "\n",
       "   release_date_decade  \n",
       "0                   10  \n",
       "1                   10  \n",
       "2                   11  \n",
       "3                   10  \n",
       "4                   10  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dt = df.loc[range(len(train))][newcols]\n",
    "train_dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7558834</th>\n",
       "      <td>2002</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7558835</th>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7558836</th>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7558837</th>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7558838</th>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         release_date_year  release_date_month  release_date_day  \\\n",
       "7558834               2002                  10                 8   \n",
       "7558835               1985                  12                31   \n",
       "7558836               1985                  12                31   \n",
       "7558837               1985                  12                31   \n",
       "7558838               1985                  12                31   \n",
       "\n",
       "         release_date_decade  \n",
       "7558834                   10  \n",
       "7558835                    8  \n",
       "7558836                    8  \n",
       "7558837                    8  \n",
       "7558838                    8  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dt = df.loc[range(len(train), len(train)+len(test))][newcols]\n",
    "test_dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dt.to_csv(path_to_release_date_train, encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_dt.to_csv(path_to_release_date_test, encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7578747</th>\n",
       "      <td>1479664431</td>\n",
       "      <td>12641</td>\n",
       "      <td>0</td>\n",
       "      <td>2016</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>0.870690</td>\n",
       "      <td>0.525862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578748</th>\n",
       "      <td>1480358099</td>\n",
       "      <td>10055</td>\n",
       "      <td>0</td>\n",
       "      <td>2016</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>0.979487</td>\n",
       "      <td>0.779487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578749</th>\n",
       "      <td>1480616853</td>\n",
       "      <td>1029</td>\n",
       "      <td>0</td>\n",
       "      <td>1991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.152223</td>\n",
       "      <td>0.001507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578750</th>\n",
       "      <td>1479644510</td>\n",
       "      <td>4630</td>\n",
       "      <td>0</td>\n",
       "      <td>1991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.052830</td>\n",
       "      <td>0.001887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578751</th>\n",
       "      <td>1480491885</td>\n",
       "      <td>6467</td>\n",
       "      <td>0</td>\n",
       "      <td>1991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.055703</td>\n",
       "      <td>0.002653</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ts_listen  user_id  is_listened  release_date_year  \\\n",
       "7578747  1479664431    12641            0               2016   \n",
       "7578748  1480358099    10055            0               2016   \n",
       "7578749  1480616853     1029            0               1991   \n",
       "7578750  1479644510     4630            0               1991   \n",
       "7578751  1480491885     6467            0               1991   \n",
       "\n",
       "         release_date_month  release_date_day  release_date_decade  \\\n",
       "7578747                  10                14                   11   \n",
       "7578748                  10                14                   11   \n",
       "7578749                   1                 1                    9   \n",
       "7578750                   1                 1                    9   \n",
       "7578751                   1                 1                    9   \n",
       "\n",
       "         release_date_listened_this_decade  release_date_listened_this_year  \n",
       "7578747                           0.870690                         0.525862  \n",
       "7578748                           0.979487                         0.779487  \n",
       "7578749                           0.152223                         0.001507  \n",
       "7578750                           0.052830                         0.001887  \n",
       "7578751                           0.055703                         0.002653  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# я хочу ще згенерити кількість, скільки користувач перед тим слухав записів цього року і цієї декади\n",
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8', usecols=['user_id', 'ts_listen', 'is_listened'])\n",
    "train = pd.concat([train, pd.read_csv(path_to_release_date_train, encoding='utf-8')],axis=1)\n",
    "\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8', usecols=['user_id', 'ts_listen'])\n",
    "test['is_listened'] = 0\n",
    "\n",
    "test = pd.concat([test, pd.read_csv(path_to_release_date_test, encoding='utf-8')],axis=1)\n",
    "\n",
    "df = pd.concat([train, test], ignore_index=True)\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>829179</th>\n",
       "      <td>1478333098</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000158</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829180</th>\n",
       "      <td>1478358099</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000317</td>\n",
       "      <td>0.000317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829181</th>\n",
       "      <td>1478359301</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000475</td>\n",
       "      <td>0.000475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829178</th>\n",
       "      <td>1478359883</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000633</td>\n",
       "      <td>0.000633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829182</th>\n",
       "      <td>1478362962</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000791</td>\n",
       "      <td>0.000791</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ts_listen  user_id  is_listened  release_date_year  \\\n",
       "829179  1478333098        0            1               1945   \n",
       "829180  1478358099        0            1               1945   \n",
       "829181  1478359301        0            1               1945   \n",
       "829178  1478359883        0            1               1945   \n",
       "829182  1478362962        0            1               1945   \n",
       "\n",
       "        release_date_month  release_date_day  release_date_decade  \\\n",
       "829179                   1                 1                    4   \n",
       "829180                   1                 1                    4   \n",
       "829181                   1                 1                    4   \n",
       "829178                   1                 1                    4   \n",
       "829182                   1                 1                    4   \n",
       "\n",
       "        release_date_listened_this_decade  release_date_listened_this_year  \n",
       "829179                           0.000158                         0.000158  \n",
       "829180                           0.000317                         0.000317  \n",
       "829181                           0.000475                         0.000475  \n",
       "829178                           0.000633                         0.000633  \n",
       "829182                           0.000791                         0.000791  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sort_values(by=['user_id', 'release_date_decade', 'ts_listen'], axis=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>829179</th>\n",
       "      <td>1478333098</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829180</th>\n",
       "      <td>1478358099</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829181</th>\n",
       "      <td>1478359301</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829178</th>\n",
       "      <td>1478359883</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829182</th>\n",
       "      <td>1478362962</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829191</th>\n",
       "      <td>1478702580</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307318</th>\n",
       "      <td>1478295198</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372249</th>\n",
       "      <td>1478371200</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303096</th>\n",
       "      <td>1478798499</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1247753</th>\n",
       "      <td>1478848107</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308181</th>\n",
       "      <td>1478862692</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308180</th>\n",
       "      <td>1478895504</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372236</th>\n",
       "      <td>1479817835</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>0.001108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305245</th>\n",
       "      <td>1480143850</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.001266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372260</th>\n",
       "      <td>1480146369</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>0.001425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305872</th>\n",
       "      <td>1480497844</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>0.001583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4011957</th>\n",
       "      <td>1478193488</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1967</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4531278</th>\n",
       "      <td>1478194813</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1967</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421793</th>\n",
       "      <td>1478200972</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929472</th>\n",
       "      <td>1478201147</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1969</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ts_listen  user_id  is_listened  release_date_year  \\\n",
       "829179   1478333098        0            1               1945   \n",
       "829180   1478358099        0            1               1945   \n",
       "829181   1478359301        0            1               1945   \n",
       "829178   1478359883        0            1               1945   \n",
       "829182   1478362962        0            1               1945   \n",
       "829191   1478702580        0            1               1945   \n",
       "307318   1478295198        0            1               1958   \n",
       "6372249  1478371200        0            1               1958   \n",
       "303096   1478798499        0            1               1958   \n",
       "1247753  1478848107        0            1               1958   \n",
       "308181   1478862692        0            1               1958   \n",
       "308180   1478895504        0            1               1958   \n",
       "6372236  1479817835        0            1               1958   \n",
       "305245   1480143850        0            1               1958   \n",
       "6372260  1480146369        0            1               1958   \n",
       "305872   1480497844        0            1               1958   \n",
       "4011957  1478193488        0            1               1967   \n",
       "4531278  1478194813        0            1               1967   \n",
       "5421793  1478200972        0            1               1962   \n",
       "929472   1478201147        0            1               1969   \n",
       "\n",
       "         release_date_month  release_date_day  release_date_decade  \\\n",
       "829179                    1                 1                    4   \n",
       "829180                    1                 1                    4   \n",
       "829181                    1                 1                    4   \n",
       "829178                    1                 1                    4   \n",
       "829182                    1                 1                    4   \n",
       "829191                    1                 1                    4   \n",
       "307318                    9                24                    5   \n",
       "6372249                  12                31                    5   \n",
       "303096                    9                24                    5   \n",
       "1247753                   9                24                    5   \n",
       "308181                    9                24                    5   \n",
       "308180                    9                24                    5   \n",
       "6372236                  12                31                    5   \n",
       "305245                    9                24                    5   \n",
       "6372260                  12                31                    5   \n",
       "305872                    9                24                    5   \n",
       "4011957                  12                31                    6   \n",
       "4531278                  12                31                    6   \n",
       "5421793                   8                20                    6   \n",
       "929472                   12                31                    6   \n",
       "\n",
       "         release_date_listened_this_decade  release_date_listened_this_year  \n",
       "829179                                   1                         0.000158  \n",
       "829180                                   2                         0.000317  \n",
       "829181                                   3                         0.000475  \n",
       "829178                                   4                         0.000633  \n",
       "829182                                   5                         0.000791  \n",
       "829191                                   6                         0.000950  \n",
       "307318                                   1                         0.000158  \n",
       "6372249                                  2                         0.000317  \n",
       "303096                                   3                         0.000475  \n",
       "1247753                                  4                         0.000633  \n",
       "308181                                   5                         0.000791  \n",
       "308180                                   6                         0.000950  \n",
       "6372236                                  7                         0.001108  \n",
       "305245                                   8                         0.001266  \n",
       "6372260                                  9                         0.001425  \n",
       "305872                                  10                         0.001583  \n",
       "4011957                                  1                         0.000158  \n",
       "4531278                                  2                         0.000317  \n",
       "5421793                                  3                         0.000158  \n",
       "929472                                   4                         0.000158  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['release_date_listened_this_decade'] = df['is_listened'].groupby((\n",
    "                (df[['user_id', 'release_date_decade']].shift() != df[['user_id', 'release_date_decade']]).any(axis=1)\n",
    "            ).cumsum()).cumcount() + 1\n",
    "\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>829179</th>\n",
       "      <td>1478333098</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829180</th>\n",
       "      <td>1478358099</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829181</th>\n",
       "      <td>1478359301</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829178</th>\n",
       "      <td>1478359883</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829182</th>\n",
       "      <td>1478362962</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000791</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ts_listen  user_id  is_listened  release_date_year  \\\n",
       "829179  1478333098        0            1               1945   \n",
       "829180  1478358099        0            1               1945   \n",
       "829181  1478359301        0            1               1945   \n",
       "829178  1478359883        0            1               1945   \n",
       "829182  1478362962        0            1               1945   \n",
       "\n",
       "        release_date_month  release_date_day  release_date_decade  \\\n",
       "829179                   1                 1                    4   \n",
       "829180                   1                 1                    4   \n",
       "829181                   1                 1                    4   \n",
       "829178                   1                 1                    4   \n",
       "829182                   1                 1                    4   \n",
       "\n",
       "        release_date_listened_this_decade  release_date_listened_this_year  \n",
       "829179                                  1                         0.000158  \n",
       "829180                                  2                         0.000317  \n",
       "829181                                  3                         0.000475  \n",
       "829178                                  4                         0.000633  \n",
       "829182                                  5                         0.000791  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sort_values(by=['user_id', 'release_date_year', 'ts_listen'], axis=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>829179</th>\n",
       "      <td>1478333098</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829180</th>\n",
       "      <td>1478358099</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829181</th>\n",
       "      <td>1478359301</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829178</th>\n",
       "      <td>1478359883</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829182</th>\n",
       "      <td>1478362962</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829191</th>\n",
       "      <td>1478702580</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307318</th>\n",
       "      <td>1478295198</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372249</th>\n",
       "      <td>1478371200</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303096</th>\n",
       "      <td>1478798499</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1247753</th>\n",
       "      <td>1478848107</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308181</th>\n",
       "      <td>1478862692</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308180</th>\n",
       "      <td>1478895504</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372236</th>\n",
       "      <td>1479817835</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305245</th>\n",
       "      <td>1480143850</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372260</th>\n",
       "      <td>1480146369</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305872</th>\n",
       "      <td>1480497844</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421793</th>\n",
       "      <td>1478200972</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421513</th>\n",
       "      <td>1478628251</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421710</th>\n",
       "      <td>1478706026</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421797</th>\n",
       "      <td>1478852818</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ts_listen  user_id  is_listened  release_date_year  \\\n",
       "829179   1478333098        0            1               1945   \n",
       "829180   1478358099        0            1               1945   \n",
       "829181   1478359301        0            1               1945   \n",
       "829178   1478359883        0            1               1945   \n",
       "829182   1478362962        0            1               1945   \n",
       "829191   1478702580        0            1               1945   \n",
       "307318   1478295198        0            1               1958   \n",
       "6372249  1478371200        0            1               1958   \n",
       "303096   1478798499        0            1               1958   \n",
       "1247753  1478848107        0            1               1958   \n",
       "308181   1478862692        0            1               1958   \n",
       "308180   1478895504        0            1               1958   \n",
       "6372236  1479817835        0            1               1958   \n",
       "305245   1480143850        0            1               1958   \n",
       "6372260  1480146369        0            1               1958   \n",
       "305872   1480497844        0            1               1958   \n",
       "5421793  1478200972        0            1               1962   \n",
       "5421513  1478628251        0            1               1962   \n",
       "5421710  1478706026        0            1               1962   \n",
       "5421797  1478852818        0            1               1962   \n",
       "\n",
       "         release_date_month  release_date_day  release_date_decade  \\\n",
       "829179                    1                 1                    4   \n",
       "829180                    1                 1                    4   \n",
       "829181                    1                 1                    4   \n",
       "829178                    1                 1                    4   \n",
       "829182                    1                 1                    4   \n",
       "829191                    1                 1                    4   \n",
       "307318                    9                24                    5   \n",
       "6372249                  12                31                    5   \n",
       "303096                    9                24                    5   \n",
       "1247753                   9                24                    5   \n",
       "308181                    9                24                    5   \n",
       "308180                    9                24                    5   \n",
       "6372236                  12                31                    5   \n",
       "305245                    9                24                    5   \n",
       "6372260                  12                31                    5   \n",
       "305872                    9                24                    5   \n",
       "5421793                   8                20                    6   \n",
       "5421513                   8                20                    6   \n",
       "5421710                   8                20                    6   \n",
       "5421797                   8                20                    6   \n",
       "\n",
       "         release_date_listened_this_decade  release_date_listened_this_year  \n",
       "829179                                   1                                1  \n",
       "829180                                   2                                2  \n",
       "829181                                   3                                3  \n",
       "829178                                   4                                4  \n",
       "829182                                   5                                5  \n",
       "829191                                   6                                6  \n",
       "307318                                   1                                1  \n",
       "6372249                                  2                                2  \n",
       "303096                                   3                                3  \n",
       "1247753                                  4                                4  \n",
       "308181                                   5                                5  \n",
       "308180                                   6                                6  \n",
       "6372236                                  7                                7  \n",
       "305245                                   8                                8  \n",
       "6372260                                  9                                9  \n",
       "305872                                  10                               10  \n",
       "5421793                                  3                                1  \n",
       "5421513                                 17                                2  \n",
       "5421710                                 20                                3  \n",
       "5421797                                 28                                4  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['release_date_listened_this_year'] = df['is_listened'].groupby((\n",
    "                (df[['user_id', 'release_date_year']].shift() != df[['user_id', 'release_date_year']]).any(axis=1)\n",
    "            ).cumsum()).cumcount() + 1\n",
    "\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>829179</th>\n",
       "      <td>1478333098</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829180</th>\n",
       "      <td>1478358099</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829181</th>\n",
       "      <td>1478359301</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829178</th>\n",
       "      <td>1478359883</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829182</th>\n",
       "      <td>1478362962</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829191</th>\n",
       "      <td>1478702580</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1945</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307318</th>\n",
       "      <td>1478295198</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372249</th>\n",
       "      <td>1478371200</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303096</th>\n",
       "      <td>1478798499</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1247753</th>\n",
       "      <td>1478848107</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308181</th>\n",
       "      <td>1478862692</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308180</th>\n",
       "      <td>1478895504</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372236</th>\n",
       "      <td>1479817835</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305245</th>\n",
       "      <td>1480143850</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6372260</th>\n",
       "      <td>1480146369</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305872</th>\n",
       "      <td>1480497844</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1958</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421793</th>\n",
       "      <td>1478200972</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421513</th>\n",
       "      <td>1478628251</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421710</th>\n",
       "      <td>1478706026</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421797</th>\n",
       "      <td>1478852818</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1962</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ts_listen  user_id  is_listened  release_date_year  \\\n",
       "829179   1478333098        0            1               1945   \n",
       "829180   1478358099        0            1               1945   \n",
       "829181   1478359301        0            1               1945   \n",
       "829178   1478359883        0            1               1945   \n",
       "829182   1478362962        0            1               1945   \n",
       "829191   1478702580        0            1               1945   \n",
       "307318   1478295198        0            1               1958   \n",
       "6372249  1478371200        0            1               1958   \n",
       "303096   1478798499        0            1               1958   \n",
       "1247753  1478848107        0            1               1958   \n",
       "308181   1478862692        0            1               1958   \n",
       "308180   1478895504        0            1               1958   \n",
       "6372236  1479817835        0            1               1958   \n",
       "305245   1480143850        0            1               1958   \n",
       "6372260  1480146369        0            1               1958   \n",
       "305872   1480497844        0            1               1958   \n",
       "5421793  1478200972        0            1               1962   \n",
       "5421513  1478628251        0            1               1962   \n",
       "5421710  1478706026        0            1               1962   \n",
       "5421797  1478852818        0            1               1962   \n",
       "\n",
       "         release_date_month  release_date_day  release_date_decade  \\\n",
       "829179                    1                 1                    4   \n",
       "829180                    1                 1                    4   \n",
       "829181                    1                 1                    4   \n",
       "829178                    1                 1                    4   \n",
       "829182                    1                 1                    4   \n",
       "829191                    1                 1                    4   \n",
       "307318                    9                24                    5   \n",
       "6372249                  12                31                    5   \n",
       "303096                    9                24                    5   \n",
       "1247753                   9                24                    5   \n",
       "308181                    9                24                    5   \n",
       "308180                    9                24                    5   \n",
       "6372236                  12                31                    5   \n",
       "305245                    9                24                    5   \n",
       "6372260                  12                31                    5   \n",
       "305872                    9                24                    5   \n",
       "5421793                   8                20                    6   \n",
       "5421513                   8                20                    6   \n",
       "5421710                   8                20                    6   \n",
       "5421797                   8                20                    6   \n",
       "\n",
       "         release_date_listened_this_decade  release_date_listened_this_year  \n",
       "829179                                   1                                1  \n",
       "829180                                   2                                2  \n",
       "829181                                   3                                3  \n",
       "829178                                   4                                4  \n",
       "829182                                   5                                5  \n",
       "829191                                   6                                6  \n",
       "307318                                   1                                1  \n",
       "6372249                                  2                                2  \n",
       "303096                                   3                                3  \n",
       "1247753                                  4                                4  \n",
       "308181                                   5                                5  \n",
       "308180                                   6                                6  \n",
       "6372236                                  7                                7  \n",
       "305245                                   8                                8  \n",
       "6372260                                  9                                9  \n",
       "305872                                  10                               10  \n",
       "5421793                                  3                                1  \n",
       "5421513                                 17                                2  \n",
       "5421710                                 20                                3  \n",
       "5421797                                 20                                4  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# user_count = df.groupby('user_id', as_index=False).agg({'ts_listen':'count'})\n",
    "# user_count['count'] = user_count['ts_listen']\n",
    "# del user_count['ts_listen']\n",
    "# df = df.join(user_count, on='user_id', rsuffix='_r')\n",
    "# df.head()\n",
    "\n",
    "df.ix[df['release_date_listened_this_year'] > 20, 'release_date_listened_this_year'] = 20\n",
    "df.ix[df['release_date_listened_this_decade'] > 20, 'release_date_listened_this_decade'] = 20\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df['release_date_listened_this_decade'] = df['release_date_listened_this_decade']/df['count']\n",
    "# df['release_date_listened_this_year'] = df['release_date_listened_this_year']/df['count']\n",
    "# del df['user_id_r']\n",
    "# del df['count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = df.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7578747</th>\n",
       "      <td>1479664431</td>\n",
       "      <td>12641</td>\n",
       "      <td>0</td>\n",
       "      <td>2016</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578748</th>\n",
       "      <td>1480358099</td>\n",
       "      <td>10055</td>\n",
       "      <td>0</td>\n",
       "      <td>2016</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578749</th>\n",
       "      <td>1480616853</td>\n",
       "      <td>1029</td>\n",
       "      <td>0</td>\n",
       "      <td>1991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578750</th>\n",
       "      <td>1479644510</td>\n",
       "      <td>4630</td>\n",
       "      <td>0</td>\n",
       "      <td>1991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578751</th>\n",
       "      <td>1480491885</td>\n",
       "      <td>6467</td>\n",
       "      <td>0</td>\n",
       "      <td>1991</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ts_listen  user_id  is_listened  release_date_year  \\\n",
       "7578747  1479664431    12641            0               2016   \n",
       "7578748  1480358099    10055            0               2016   \n",
       "7578749  1480616853     1029            0               1991   \n",
       "7578750  1479644510     4630            0               1991   \n",
       "7578751  1480491885     6467            0               1991   \n",
       "\n",
       "         release_date_month  release_date_day  release_date_decade  \\\n",
       "7578747                  10                14                   11   \n",
       "7578748                  10                14                   11   \n",
       "7578749                   1                 1                    9   \n",
       "7578750                   1                 1                    9   \n",
       "7578751                   1                 1                    9   \n",
       "\n",
       "         release_date_listened_this_decade  release_date_listened_this_year  \n",
       "7578747                                 20                               20  \n",
       "7578748                                 20                               20  \n",
       "7578749                                 20                                2  \n",
       "7578750                                 20                                1  \n",
       "7578751                                 20                                1  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_release_date_train = './data/features/release_date.train.txt'\n",
    "path_to_release_date_test = './data/features/release_date.test.txt'\n",
    "newcols = ['release_date_year','release_date_month','release_date_day','release_date_decade', \n",
    "           'release_date_listened_this_decade','release_date_listened_this_year']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dt = df.loc[range(len(train))][newcols]\n",
    "train_dt.to_csv(path_to_release_date_train, encoding='utf-8', index=False)\n",
    "\n",
    "test_dt = df.loc[range(len(train), len(train)+len(test))][newcols]\n",
    "test_dt.to_csv(path_to_release_date_test, encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre_id</th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>media_id</th>\n",
       "      <th>album_id</th>\n",
       "      <th>context_type</th>\n",
       "      <th>release_date</th>\n",
       "      <th>platform_name</th>\n",
       "      <th>platform_family</th>\n",
       "      <th>media_duration</th>\n",
       "      <th>listen_type</th>\n",
       "      <th>...</th>\n",
       "      <th>user_id</th>\n",
       "      <th>artist_id</th>\n",
       "      <th>user_age</th>\n",
       "      <th>is_listened</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25471</td>\n",
       "      <td>1480597215</td>\n",
       "      <td>222606</td>\n",
       "      <td>41774</td>\n",
       "      <td>12</td>\n",
       "      <td>20040704</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>223</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9241</td>\n",
       "      <td>55164</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>2004</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.343612</td>\n",
       "      <td>0.140969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25571</td>\n",
       "      <td>1480544735</td>\n",
       "      <td>250467</td>\n",
       "      <td>43941</td>\n",
       "      <td>0</td>\n",
       "      <td>20060301</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>171</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>16547</td>\n",
       "      <td>55830</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>2006</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.097561</td>\n",
       "      <td>0.024390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>1479563953</td>\n",
       "      <td>305197</td>\n",
       "      <td>48078</td>\n",
       "      <td>1</td>\n",
       "      <td>20140714</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>149</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>7665</td>\n",
       "      <td>2704</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>2014</td>\n",
       "      <td>7</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>0.518152</td>\n",
       "      <td>0.066007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1480152098</td>\n",
       "      <td>900502</td>\n",
       "      <td>71521</td>\n",
       "      <td>0</td>\n",
       "      <td>20001030</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>240</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1580</td>\n",
       "      <td>938</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>0.039033</td>\n",
       "      <td>0.002788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1478368974</td>\n",
       "      <td>542335</td>\n",
       "      <td>71718</td>\n",
       "      <td>0</td>\n",
       "      <td>20080215</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1812</td>\n",
       "      <td>2939</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>2008</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.047047</td>\n",
       "      <td>0.005005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   genre_id   ts_listen  media_id  album_id  context_type  release_date  \\\n",
       "0     25471  1480597215    222606     41774            12      20040704   \n",
       "1     25571  1480544735    250467     43941             0      20060301   \n",
       "2        16  1479563953    305197     48078             1      20140714   \n",
       "3         7  1480152098    900502     71521             0      20001030   \n",
       "4         7  1478368974    542335     71718             0      20080215   \n",
       "\n",
       "   platform_name  platform_family  media_duration  listen_type  \\\n",
       "0              1                0             223            0   \n",
       "1              2                1             171            0   \n",
       "2              2                1             149            1   \n",
       "3              0                0             240            0   \n",
       "4              0                0             150            0   \n",
       "\n",
       "                ...                 user_id  artist_id  user_age  is_listened  \\\n",
       "0               ...                    9241      55164        29            0   \n",
       "1               ...                   16547      55830        30            1   \n",
       "2               ...                    7665       2704        29            1   \n",
       "3               ...                    1580        938        30            0   \n",
       "4               ...                    1812       2939        24            1   \n",
       "\n",
       "   release_date_year  release_date_month  release_date_day  \\\n",
       "0               2004                   7                 4   \n",
       "1               2006                   3                 1   \n",
       "2               2014                   7                14   \n",
       "3               2000                  10                30   \n",
       "4               2008                   2                15   \n",
       "\n",
       "   release_date_decade  release_date_listened_this_decade  \\\n",
       "0                   10                           0.343612   \n",
       "1                   10                           0.097561   \n",
       "2                   11                           0.518152   \n",
       "3                   10                           0.039033   \n",
       "4                   10                           0.047047   \n",
       "\n",
       "   release_date_listened_this_year  \n",
       "0                         0.140969  \n",
       "1                         0.024390  \n",
       "2                         0.066007  \n",
       "3                         0.002788  \n",
       "4                         0.005005  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import validation as v\n",
    "import pandas as pd\n",
    "\n",
    "path_to_train = './data/train.csv'\n",
    "\n",
    "train = pd.read_csv(path_to_train, encoding='utf-8')\n",
    "\n",
    "train = pd.concat([train, pd.read_csv(path_to_release_date_train, encoding='utf-8')], axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_folds = './data/lightgbm'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1\n"
     ]
    }
   ],
   "source": [
    "v.generate_validation_folds(train, path_to_folds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import model_lgbm as mlgbm\n",
    "\n",
    "cols = [\n",
    "        'genre_id', 'media_id', 'album_id', 'context_type',\n",
    "        'release_date', 'platform_name', 'platform_family', 'media_duration',\n",
    "        'listen_type', 'user_gender', 'user_id', 'artist_id', 'user_age', \n",
    "       ] + newcols\n",
    "\n",
    "params = {\n",
    "    'application':'binary',\n",
    "    'num_leaves': 31,\n",
    "    'max_depth': 20,\n",
    "    'feature_fraction': 0.7,\n",
    "    'bagging_fraction': 0.7,\n",
    "    'max_bin': 200,\n",
    "    'metric': 'auc',\n",
    "    'verbose': 1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttraining's auc: 0.672938\tvalid_1's auc: 0.558629\n",
      "[2]\ttraining's auc: 0.676314\tvalid_1's auc: 0.564966\n",
      "[3]\ttraining's auc: 0.677163\tvalid_1's auc: 0.56659\n",
      "[4]\ttraining's auc: 0.678667\tvalid_1's auc: 0.56105\n",
      "[5]\ttraining's auc: 0.679728\tvalid_1's auc: 0.559787\n",
      "[6]\ttraining's auc: 0.680124\tvalid_1's auc: 0.56142\n",
      "[7]\ttraining's auc: 0.681024\tvalid_1's auc: 0.564091\n",
      "[8]\ttraining's auc: 0.680954\tvalid_1's auc: 0.564038\n",
      "[9]\ttraining's auc: 0.6811\tvalid_1's auc: 0.56343\n",
      "[10]\ttraining's auc: 0.681078\tvalid_1's auc: 0.562148\n",
      "[11]\ttraining's auc: 0.681641\tvalid_1's auc: 0.562456\n",
      "[12]\ttraining's auc: 0.682176\tvalid_1's auc: 0.563342\n",
      "[13]\ttraining's auc: 0.68236\tvalid_1's auc: 0.562852\n",
      "[14]\ttraining's auc: 0.683178\tvalid_1's auc: 0.563253\n",
      "[15]\ttraining's auc: 0.683953\tvalid_1's auc: 0.564359\n",
      "[16]\ttraining's auc: 0.684275\tvalid_1's auc: 0.565012\n",
      "[17]\ttraining's auc: 0.684884\tvalid_1's auc: 0.565649\n",
      "[18]\ttraining's auc: 0.685483\tvalid_1's auc: 0.566092\n",
      "[19]\ttraining's auc: 0.686013\tvalid_1's auc: 0.566196\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-ee6324307edb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mevs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmlgbm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrossvalidate_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_folds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidate_on_both\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/media/windows_d/DRIVE/MS CS UCU/DataScienceGame/model_lgbm.py\u001b[0m in \u001b[0;36mcrossvalidate_model\u001b[0;34m(path_to_folds, cols, params, verbose, iterations, validate_on_both)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidate_on_both\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m             \u001b[0mvalidation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlgb_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlgb_test\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mmodel_lgm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlgb_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_sets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"valid_0\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"auc\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tsdaemon/anaconda3/lib/python3.6/site-packages/lightgbm-0.1-py3.6.egg/lightgbm/engine.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, callbacks)\u001b[0m\n\u001b[1;32m    178\u001b[0m                                     evaluation_result_list=None))\n\u001b[1;32m    179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m         \u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tsdaemon/anaconda3/lib/python3.6/site-packages/lightgbm-0.1-py3.6.egg/lightgbm/basic.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, train_set, fobj)\u001b[0m\n\u001b[1;32m   1353\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[1;32m   1354\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1355\u001b[0;31m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[1;32m   1356\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mFalse\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "evs = mlgbm.crossvalidate_model(path_to_folds,cols,params,validate_on_both=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>genre_id</th>\n",
       "      <th>ts_listen</th>\n",
       "      <th>media_id</th>\n",
       "      <th>album_id</th>\n",
       "      <th>context_type</th>\n",
       "      <th>release_date</th>\n",
       "      <th>platform_name</th>\n",
       "      <th>platform_family</th>\n",
       "      <th>media_duration</th>\n",
       "      <th>...</th>\n",
       "      <th>user_gender</th>\n",
       "      <th>user_id</th>\n",
       "      <th>artist_id</th>\n",
       "      <th>user_age</th>\n",
       "      <th>release_date_year</th>\n",
       "      <th>release_date_month</th>\n",
       "      <th>release_date_day</th>\n",
       "      <th>release_date_decade</th>\n",
       "      <th>release_date_listened_this_decade</th>\n",
       "      <th>release_date_listened_this_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>1478104371</td>\n",
       "      <td>683078</td>\n",
       "      <td>82356</td>\n",
       "      <td>1</td>\n",
       "      <td>20021008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>542</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>17698</td>\n",
       "      <td>2076</td>\n",
       "      <td>30</td>\n",
       "      <td>2002</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.115385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2744</td>\n",
       "      <td>1479317140</td>\n",
       "      <td>876497</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>307</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>10525</td>\n",
       "      <td>26</td>\n",
       "      <td>28</td>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>0.016854</td>\n",
       "      <td>0.011236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2744</td>\n",
       "      <td>1479546361</td>\n",
       "      <td>876497</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>307</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>8716</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>0.004000</td>\n",
       "      <td>0.004000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2744</td>\n",
       "      <td>1478457729</td>\n",
       "      <td>876500</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>265</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>5443</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>0.044053</td>\n",
       "      <td>0.008811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2744</td>\n",
       "      <td>1480448560</td>\n",
       "      <td>876504</td>\n",
       "      <td>99692</td>\n",
       "      <td>1</td>\n",
       "      <td>19851231</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>356</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>7600</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "      <td>1985</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>0.094771</td>\n",
       "      <td>0.068627</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample_id  genre_id   ts_listen  media_id  album_id  context_type  \\\n",
       "0          0        50  1478104371    683078     82356             1   \n",
       "1          1      2744  1479317140    876497     99692             1   \n",
       "2          2      2744  1479546361    876497     99692             1   \n",
       "3          3      2744  1478457729    876500     99692             1   \n",
       "4          4      2744  1480448560    876504     99692             1   \n",
       "\n",
       "   release_date  platform_name  platform_family  media_duration  \\\n",
       "0      20021008              0                0             542   \n",
       "1      19851231              0                0             307   \n",
       "2      19851231              0                0             307   \n",
       "3      19851231              2                1             265   \n",
       "4      19851231              2                1             356   \n",
       "\n",
       "                ...                 user_gender  user_id  artist_id  user_age  \\\n",
       "0               ...                           0    17698       2076        30   \n",
       "1               ...                           0    10525         26        28   \n",
       "2               ...                           0     8716         26        27   \n",
       "3               ...                           0     5443         26        30   \n",
       "4               ...                           0     7600         26        29   \n",
       "\n",
       "   release_date_year  release_date_month  release_date_day  \\\n",
       "0               2002                  10                 8   \n",
       "1               1985                  12                31   \n",
       "2               1985                  12                31   \n",
       "3               1985                  12                31   \n",
       "4               1985                  12                31   \n",
       "\n",
       "   release_date_decade  release_date_listened_this_decade  \\\n",
       "0                   10                           0.538462   \n",
       "1                    8                           0.016854   \n",
       "2                    8                           0.004000   \n",
       "3                    8                           0.044053   \n",
       "4                    8                           0.094771   \n",
       "\n",
       "   release_date_listened_this_year  \n",
       "0                         0.115385  \n",
       "1                         0.011236  \n",
       "2                         0.004000  \n",
       "3                         0.008811  \n",
       "4                         0.068627  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path_to_sample = './data/sample_submission_kaggle.csv'\n",
    "path_to_submission = './data/submission.csv'\n",
    "path_to_test = './data/test.csv'\n",
    "\n",
    "test = pd.read_csv(path_to_test, encoding='utf-8')\n",
    "test = pd.concat([test, pd.read_csv(path_to_release_date_test, encoding='utf-8')], axis=1)\n",
    "\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fs = mlgbm.create_submission(train, test, path_to_sample, params, cols, path_to_submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('user_id', 6422),\n",
       " ('user_age', 3412),\n",
       " ('context_type', 3310),\n",
       " ('artist_id', 2079),\n",
       " ('release_date_listened_this_decade', 1631),\n",
       " ('release_date', 1625),\n",
       " ('release_date_listened_this_year', 1458),\n",
       " ('media_duration', 1284),\n",
       " ('media_id', 1283),\n",
       " ('genre_id', 1238),\n",
       " ('platform_name', 1132),\n",
       " ('album_id', 1038),\n",
       " ('user_gender', 929),\n",
       " ('platform_family', 849),\n",
       " ('release_date_day', 784),\n",
       " ('listen_type', 762),\n",
       " ('release_date_month', 415),\n",
       " ('release_date_year', 329),\n",
       " ('release_date_decade', 20)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlgbm.feature_score(fs,cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}