Part_3_Processing_new_data_for_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Add sentiments to, clean up and engineer the twitter data and combine it with stock data\n",
    "Use this notebook for adding sentiments, cleaning and engineering the twitter data and combining it with stock data (data scraped during March 2019)\n",
    "\n",
    "Resources:\n",
    "Original twitter data from 2016 for the training:\n",
    "http://followthehashtag.com/datasets/nasdaq-100-companies-free-twitter-dataset/\n",
    "\n",
    "Sentiment analysis:\n",
    "https://github.com/cjhutto/vaderSentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "import datetime as dt\n",
    "import pandas_datareader.data as web\n",
    "import math\n",
    "import xlsxwriter\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Change file- and stockname/cashtag and run all cells\n",
    "In the cell below, as well as the last cell in this notebook, for each new stock change file- and stockname/cashtag and run all cells."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xls = pd.ExcelFile('C:/Users/jangn/OneDrive/CODE/Stock_markets/Cashtag_Tweets_scraper/$WDC/$WDC_3.xlsx') # CHANGE FILE NAME!!!\n",
    "\n",
    "stock = \"WDC\" #CHANGE STOCK TICKER!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created at</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-03-28 18:43:06</td>\n",
       "      <td>Bullish Option Order Flow Sentiment March 28 h...</td>\n",
       "      <td>2089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-03-28 18:10:26</td>\n",
       "      <td>RT @247WallSt: Thursday’s Top Analyst Upgrades...</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-03-28 17:55:39</td>\n",
       "      <td>$WDC in at $45.68</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-03-28 17:54:13</td>\n",
       "      <td>$WDC potential EOD runner</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-03-28 17:24:12</td>\n",
       "      <td>Western Digital $WDC Holding Boosted by Orca I...</td>\n",
       "      <td>419</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           created at                                               text  \\\n",
       "0 2019-03-28 18:43:06  Bullish Option Order Flow Sentiment March 28 h...   \n",
       "1 2019-03-28 18:10:26  RT @247WallSt: Thursday’s Top Analyst Upgrades...   \n",
       "2 2019-03-28 17:55:39                                  $WDC in at $45.68   \n",
       "3 2019-03-28 17:54:13                          $WDC potential EOD runner   \n",
       "4 2019-03-28 17:24:12  Western Digital $WDC Holding Boosted by Orca I...   \n",
       "\n",
       "   follower count  \n",
       "0            2089  \n",
       "1              32  \n",
       "2              49  \n",
       "3              49  \n",
       "4             419  "
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel(xls, header = 0,encoding='latin-1')\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Add column with just the date, remove column with date & time, rearrange columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created at</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>2019-02-28 23:13:26</td>\n",
       "      <td>Thurs (2/28/19) highest % above OR 30-min high...</td>\n",
       "      <td>792</td>\n",
       "      <td>2019-02-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>2019-02-28 22:55:24</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>186</td>\n",
       "      <td>2019-02-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1115</th>\n",
       "      <td>2019-02-28 22:44:00</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>149</td>\n",
       "      <td>2019-02-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1116</th>\n",
       "      <td>2019-02-28 22:28:30</td>\n",
       "      <td>$MNST gapped up on #earnings, gaining over 8% ...</td>\n",
       "      <td>1569</td>\n",
       "      <td>2019-02-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1117</th>\n",
       "      <td>2019-02-28 21:00:02</td>\n",
       "      <td>Here is a snapshot of the current market. Reme...</td>\n",
       "      <td>1292</td>\n",
       "      <td>2019-02-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              created at                                               text  \\\n",
       "1113 2019-02-28 23:13:26  Thurs (2/28/19) highest % above OR 30-min high...   \n",
       "1114 2019-02-28 22:55:24  RT @SharePlanner: Stocks For You To Swing-Trad...   \n",
       "1115 2019-02-28 22:44:00  RT @SharePlanner: Stocks For You To Swing-Trad...   \n",
       "1116 2019-02-28 22:28:30  $MNST gapped up on #earnings, gaining over 8% ...   \n",
       "1117 2019-02-28 21:00:02  Here is a snapshot of the current market. Reme...   \n",
       "\n",
       "      follower count        date  \n",
       "1113             792  2019-02-28  \n",
       "1114             186  2019-02-28  \n",
       "1115             149  2019-02-28  \n",
       "1116            1569  2019-02-28  \n",
       "1117            1292  2019-02-28  "
      ]
     },
     "execution_count": 392,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"date\"] = df[\"created at\"].dt.date\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Add sentiment to each tweet using Vader\n",
    "Extract the content.<br>\n",
    "Make sure all tweets with an empty text field are deleted before doing the senitment analysis!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Bullish Option Order Flow Sentiment March 28 h...\n",
       "1    RT @247WallSt: Thursday’s Top Analyst Upgrades...\n",
       "2                                    $WDC in at $45.68\n",
       "3                            $WDC potential EOD runner\n",
       "4    Western Digital $WDC Holding Boosted by Orca I...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Tweet = df['text']\n",
    "Tweet.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sentimentScore(Tweet):\n",
    "    analyzer = SentimentIntensityAnalyzer()\n",
    "    results = []\n",
    "    for sentence in Tweet:\n",
    "        vs = analyzer.polarity_scores(sentence)\n",
    "        print(\"Vader score: \" + str(vs))\n",
    "        #print(\"{: <65} {}\".format(sentence, str(vs))) \n",
    "        #NOTE! I blocked the second print command so the sentences are\n",
    "        #left out in the cell below, purely for clarity reasons\n",
    "        results.append(vs)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(sentimentScore(Tweet));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Combining the two dataframes\n",
    "Next, I combine the **df_tweets** dataframe with the **df_results** sentiment dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.2023</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.917</td>\n",
       "      <td>0.083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.048</td>\n",
       "      <td>0.801</td>\n",
       "      <td>0.151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   compound    neg    neu    pos\n",
       "0    0.0000  0.000  1.000  0.000\n",
       "1    0.2023  0.000  0.917  0.083\n",
       "2    0.0000  0.000  1.000  0.000\n",
       "3    0.0000  0.000  1.000  0.000\n",
       "4    0.4939  0.048  0.801  0.151"
      ]
     },
     "execution_count": 396,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created at</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>date</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>2019-02-28 23:13:26</td>\n",
       "      <td>Thurs (2/28/19) highest % above OR 30-min high...</td>\n",
       "      <td>792</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>0.032</td>\n",
       "      <td>0.896</td>\n",
       "      <td>0.072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>2019-02-28 22:55:24</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>186</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1115</th>\n",
       "      <td>2019-02-28 22:44:00</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>149</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1116</th>\n",
       "      <td>2019-02-28 22:28:30</td>\n",
       "      <td>$MNST gapped up on #earnings, gaining over 8% ...</td>\n",
       "      <td>1569</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.7351</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.829</td>\n",
       "      <td>0.171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1117</th>\n",
       "      <td>2019-02-28 21:00:02</td>\n",
       "      <td>Here is a snapshot of the current market. Reme...</td>\n",
       "      <td>1292</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.3595</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              created at                                               text  \\\n",
       "1113 2019-02-28 23:13:26  Thurs (2/28/19) highest % above OR 30-min high...   \n",
       "1114 2019-02-28 22:55:24  RT @SharePlanner: Stocks For You To Swing-Trad...   \n",
       "1115 2019-02-28 22:44:00  RT @SharePlanner: Stocks For You To Swing-Trad...   \n",
       "1116 2019-02-28 22:28:30  $MNST gapped up on #earnings, gaining over 8% ...   \n",
       "1117 2019-02-28 21:00:02  Here is a snapshot of the current market. Reme...   \n",
       "\n",
       "      follower count        date  compound    neg    neu    pos  \n",
       "1113             792  2019-02-28    0.4404  0.032  0.896  0.072  \n",
       "1114             186  2019-02-28    0.0000  0.000  1.000  0.000  \n",
       "1115             149  2019-02-28    0.0000  0.000  1.000  0.000  \n",
       "1116            1569  2019-02-28    0.7351  0.000  0.829  0.171  \n",
       "1117            1292  2019-02-28    0.3595  0.000  0.935  0.065  "
      ]
     },
     "execution_count": 397,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweets = pd.merge(df, df_results, left_index=True, right_index=True)\n",
    "\n",
    "df_tweets.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Converting 'date' column from object to datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tweets['date'] = pd.to_datetime(df_tweets['date'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Choose the common range for the dataframes to be used for all tweet data\n",
    "Using range 28.2.2019 - 28.3.2019 = nn days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created at</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>date</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>2019-02-28 23:13:26</td>\n",
       "      <td>Thurs (2/28/19) highest % above OR 30-min high...</td>\n",
       "      <td>792</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>0.032</td>\n",
       "      <td>0.896</td>\n",
       "      <td>0.072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>2019-02-28 22:55:24</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>186</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1115</th>\n",
       "      <td>2019-02-28 22:44:00</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>149</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1116</th>\n",
       "      <td>2019-02-28 22:28:30</td>\n",
       "      <td>$MNST gapped up on #earnings, gaining over 8% ...</td>\n",
       "      <td>1569</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.7351</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.829</td>\n",
       "      <td>0.171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1117</th>\n",
       "      <td>2019-02-28 21:00:02</td>\n",
       "      <td>Here is a snapshot of the current market. Reme...</td>\n",
       "      <td>1292</td>\n",
       "      <td>2019-02-28</td>\n",
       "      <td>0.3595</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              created at                                               text  \\\n",
       "1113 2019-02-28 23:13:26  Thurs (2/28/19) highest % above OR 30-min high...   \n",
       "1114 2019-02-28 22:55:24  RT @SharePlanner: Stocks For You To Swing-Trad...   \n",
       "1115 2019-02-28 22:44:00  RT @SharePlanner: Stocks For You To Swing-Trad...   \n",
       "1116 2019-02-28 22:28:30  $MNST gapped up on #earnings, gaining over 8% ...   \n",
       "1117 2019-02-28 21:00:02  Here is a snapshot of the current market. Reme...   \n",
       "\n",
       "      follower count       date  compound    neg    neu    pos  \n",
       "1113             792 2019-02-28    0.4404  0.032  0.896  0.072  \n",
       "1114             186 2019-02-28    0.0000  0.000  1.000  0.000  \n",
       "1115             149 2019-02-28    0.0000  0.000  1.000  0.000  \n",
       "1116            1569 2019-02-28    0.7351  0.000  0.829  0.171  \n",
       "1117            1292 2019-02-28    0.3595  0.000  0.935  0.065  "
      ]
     },
     "execution_count": 399,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweets = df_tweets[(df_tweets['date'] >= '2019-02-28') & (df_tweets['date'] <= '2019-03-28')]\n",
    "\n",
    "df_tweets.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Adding a datetime column/index\n",
    "Next, removing the 'Date' column and adding a datetime column/index. This format is needed for the daily average calculations!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created at</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>2019-02-28 23:13:26</td>\n",
       "      <td>Thurs (2/28/19) highest % above OR 30-min high...</td>\n",
       "      <td>792</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>0.032</td>\n",
       "      <td>0.896</td>\n",
       "      <td>0.072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>2019-02-28 22:55:24</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>186</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>2019-02-28 22:44:00</td>\n",
       "      <td>RT @SharePlanner: Stocks For You To Swing-Trad...</td>\n",
       "      <td>149</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>2019-02-28 22:28:30</td>\n",
       "      <td>$MNST gapped up on #earnings, gaining over 8% ...</td>\n",
       "      <td>1569</td>\n",
       "      <td>0.7351</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.829</td>\n",
       "      <td>0.171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>2019-02-28 21:00:02</td>\n",
       "      <td>Here is a snapshot of the current market. Reme...</td>\n",
       "      <td>1292</td>\n",
       "      <td>0.3595</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    created at  \\\n",
       "datetime                         \n",
       "2019-02-28 2019-02-28 23:13:26   \n",
       "2019-02-28 2019-02-28 22:55:24   \n",
       "2019-02-28 2019-02-28 22:44:00   \n",
       "2019-02-28 2019-02-28 22:28:30   \n",
       "2019-02-28 2019-02-28 21:00:02   \n",
       "\n",
       "                                                         text  follower count  \\\n",
       "datetime                                                                        \n",
       "2019-02-28  Thurs (2/28/19) highest % above OR 30-min high...             792   \n",
       "2019-02-28  RT @SharePlanner: Stocks For You To Swing-Trad...             186   \n",
       "2019-02-28  RT @SharePlanner: Stocks For You To Swing-Trad...             149   \n",
       "2019-02-28  $MNST gapped up on #earnings, gaining over 8% ...            1569   \n",
       "2019-02-28  Here is a snapshot of the current market. Reme...            1292   \n",
       "\n",
       "            compound    neg    neu    pos  \n",
       "datetime                                   \n",
       "2019-02-28    0.4404  0.032  0.896  0.072  \n",
       "2019-02-28    0.0000  0.000  1.000  0.000  \n",
       "2019-02-28    0.0000  0.000  1.000  0.000  \n",
       "2019-02-28    0.7351  0.000  0.829  0.171  \n",
       "2019-02-28    0.3595  0.000  0.935  0.065  "
      ]
     },
     "execution_count": 400,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweets['datetime'] = pd.to_datetime(df_tweets['date']) # change of created at column to datetime columns\n",
    "df_tweet_SA = df_tweets.set_index('datetime') # creates a new dataframe 'df_tweet_SA' with the new index column datetime \n",
    "df_tweet_SA.drop(['date'], axis=1, inplace=True) #drops the original 'created at' column from the dataframe\n",
    "df_tweet_SA.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Slimming down the stream into a dataframe with only relevant columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>Bullish Option Order Flow Sentiment March 28 h...</td>\n",
       "      <td>2089</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>RT @247WallSt: Thursday’s Top Analyst Upgrades...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.2023</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.917</td>\n",
       "      <td>0.083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>$WDC in at $45.68</td>\n",
       "      <td>49</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>$WDC potential EOD runner</td>\n",
       "      <td>49</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>Western Digital $WDC Holding Boosted by Orca I...</td>\n",
       "      <td>419</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.048</td>\n",
       "      <td>0.801</td>\n",
       "      <td>0.151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    datetime                                               text  \\\n",
       "0 2019-03-28  Bullish Option Order Flow Sentiment March 28 h...   \n",
       "1 2019-03-28  RT @247WallSt: Thursday’s Top Analyst Upgrades...   \n",
       "2 2019-03-28                                  $WDC in at $45.68   \n",
       "3 2019-03-28                          $WDC potential EOD runner   \n",
       "4 2019-03-28  Western Digital $WDC Holding Boosted by Orca I...   \n",
       "\n",
       "   follower count  compound    neg    neu    pos  \n",
       "0            2089    0.0000  0.000  1.000  0.000  \n",
       "1              32    0.2023  0.000  0.917  0.083  \n",
       "2              49    0.0000  0.000  1.000  0.000  \n",
       "3              49    0.0000  0.000  1.000  0.000  \n",
       "4             419    0.4939  0.048  0.801  0.151  "
      ]
     },
     "execution_count": 401,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweet_SA = df_tweets[['datetime', 'text', 'follower count', 'compound','neg','neu','pos']]\n",
    "\n",
    "df_tweet_SA.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Remove tweets were compound is zero, i.e. sentiment is neutral\n",
    "Neutral tweets do not reflect a positive or negative mood and serve therefore no purpose to this analysis. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_tweet_SA = df_tweet_SA[(df_tweet_SA[['compound']] != 0).all(axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Create new column with the 'compound' multiplied by nr of followers of the Tweeter\n",
    "The logic being, if the tweeter has many followers, the compound of the his/her tweet has more significance compared to a tweet by a \"less significant\" tweeter, with less followers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_tweet_SA['Compound_multiplied'] = df_tweet_SA['compound']*df_tweet_SA['follower count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>RT @247WallSt: Thursday’s Top Analyst Upgrades...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.2023</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.917</td>\n",
       "      <td>0.083</td>\n",
       "      <td>6.4736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>Western Digital $WDC Holding Boosted by Orca I...</td>\n",
       "      <td>419</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.048</td>\n",
       "      <td>0.801</td>\n",
       "      <td>0.151</td>\n",
       "      <td>206.9441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>Top Bullish Flow (a/o12:50pmEST): $BAC $FB $W ...</td>\n",
       "      <td>5475</td>\n",
       "      <td>0.5574</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.805</td>\n",
       "      <td>0.195</td>\n",
       "      <td>3051.7650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>$WDC - Western Digital: Deep Value Trap Or Gem...</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0258</td>\n",
       "      <td>0.197</td>\n",
       "      <td>0.598</td>\n",
       "      <td>0.205</td>\n",
       "      <td>0.4128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2019-03-28</td>\n",
       "      <td>#estate asset watch - stocks with the highest ...</td>\n",
       "      <td>12567</td>\n",
       "      <td>0.6705</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.899</td>\n",
       "      <td>0.101</td>\n",
       "      <td>8426.1735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    datetime                                               text  \\\n",
       "1 2019-03-28  RT @247WallSt: Thursday’s Top Analyst Upgrades...   \n",
       "4 2019-03-28  Western Digital $WDC Holding Boosted by Orca I...   \n",
       "6 2019-03-28  Top Bullish Flow (a/o12:50pmEST): $BAC $FB $W ...   \n",
       "7 2019-03-28  $WDC - Western Digital: Deep Value Trap Or Gem...   \n",
       "9 2019-03-28  #estate asset watch - stocks with the highest ...   \n",
       "\n",
       "   follower count  compound    neg    neu    pos  Compound_multiplied  \n",
       "1              32    0.2023  0.000  0.917  0.083               6.4736  \n",
       "4             419    0.4939  0.048  0.801  0.151             206.9441  \n",
       "6            5475    0.5574  0.000  0.805  0.195            3051.7650  \n",
       "7              16    0.0258  0.197  0.598  0.205               0.4128  \n",
       "9           12567    0.6705  0.000  0.899  0.101            8426.1735  "
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweet_SA.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Remove rows where 'follower count' is NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>text</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [datetime, text, follower count, compound, neg, neu, pos, Compound_multiplied]\n",
       "Index: []"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nan_rows = df_tweet_SA[df_tweet_SA['follower count'].isnull()]\n",
    "nan_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_tweet_SA = df_tweet_SA[np.isfinite(df_tweet_SA['follower count'])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Create a df with daily MEANS of each column \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-03-24</th>\n",
       "      <td>1795.666667</td>\n",
       "      <td>0.301833</td>\n",
       "      <td>0.024000</td>\n",
       "      <td>0.857778</td>\n",
       "      <td>0.118333</td>\n",
       "      <td>664.875244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-25</th>\n",
       "      <td>9106.875000</td>\n",
       "      <td>0.501550</td>\n",
       "      <td>0.026937</td>\n",
       "      <td>0.769750</td>\n",
       "      <td>0.203312</td>\n",
       "      <td>1562.571862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-26</th>\n",
       "      <td>802.333333</td>\n",
       "      <td>0.319411</td>\n",
       "      <td>0.042222</td>\n",
       "      <td>0.836944</td>\n",
       "      <td>0.120833</td>\n",
       "      <td>84.944561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-27</th>\n",
       "      <td>1880.969697</td>\n",
       "      <td>0.168927</td>\n",
       "      <td>0.035091</td>\n",
       "      <td>0.861788</td>\n",
       "      <td>0.103121</td>\n",
       "      <td>659.380088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-28</th>\n",
       "      <td>10861.230769</td>\n",
       "      <td>0.400405</td>\n",
       "      <td>0.036846</td>\n",
       "      <td>0.764385</td>\n",
       "      <td>0.198718</td>\n",
       "      <td>3291.307072</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            follower count  compound       neg       neu       pos  \\\n",
       "datetime                                                             \n",
       "2019-03-24     1795.666667  0.301833  0.024000  0.857778  0.118333   \n",
       "2019-03-25     9106.875000  0.501550  0.026937  0.769750  0.203312   \n",
       "2019-03-26      802.333333  0.319411  0.042222  0.836944  0.120833   \n",
       "2019-03-27     1880.969697  0.168927  0.035091  0.861788  0.103121   \n",
       "2019-03-28    10861.230769  0.400405  0.036846  0.764385  0.198718   \n",
       "\n",
       "            Compound_multiplied  \n",
       "datetime                         \n",
       "2019-03-24           664.875244  \n",
       "2019-03-25          1562.571862  \n",
       "2019-03-26            84.944561  \n",
       "2019-03-27           659.380088  \n",
       "2019-03-28          3291.307072  "
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_daily_mean=(df_tweet_SA.groupby(df_tweet_SA.datetime).mean())\n",
    "\n",
    "df_daily_mean.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_daily_mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Downloading stock data from Yahoo Finance - remember to adjust dates when needed\n",
    "They should be the same as for the new, scraped twitter data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#import pandas_datareader.data as web\n",
    "\n",
    "start = dt.datetime(2019, 2, 28)\n",
    "end =  dt.datetime(2019, 3, 28) #dt.datetime.now() \n",
    "\n",
    "df_stock = web.DataReader(stock, 'yahoo', start, end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Adj Close</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-03-22</th>\n",
       "      <td>51.450001</td>\n",
       "      <td>48.369999</td>\n",
       "      <td>51.250000</td>\n",
       "      <td>48.389999</td>\n",
       "      <td>8670800.0</td>\n",
       "      <td>47.856834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-25</th>\n",
       "      <td>48.849998</td>\n",
       "      <td>47.090000</td>\n",
       "      <td>48.250000</td>\n",
       "      <td>47.889999</td>\n",
       "      <td>4426500.0</td>\n",
       "      <td>47.362343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-26</th>\n",
       "      <td>48.930000</td>\n",
       "      <td>46.869999</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>47.080002</td>\n",
       "      <td>6381900.0</td>\n",
       "      <td>46.561272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-27</th>\n",
       "      <td>47.450001</td>\n",
       "      <td>44.939999</td>\n",
       "      <td>47.230000</td>\n",
       "      <td>45.380001</td>\n",
       "      <td>8924400.0</td>\n",
       "      <td>44.880001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-28</th>\n",
       "      <td>46.279999</td>\n",
       "      <td>44.759998</td>\n",
       "      <td>44.919998</td>\n",
       "      <td>45.750000</td>\n",
       "      <td>5710600.0</td>\n",
       "      <td>45.750000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 High        Low       Open      Close     Volume  Adj Close\n",
       "Date                                                                        \n",
       "2019-03-22  51.450001  48.369999  51.250000  48.389999  8670800.0  47.856834\n",
       "2019-03-25  48.849998  47.090000  48.250000  47.889999  4426500.0  47.362343\n",
       "2019-03-26  48.930000  46.869999  48.189999  47.080002  6381900.0  46.561272\n",
       "2019-03-27  47.450001  44.939999  47.230000  45.380001  8924400.0  44.880001\n",
       "2019-03-28  46.279999  44.759998  44.919998  45.750000  5710600.0  45.750000"
      ]
     },
     "execution_count": 410,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stock.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### New column for daily percent change - stock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_stock['Pct_change'] = (df_stock['Close'] - df_stock['Open']) / df_stock['Open'] * 100.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Pct_change</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-03-22</th>\n",
       "      <td>51.450001</td>\n",
       "      <td>48.369999</td>\n",
       "      <td>51.250000</td>\n",
       "      <td>48.389999</td>\n",
       "      <td>8670800.0</td>\n",
       "      <td>47.856834</td>\n",
       "      <td>-5.580489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-25</th>\n",
       "      <td>48.849998</td>\n",
       "      <td>47.090000</td>\n",
       "      <td>48.250000</td>\n",
       "      <td>47.889999</td>\n",
       "      <td>4426500.0</td>\n",
       "      <td>47.362343</td>\n",
       "      <td>-0.746115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-26</th>\n",
       "      <td>48.930000</td>\n",
       "      <td>46.869999</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>47.080002</td>\n",
       "      <td>6381900.0</td>\n",
       "      <td>46.561272</td>\n",
       "      <td>-2.303376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-27</th>\n",
       "      <td>47.450001</td>\n",
       "      <td>44.939999</td>\n",
       "      <td>47.230000</td>\n",
       "      <td>45.380001</td>\n",
       "      <td>8924400.0</td>\n",
       "      <td>44.880001</td>\n",
       "      <td>-3.916999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-28</th>\n",
       "      <td>46.279999</td>\n",
       "      <td>44.759998</td>\n",
       "      <td>44.919998</td>\n",
       "      <td>45.750000</td>\n",
       "      <td>5710600.0</td>\n",
       "      <td>45.750000</td>\n",
       "      <td>1.847733</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 High        Low       Open      Close     Volume  Adj Close  \\\n",
       "Date                                                                           \n",
       "2019-03-22  51.450001  48.369999  51.250000  48.389999  8670800.0  47.856834   \n",
       "2019-03-25  48.849998  47.090000  48.250000  47.889999  4426500.0  47.362343   \n",
       "2019-03-26  48.930000  46.869999  48.189999  47.080002  6381900.0  46.561272   \n",
       "2019-03-27  47.450001  44.939999  47.230000  45.380001  8924400.0  44.880001   \n",
       "2019-03-28  46.279999  44.759998  44.919998  45.750000  5710600.0  45.750000   \n",
       "\n",
       "            Pct_change  \n",
       "Date                    \n",
       "2019-03-22   -5.580489  \n",
       "2019-03-25   -0.746115  \n",
       "2019-03-26   -2.303376  \n",
       "2019-03-27   -3.916999  \n",
       "2019-03-28    1.847733  "
      ]
     },
     "execution_count": 412,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stock.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Combine the tweet sentiment dataframe with the stock data dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Pct_change</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-03-18</th>\n",
       "      <td>48.669998</td>\n",
       "      <td>47.830002</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>47.757938</td>\n",
       "      <td>0.207517</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-19</th>\n",
       "      <td>48.840000</td>\n",
       "      <td>47.720001</td>\n",
       "      <td>48.720001</td>\n",
       "      <td>47.352455</td>\n",
       "      <td>-1.724138</td>\n",
       "      <td>1701.100000</td>\n",
       "      <td>0.549080</td>\n",
       "      <td>0.042500</td>\n",
       "      <td>0.756800</td>\n",
       "      <td>0.200600</td>\n",
       "      <td>1048.109500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-20</th>\n",
       "      <td>48.169998</td>\n",
       "      <td>46.560001</td>\n",
       "      <td>47.840000</td>\n",
       "      <td>46.650276</td>\n",
       "      <td>-1.400506</td>\n",
       "      <td>1040.625000</td>\n",
       "      <td>0.308138</td>\n",
       "      <td>0.024250</td>\n",
       "      <td>0.849938</td>\n",
       "      <td>0.125750</td>\n",
       "      <td>343.374687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-21</th>\n",
       "      <td>52.270000</td>\n",
       "      <td>47.869999</td>\n",
       "      <td>47.869999</td>\n",
       "      <td>51.209480</td>\n",
       "      <td>8.167955</td>\n",
       "      <td>4087.935065</td>\n",
       "      <td>0.290555</td>\n",
       "      <td>0.036558</td>\n",
       "      <td>0.832312</td>\n",
       "      <td>0.131117</td>\n",
       "      <td>1253.919414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-22</th>\n",
       "      <td>51.450001</td>\n",
       "      <td>48.369999</td>\n",
       "      <td>51.250000</td>\n",
       "      <td>47.856834</td>\n",
       "      <td>-5.580489</td>\n",
       "      <td>3233.733333</td>\n",
       "      <td>0.276140</td>\n",
       "      <td>0.020133</td>\n",
       "      <td>0.875667</td>\n",
       "      <td>0.104200</td>\n",
       "      <td>1158.498953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2556.142857</td>\n",
       "      <td>0.240871</td>\n",
       "      <td>0.026857</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>0.141143</td>\n",
       "      <td>975.690900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1795.666667</td>\n",
       "      <td>0.301833</td>\n",
       "      <td>0.024000</td>\n",
       "      <td>0.857778</td>\n",
       "      <td>0.118333</td>\n",
       "      <td>664.875244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-25</th>\n",
       "      <td>48.849998</td>\n",
       "      <td>47.090000</td>\n",
       "      <td>48.250000</td>\n",
       "      <td>47.362343</td>\n",
       "      <td>-0.746115</td>\n",
       "      <td>9106.875000</td>\n",
       "      <td>0.501550</td>\n",
       "      <td>0.026937</td>\n",
       "      <td>0.769750</td>\n",
       "      <td>0.203312</td>\n",
       "      <td>1562.571862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-26</th>\n",
       "      <td>48.930000</td>\n",
       "      <td>46.869999</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>46.561272</td>\n",
       "      <td>-2.303376</td>\n",
       "      <td>802.333333</td>\n",
       "      <td>0.319411</td>\n",
       "      <td>0.042222</td>\n",
       "      <td>0.836944</td>\n",
       "      <td>0.120833</td>\n",
       "      <td>84.944561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-27</th>\n",
       "      <td>47.450001</td>\n",
       "      <td>44.939999</td>\n",
       "      <td>47.230000</td>\n",
       "      <td>44.880001</td>\n",
       "      <td>-3.916999</td>\n",
       "      <td>1880.969697</td>\n",
       "      <td>0.168927</td>\n",
       "      <td>0.035091</td>\n",
       "      <td>0.861788</td>\n",
       "      <td>0.103121</td>\n",
       "      <td>659.380088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-28</th>\n",
       "      <td>46.279999</td>\n",
       "      <td>44.759998</td>\n",
       "      <td>44.919998</td>\n",
       "      <td>45.750000</td>\n",
       "      <td>1.847733</td>\n",
       "      <td>10861.230769</td>\n",
       "      <td>0.400405</td>\n",
       "      <td>0.036846</td>\n",
       "      <td>0.764385</td>\n",
       "      <td>0.198718</td>\n",
       "      <td>3291.307072</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 High        Low       Open  Adj Close  Pct_change  \\\n",
       "2019-03-18  48.669998  47.830002  48.189999  47.757938    0.207517   \n",
       "2019-03-19  48.840000  47.720001  48.720001  47.352455   -1.724138   \n",
       "2019-03-20  48.169998  46.560001  47.840000  46.650276   -1.400506   \n",
       "2019-03-21  52.270000  47.869999  47.869999  51.209480    8.167955   \n",
       "2019-03-22  51.450001  48.369999  51.250000  47.856834   -5.580489   \n",
       "2019-03-23        NaN        NaN        NaN        NaN         NaN   \n",
       "2019-03-24        NaN        NaN        NaN        NaN         NaN   \n",
       "2019-03-25  48.849998  47.090000  48.250000  47.362343   -0.746115   \n",
       "2019-03-26  48.930000  46.869999  48.189999  46.561272   -2.303376   \n",
       "2019-03-27  47.450001  44.939999  47.230000  44.880001   -3.916999   \n",
       "2019-03-28  46.279999  44.759998  44.919998  45.750000    1.847733   \n",
       "\n",
       "            follower count  compound       neg       neu       pos  \\\n",
       "2019-03-18             NaN       NaN       NaN       NaN       NaN   \n",
       "2019-03-19     1701.100000  0.549080  0.042500  0.756800  0.200600   \n",
       "2019-03-20     1040.625000  0.308138  0.024250  0.849938  0.125750   \n",
       "2019-03-21     4087.935065  0.290555  0.036558  0.832312  0.131117   \n",
       "2019-03-22     3233.733333  0.276140  0.020133  0.875667  0.104200   \n",
       "2019-03-23     2556.142857  0.240871  0.026857  0.832000  0.141143   \n",
       "2019-03-24     1795.666667  0.301833  0.024000  0.857778  0.118333   \n",
       "2019-03-25     9106.875000  0.501550  0.026937  0.769750  0.203312   \n",
       "2019-03-26      802.333333  0.319411  0.042222  0.836944  0.120833   \n",
       "2019-03-27     1880.969697  0.168927  0.035091  0.861788  0.103121   \n",
       "2019-03-28    10861.230769  0.400405  0.036846  0.764385  0.198718   \n",
       "\n",
       "            Compound_multiplied  \n",
       "2019-03-18                  NaN  \n",
       "2019-03-19          1048.109500  \n",
       "2019-03-20           343.374687  \n",
       "2019-03-21          1253.919414  \n",
       "2019-03-22          1158.498953  \n",
       "2019-03-23           975.690900  \n",
       "2019-03-24           664.875244  \n",
       "2019-03-25          1562.571862  \n",
       "2019-03-26            84.944561  \n",
       "2019-03-27           659.380088  \n",
       "2019-03-28          3291.307072  "
      ]
     },
     "execution_count": 413,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full = pd.concat([df_stock[['High', 'Low', 'Open', 'Adj Close', 'Pct_change']],\\\n",
    "                     df_daily_mean], axis=1, sort=False)\n",
    "df_full.tail(11)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Impute missing data with their means\n",
    "For some of the cashtag twitter data entire days were missing. Mean imputation replaces missing values with the mean value of that feature/variable. Mean imputation is one of the most ‘naive’ imputation methods because unlike more complex methods like k-nearest neighbors imputation, it does not use the information we have about an observation to estimate a value for it.\n",
    "\n",
    "It's not pretty, it's not fully accurate, but better than removing these days altogether. Besides, there might be other missing days for the other cashtag tweets and this cell fixes them all."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Pct_change</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-02-27</th>\n",
       "      <td>51.369999</td>\n",
       "      <td>48.50</td>\n",
       "      <td>50.830002</td>\n",
       "      <td>48.005184</td>\n",
       "      <td>-4.505215</td>\n",
       "      <td>4943.724921</td>\n",
       "      <td>0.298596</td>\n",
       "      <td>0.037299</td>\n",
       "      <td>0.830733</td>\n",
       "      <td>0.131963</td>\n",
       "      <td>1111.553143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>50.520000</td>\n",
       "      <td>46.66</td>\n",
       "      <td>48.150002</td>\n",
       "      <td>49.745789</td>\n",
       "      <td>4.465208</td>\n",
       "      <td>916.600000</td>\n",
       "      <td>0.255800</td>\n",
       "      <td>0.023200</td>\n",
       "      <td>0.915200</td>\n",
       "      <td>0.061600</td>\n",
       "      <td>369.520540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-01</th>\n",
       "      <td>51.950001</td>\n",
       "      <td>50.59</td>\n",
       "      <td>51.020000</td>\n",
       "      <td>51.090805</td>\n",
       "      <td>1.254409</td>\n",
       "      <td>6370.678571</td>\n",
       "      <td>0.378829</td>\n",
       "      <td>0.035893</td>\n",
       "      <td>0.805286</td>\n",
       "      <td>0.158893</td>\n",
       "      <td>2136.179221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-02</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1557.916667</td>\n",
       "      <td>0.535575</td>\n",
       "      <td>0.021583</td>\n",
       "      <td>0.785583</td>\n",
       "      <td>0.192750</td>\n",
       "      <td>886.068175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-03</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5174.818182</td>\n",
       "      <td>0.296073</td>\n",
       "      <td>0.010545</td>\n",
       "      <td>0.883000</td>\n",
       "      <td>0.106455</td>\n",
       "      <td>1072.697936</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 High    Low       Open  Adj Close  Pct_change  \\\n",
       "2019-02-27  51.369999  48.50  50.830002  48.005184   -4.505215   \n",
       "2019-02-28  50.520000  46.66  48.150002  49.745789    4.465208   \n",
       "2019-03-01  51.950001  50.59  51.020000  51.090805    1.254409   \n",
       "2019-03-02        NaN    NaN        NaN        NaN         NaN   \n",
       "2019-03-03        NaN    NaN        NaN        NaN         NaN   \n",
       "\n",
       "            follower count  compound       neg       neu       pos  \\\n",
       "2019-02-27     4943.724921  0.298596  0.037299  0.830733  0.131963   \n",
       "2019-02-28      916.600000  0.255800  0.023200  0.915200  0.061600   \n",
       "2019-03-01     6370.678571  0.378829  0.035893  0.805286  0.158893   \n",
       "2019-03-02     1557.916667  0.535575  0.021583  0.785583  0.192750   \n",
       "2019-03-03     5174.818182  0.296073  0.010545  0.883000  0.106455   \n",
       "\n",
       "            Compound_multiplied  \n",
       "2019-02-27          1111.553143  \n",
       "2019-02-28           369.520540  \n",
       "2019-03-01          2136.179221  \n",
       "2019-03-02           886.068175  \n",
       "2019-03-03          1072.697936  "
      ]
     },
     "execution_count": 414,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full['follower count'].fillna(df_full['follower count'].mean(), inplace=True)\n",
    "df_full['compound'].fillna(df_full['compound'].mean(), inplace=True)\n",
    "df_full['neg'].fillna(df_full['neg'].mean(), inplace=True)\n",
    "df_full['neu'].fillna(df_full['neu'].mean(), inplace=True)\n",
    "df_full['pos'].fillna(df_full['pos'].mean(), inplace=True)\n",
    "df_full['Compound_multiplied'].fillna(df_full['Compound_multiplied'].mean(), inplace=True)\n",
    "\n",
    "df_full.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Interpolate for missing weekend stock data!\n",
    "NOTE! Last date has no values (=NaN) for 'Predicted_change_stock' or 'Buy/Sell'.\n",
    "NOTE also! Tweets were missing for day Monday 18.3.2019 so sentiment values interpolated also for that date... :-("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_full = df_full[[ 'High', 'Low', 'Open', 'Adj Close', 'Pct_change', 'follower count', 'compound','neg', 'neu', 'pos',\\\n",
    "         'Compound_multiplied' ]].interpolate(method='linear', limit_direction='forward', axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Pct_change</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-03-07</th>\n",
       "      <td>47.860001</td>\n",
       "      <td>45.410000</td>\n",
       "      <td>47.860001</td>\n",
       "      <td>46.472260</td>\n",
       "      <td>-1.817800</td>\n",
       "      <td>2621.500000</td>\n",
       "      <td>0.307390</td>\n",
       "      <td>0.045350</td>\n",
       "      <td>0.823650</td>\n",
       "      <td>0.130950</td>\n",
       "      <td>1406.704325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-08</th>\n",
       "      <td>47.560001</td>\n",
       "      <td>45.730000</td>\n",
       "      <td>46.040001</td>\n",
       "      <td>46.956860</td>\n",
       "      <td>3.127712</td>\n",
       "      <td>13547.103448</td>\n",
       "      <td>0.151948</td>\n",
       "      <td>0.048069</td>\n",
       "      <td>0.853172</td>\n",
       "      <td>0.098759</td>\n",
       "      <td>-2245.066659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-09</th>\n",
       "      <td>48.373334</td>\n",
       "      <td>46.373333</td>\n",
       "      <td>46.610001</td>\n",
       "      <td>47.675521</td>\n",
       "      <td>3.418475</td>\n",
       "      <td>702.916667</td>\n",
       "      <td>0.360208</td>\n",
       "      <td>0.036833</td>\n",
       "      <td>0.799750</td>\n",
       "      <td>0.163417</td>\n",
       "      <td>-34.028050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-10</th>\n",
       "      <td>49.186667</td>\n",
       "      <td>47.016666</td>\n",
       "      <td>47.180000</td>\n",
       "      <td>48.394182</td>\n",
       "      <td>3.709237</td>\n",
       "      <td>2524.800000</td>\n",
       "      <td>-0.288960</td>\n",
       "      <td>0.091500</td>\n",
       "      <td>0.887200</td>\n",
       "      <td>0.021300</td>\n",
       "      <td>34.763340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-11</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>47.660000</td>\n",
       "      <td>47.750000</td>\n",
       "      <td>49.112843</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3704.083333</td>\n",
       "      <td>0.399446</td>\n",
       "      <td>0.024542</td>\n",
       "      <td>0.785167</td>\n",
       "      <td>0.190292</td>\n",
       "      <td>1394.352987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-12</th>\n",
       "      <td>50.560001</td>\n",
       "      <td>48.750000</td>\n",
       "      <td>49.689999</td>\n",
       "      <td>48.608463</td>\n",
       "      <td>-1.086732</td>\n",
       "      <td>4934.961538</td>\n",
       "      <td>0.183062</td>\n",
       "      <td>0.045692</td>\n",
       "      <td>0.852692</td>\n",
       "      <td>0.101615</td>\n",
       "      <td>648.260250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-13</th>\n",
       "      <td>49.470001</td>\n",
       "      <td>48.070000</td>\n",
       "      <td>49.139999</td>\n",
       "      <td>48.450226</td>\n",
       "      <td>-0.305246</td>\n",
       "      <td>2240.200000</td>\n",
       "      <td>0.235567</td>\n",
       "      <td>0.051667</td>\n",
       "      <td>0.796400</td>\n",
       "      <td>0.152000</td>\n",
       "      <td>744.901227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-14</th>\n",
       "      <td>48.720001</td>\n",
       "      <td>47.150002</td>\n",
       "      <td>48.599998</td>\n",
       "      <td>47.055759</td>\n",
       "      <td>-2.098759</td>\n",
       "      <td>2547.928571</td>\n",
       "      <td>0.392025</td>\n",
       "      <td>0.012607</td>\n",
       "      <td>0.835821</td>\n",
       "      <td>0.151571</td>\n",
       "      <td>1281.700721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-15</th>\n",
       "      <td>49.130001</td>\n",
       "      <td>47.639999</td>\n",
       "      <td>47.650002</td>\n",
       "      <td>47.570026</td>\n",
       "      <td>0.944380</td>\n",
       "      <td>1986.500000</td>\n",
       "      <td>0.067900</td>\n",
       "      <td>0.064111</td>\n",
       "      <td>0.864333</td>\n",
       "      <td>0.071556</td>\n",
       "      <td>520.731389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-16</th>\n",
       "      <td>48.976667</td>\n",
       "      <td>47.703334</td>\n",
       "      <td>47.830001</td>\n",
       "      <td>47.632664</td>\n",
       "      <td>0.698759</td>\n",
       "      <td>710.428571</td>\n",
       "      <td>0.595800</td>\n",
       "      <td>0.017286</td>\n",
       "      <td>0.853286</td>\n",
       "      <td>0.129429</td>\n",
       "      <td>502.585871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-17</th>\n",
       "      <td>48.823332</td>\n",
       "      <td>47.766668</td>\n",
       "      <td>48.010000</td>\n",
       "      <td>47.695301</td>\n",
       "      <td>0.453138</td>\n",
       "      <td>527.625000</td>\n",
       "      <td>0.547662</td>\n",
       "      <td>0.032875</td>\n",
       "      <td>0.798625</td>\n",
       "      <td>0.168500</td>\n",
       "      <td>32.746600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-18</th>\n",
       "      <td>48.669998</td>\n",
       "      <td>47.830002</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>47.757938</td>\n",
       "      <td>0.207517</td>\n",
       "      <td>4943.724921</td>\n",
       "      <td>0.298596</td>\n",
       "      <td>0.037299</td>\n",
       "      <td>0.830733</td>\n",
       "      <td>0.131963</td>\n",
       "      <td>1111.553143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-19</th>\n",
       "      <td>48.840000</td>\n",
       "      <td>47.720001</td>\n",
       "      <td>48.720001</td>\n",
       "      <td>47.352455</td>\n",
       "      <td>-1.724138</td>\n",
       "      <td>1701.100000</td>\n",
       "      <td>0.549080</td>\n",
       "      <td>0.042500</td>\n",
       "      <td>0.756800</td>\n",
       "      <td>0.200600</td>\n",
       "      <td>1048.109500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-20</th>\n",
       "      <td>48.169998</td>\n",
       "      <td>46.560001</td>\n",
       "      <td>47.840000</td>\n",
       "      <td>46.650276</td>\n",
       "      <td>-1.400506</td>\n",
       "      <td>1040.625000</td>\n",
       "      <td>0.308138</td>\n",
       "      <td>0.024250</td>\n",
       "      <td>0.849938</td>\n",
       "      <td>0.125750</td>\n",
       "      <td>343.374687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-21</th>\n",
       "      <td>52.270000</td>\n",
       "      <td>47.869999</td>\n",
       "      <td>47.869999</td>\n",
       "      <td>51.209480</td>\n",
       "      <td>8.167955</td>\n",
       "      <td>4087.935065</td>\n",
       "      <td>0.290555</td>\n",
       "      <td>0.036558</td>\n",
       "      <td>0.832312</td>\n",
       "      <td>0.131117</td>\n",
       "      <td>1253.919414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-22</th>\n",
       "      <td>51.450001</td>\n",
       "      <td>48.369999</td>\n",
       "      <td>51.250000</td>\n",
       "      <td>47.856834</td>\n",
       "      <td>-5.580489</td>\n",
       "      <td>3233.733333</td>\n",
       "      <td>0.276140</td>\n",
       "      <td>0.020133</td>\n",
       "      <td>0.875667</td>\n",
       "      <td>0.104200</td>\n",
       "      <td>1158.498953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-23</th>\n",
       "      <td>50.583333</td>\n",
       "      <td>47.943333</td>\n",
       "      <td>50.250000</td>\n",
       "      <td>47.692004</td>\n",
       "      <td>-3.969031</td>\n",
       "      <td>2556.142857</td>\n",
       "      <td>0.240871</td>\n",
       "      <td>0.026857</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>0.141143</td>\n",
       "      <td>975.690900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-24</th>\n",
       "      <td>49.716666</td>\n",
       "      <td>47.516666</td>\n",
       "      <td>49.250000</td>\n",
       "      <td>47.527173</td>\n",
       "      <td>-2.357573</td>\n",
       "      <td>1795.666667</td>\n",
       "      <td>0.301833</td>\n",
       "      <td>0.024000</td>\n",
       "      <td>0.857778</td>\n",
       "      <td>0.118333</td>\n",
       "      <td>664.875244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-25</th>\n",
       "      <td>48.849998</td>\n",
       "      <td>47.090000</td>\n",
       "      <td>48.250000</td>\n",
       "      <td>47.362343</td>\n",
       "      <td>-0.746115</td>\n",
       "      <td>9106.875000</td>\n",
       "      <td>0.501550</td>\n",
       "      <td>0.026937</td>\n",
       "      <td>0.769750</td>\n",
       "      <td>0.203312</td>\n",
       "      <td>1562.571862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-26</th>\n",
       "      <td>48.930000</td>\n",
       "      <td>46.869999</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>46.561272</td>\n",
       "      <td>-2.303376</td>\n",
       "      <td>802.333333</td>\n",
       "      <td>0.319411</td>\n",
       "      <td>0.042222</td>\n",
       "      <td>0.836944</td>\n",
       "      <td>0.120833</td>\n",
       "      <td>84.944561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-27</th>\n",
       "      <td>47.450001</td>\n",
       "      <td>44.939999</td>\n",
       "      <td>47.230000</td>\n",
       "      <td>44.880001</td>\n",
       "      <td>-3.916999</td>\n",
       "      <td>1880.969697</td>\n",
       "      <td>0.168927</td>\n",
       "      <td>0.035091</td>\n",
       "      <td>0.861788</td>\n",
       "      <td>0.103121</td>\n",
       "      <td>659.380088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-28</th>\n",
       "      <td>46.279999</td>\n",
       "      <td>44.759998</td>\n",
       "      <td>44.919998</td>\n",
       "      <td>45.750000</td>\n",
       "      <td>1.847733</td>\n",
       "      <td>10861.230769</td>\n",
       "      <td>0.400405</td>\n",
       "      <td>0.036846</td>\n",
       "      <td>0.764385</td>\n",
       "      <td>0.198718</td>\n",
       "      <td>3291.307072</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 High        Low       Open  Adj Close  Pct_change  \\\n",
       "2019-03-07  47.860001  45.410000  47.860001  46.472260   -1.817800   \n",
       "2019-03-08  47.560001  45.730000  46.040001  46.956860    3.127712   \n",
       "2019-03-09  48.373334  46.373333  46.610001  47.675521    3.418475   \n",
       "2019-03-10  49.186667  47.016666  47.180000  48.394182    3.709237   \n",
       "2019-03-11  50.000000  47.660000  47.750000  49.112843    4.000000   \n",
       "2019-03-12  50.560001  48.750000  49.689999  48.608463   -1.086732   \n",
       "2019-03-13  49.470001  48.070000  49.139999  48.450226   -0.305246   \n",
       "2019-03-14  48.720001  47.150002  48.599998  47.055759   -2.098759   \n",
       "2019-03-15  49.130001  47.639999  47.650002  47.570026    0.944380   \n",
       "2019-03-16  48.976667  47.703334  47.830001  47.632664    0.698759   \n",
       "2019-03-17  48.823332  47.766668  48.010000  47.695301    0.453138   \n",
       "2019-03-18  48.669998  47.830002  48.189999  47.757938    0.207517   \n",
       "2019-03-19  48.840000  47.720001  48.720001  47.352455   -1.724138   \n",
       "2019-03-20  48.169998  46.560001  47.840000  46.650276   -1.400506   \n",
       "2019-03-21  52.270000  47.869999  47.869999  51.209480    8.167955   \n",
       "2019-03-22  51.450001  48.369999  51.250000  47.856834   -5.580489   \n",
       "2019-03-23  50.583333  47.943333  50.250000  47.692004   -3.969031   \n",
       "2019-03-24  49.716666  47.516666  49.250000  47.527173   -2.357573   \n",
       "2019-03-25  48.849998  47.090000  48.250000  47.362343   -0.746115   \n",
       "2019-03-26  48.930000  46.869999  48.189999  46.561272   -2.303376   \n",
       "2019-03-27  47.450001  44.939999  47.230000  44.880001   -3.916999   \n",
       "2019-03-28  46.279999  44.759998  44.919998  45.750000    1.847733   \n",
       "\n",
       "            follower count  compound       neg       neu       pos  \\\n",
       "2019-03-07     2621.500000  0.307390  0.045350  0.823650  0.130950   \n",
       "2019-03-08    13547.103448  0.151948  0.048069  0.853172  0.098759   \n",
       "2019-03-09      702.916667  0.360208  0.036833  0.799750  0.163417   \n",
       "2019-03-10     2524.800000 -0.288960  0.091500  0.887200  0.021300   \n",
       "2019-03-11     3704.083333  0.399446  0.024542  0.785167  0.190292   \n",
       "2019-03-12     4934.961538  0.183062  0.045692  0.852692  0.101615   \n",
       "2019-03-13     2240.200000  0.235567  0.051667  0.796400  0.152000   \n",
       "2019-03-14     2547.928571  0.392025  0.012607  0.835821  0.151571   \n",
       "2019-03-15     1986.500000  0.067900  0.064111  0.864333  0.071556   \n",
       "2019-03-16      710.428571  0.595800  0.017286  0.853286  0.129429   \n",
       "2019-03-17      527.625000  0.547662  0.032875  0.798625  0.168500   \n",
       "2019-03-18     4943.724921  0.298596  0.037299  0.830733  0.131963   \n",
       "2019-03-19     1701.100000  0.549080  0.042500  0.756800  0.200600   \n",
       "2019-03-20     1040.625000  0.308138  0.024250  0.849938  0.125750   \n",
       "2019-03-21     4087.935065  0.290555  0.036558  0.832312  0.131117   \n",
       "2019-03-22     3233.733333  0.276140  0.020133  0.875667  0.104200   \n",
       "2019-03-23     2556.142857  0.240871  0.026857  0.832000  0.141143   \n",
       "2019-03-24     1795.666667  0.301833  0.024000  0.857778  0.118333   \n",
       "2019-03-25     9106.875000  0.501550  0.026937  0.769750  0.203312   \n",
       "2019-03-26      802.333333  0.319411  0.042222  0.836944  0.120833   \n",
       "2019-03-27     1880.969697  0.168927  0.035091  0.861788  0.103121   \n",
       "2019-03-28    10861.230769  0.400405  0.036846  0.764385  0.198718   \n",
       "\n",
       "            Compound_multiplied  \n",
       "2019-03-07          1406.704325  \n",
       "2019-03-08         -2245.066659  \n",
       "2019-03-09           -34.028050  \n",
       "2019-03-10            34.763340  \n",
       "2019-03-11          1394.352987  \n",
       "2019-03-12           648.260250  \n",
       "2019-03-13           744.901227  \n",
       "2019-03-14          1281.700721  \n",
       "2019-03-15           520.731389  \n",
       "2019-03-16           502.585871  \n",
       "2019-03-17            32.746600  \n",
       "2019-03-18          1111.553143  \n",
       "2019-03-19          1048.109500  \n",
       "2019-03-20           343.374687  \n",
       "2019-03-21          1253.919414  \n",
       "2019-03-22          1158.498953  \n",
       "2019-03-23           975.690900  \n",
       "2019-03-24           664.875244  \n",
       "2019-03-25          1562.571862  \n",
       "2019-03-26            84.944561  \n",
       "2019-03-27           659.380088  \n",
       "2019-03-28          3291.307072  "
      ]
     },
     "execution_count": 416,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full.tail(22)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Pct_change</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>49.835334</td>\n",
       "      <td>47.827333</td>\n",
       "      <td>48.874667</td>\n",
       "      <td>48.266263</td>\n",
       "      <td>-0.127772</td>\n",
       "      <td>4943.724921</td>\n",
       "      <td>0.298596</td>\n",
       "      <td>0.037299</td>\n",
       "      <td>0.830733</td>\n",
       "      <td>0.131963</td>\n",
       "      <td>1111.553143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.785783</td>\n",
       "      <td>1.738155</td>\n",
       "      <td>1.855279</td>\n",
       "      <td>1.800938</td>\n",
       "      <td>3.106129</td>\n",
       "      <td>6015.218435</td>\n",
       "      <td>0.176918</td>\n",
       "      <td>0.020698</td>\n",
       "      <td>0.038520</td>\n",
       "      <td>0.042410</td>\n",
       "      <td>1824.947502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>46.279999</td>\n",
       "      <td>44.759998</td>\n",
       "      <td>44.919998</td>\n",
       "      <td>44.880001</td>\n",
       "      <td>-5.580489</td>\n",
       "      <td>527.625000</td>\n",
       "      <td>-0.288960</td>\n",
       "      <td>0.010545</td>\n",
       "      <td>0.756800</td>\n",
       "      <td>0.021300</td>\n",
       "      <td>-2245.066659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>48.745834</td>\n",
       "      <td>46.906666</td>\n",
       "      <td>47.832500</td>\n",
       "      <td>47.354927</td>\n",
       "      <td>-2.252222</td>\n",
       "      <td>1724.741667</td>\n",
       "      <td>0.236893</td>\n",
       "      <td>0.024063</td>\n",
       "      <td>0.798906</td>\n",
       "      <td>0.104764</td>\n",
       "      <td>402.786873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>49.328334</td>\n",
       "      <td>47.711667</td>\n",
       "      <td>48.219999</td>\n",
       "      <td>47.693652</td>\n",
       "      <td>-0.048864</td>\n",
       "      <td>2588.821429</td>\n",
       "      <td>0.300214</td>\n",
       "      <td>0.036226</td>\n",
       "      <td>0.833864</td>\n",
       "      <td>0.131033</td>\n",
       "      <td>930.879538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>51.173333</td>\n",
       "      <td>48.302499</td>\n",
       "      <td>50.332500</td>\n",
       "      <td>48.986748</td>\n",
       "      <td>1.204081</td>\n",
       "      <td>5117.044867</td>\n",
       "      <td>0.395710</td>\n",
       "      <td>0.044637</td>\n",
       "      <td>0.853257</td>\n",
       "      <td>0.157170</td>\n",
       "      <td>1274.755395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>53.400002</td>\n",
       "      <td>51.580002</td>\n",
       "      <td>52.270000</td>\n",
       "      <td>52.030338</td>\n",
       "      <td>8.167955</td>\n",
       "      <td>30960.096774</td>\n",
       "      <td>0.595800</td>\n",
       "      <td>0.100806</td>\n",
       "      <td>0.915200</td>\n",
       "      <td>0.203312</td>\n",
       "      <td>7489.434700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            High        Low       Open  Adj Close  Pct_change  follower count  \\\n",
       "count  30.000000  30.000000  30.000000  30.000000   30.000000       30.000000   \n",
       "mean   49.835334  47.827333  48.874667  48.266263   -0.127772     4943.724921   \n",
       "std     1.785783   1.738155   1.855279   1.800938    3.106129     6015.218435   \n",
       "min    46.279999  44.759998  44.919998  44.880001   -5.580489      527.625000   \n",
       "25%    48.745834  46.906666  47.832500  47.354927   -2.252222     1724.741667   \n",
       "50%    49.328334  47.711667  48.219999  47.693652   -0.048864     2588.821429   \n",
       "75%    51.173333  48.302499  50.332500  48.986748    1.204081     5117.044867   \n",
       "max    53.400002  51.580002  52.270000  52.030338    8.167955    30960.096774   \n",
       "\n",
       "        compound        neg        neu        pos  Compound_multiplied  \n",
       "count  30.000000  30.000000  30.000000  30.000000            30.000000  \n",
       "mean    0.298596   0.037299   0.830733   0.131963          1111.553143  \n",
       "std     0.176918   0.020698   0.038520   0.042410          1824.947502  \n",
       "min    -0.288960   0.010545   0.756800   0.021300         -2245.066659  \n",
       "25%     0.236893   0.024063   0.798906   0.104764           402.786873  \n",
       "50%     0.300214   0.036226   0.833864   0.131033           930.879538  \n",
       "75%     0.395710   0.044637   0.853257   0.157170          1274.755395  \n",
       "max     0.595800   0.100806   0.915200   0.203312          7489.434700  "
      ]
     },
     "execution_count": 417,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.describe(df_full)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Create  'label' -column for the forecast; 'Predicted_change' for the next day - ADJUST FOR THE NR OF DAYS!\n",
    "With forecasting, you want to forecast out a certain number of datapoints. Let's we want to forecast out 1% of the entire length of the dataset. Thus, if our data is 100 days of stock prices and you are just trying to predict tomorrow's price, then you would just do 1 day out, and the forecast would be just one day out. If you predict 10 days out, you can actually generate a forecast for every day, for the next week and a half.\n",
    "\n",
    "In this case, with 30 days of data, one day would reprecent 3.33 % of the data\n",
    "\n",
    "I just add a new column - 'Predicted_change' - into the dataframe with a simple pandas operation:\n",
    "\n",
    "Previous day's 'Compound_multiplied_scaled' should predict next day's 'Pct_change'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 418,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#import math\n",
    "\n",
    "forecast_col = 'Pct_change'\n",
    "\n",
    "forecast_out = int(math.ceil(0.0333 * len(df_full)))\n",
    "\n",
    "df_full['Predicted_change'] = df_full[forecast_col].shift(-forecast_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Create another 'label' -column - 'Buy/Sell' - which is 1 if 'Predicted_change' is positive (=buy) and -1 if negative (=sell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "buy_or_sell = []\n",
    "\n",
    "for row in df_full['Pct_change']:\n",
    "    if row >= 0:\n",
    "        buy_or_sell.append(1)\n",
    "    elif row < 0:\n",
    "        buy_or_sell.append(-1) \n",
    "\n",
    "#Adds minus 1 or 1 to the column based on if 'Predicted_change' is negative or positive\n",
    "df_full['Buy/Sell'] = buy_or_sell\n",
    "\n",
    "# The 'Buy/Sell' values need to be shifte up on row to match the 'Predicted_change' values\n",
    "df_full['Buy/Sell'] = df_full['Buy/Sell'].shift(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Open</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Pct_change</th>\n",
       "      <th>follower count</th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>Compound_multiplied</th>\n",
       "      <th>Predicted_change</th>\n",
       "      <th>Buy/Sell</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-02-27</th>\n",
       "      <td>51.369999</td>\n",
       "      <td>48.500000</td>\n",
       "      <td>50.830002</td>\n",
       "      <td>48.005184</td>\n",
       "      <td>-4.505215</td>\n",
       "      <td>4943.724921</td>\n",
       "      <td>0.298596</td>\n",
       "      <td>0.037299</td>\n",
       "      <td>0.830733</td>\n",
       "      <td>0.131963</td>\n",
       "      <td>1111.553143</td>\n",
       "      <td>4.465208</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-28</th>\n",
       "      <td>50.520000</td>\n",
       "      <td>46.660000</td>\n",
       "      <td>48.150002</td>\n",
       "      <td>49.745789</td>\n",
       "      <td>4.465208</td>\n",
       "      <td>916.600000</td>\n",
       "      <td>0.255800</td>\n",
       "      <td>0.023200</td>\n",
       "      <td>0.915200</td>\n",
       "      <td>0.061600</td>\n",
       "      <td>369.520540</td>\n",
       "      <td>1.254409</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-01</th>\n",
       "      <td>51.950001</td>\n",
       "      <td>50.590000</td>\n",
       "      <td>51.020000</td>\n",
       "      <td>51.090805</td>\n",
       "      <td>1.254409</td>\n",
       "      <td>6370.678571</td>\n",
       "      <td>0.378829</td>\n",
       "      <td>0.035893</td>\n",
       "      <td>0.805286</td>\n",
       "      <td>0.158893</td>\n",
       "      <td>2136.179221</td>\n",
       "      <td>1.053096</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-02</th>\n",
       "      <td>52.433334</td>\n",
       "      <td>50.920001</td>\n",
       "      <td>51.436667</td>\n",
       "      <td>51.403983</td>\n",
       "      <td>1.053096</td>\n",
       "      <td>1557.916667</td>\n",
       "      <td>0.535575</td>\n",
       "      <td>0.021583</td>\n",
       "      <td>0.785583</td>\n",
       "      <td>0.192750</td>\n",
       "      <td>886.068175</td>\n",
       "      <td>0.851782</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-03</th>\n",
       "      <td>52.916668</td>\n",
       "      <td>51.250001</td>\n",
       "      <td>51.853334</td>\n",
       "      <td>51.717161</td>\n",
       "      <td>0.851782</td>\n",
       "      <td>5174.818182</td>\n",
       "      <td>0.296073</td>\n",
       "      <td>0.010545</td>\n",
       "      <td>0.883000</td>\n",
       "      <td>0.106455</td>\n",
       "      <td>1072.697936</td>\n",
       "      <td>0.650469</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-04</th>\n",
       "      <td>53.400002</td>\n",
       "      <td>51.580002</td>\n",
       "      <td>52.270000</td>\n",
       "      <td>52.030338</td>\n",
       "      <td>0.650469</td>\n",
       "      <td>10528.862069</td>\n",
       "      <td>0.396938</td>\n",
       "      <td>0.011172</td>\n",
       "      <td>0.851897</td>\n",
       "      <td>0.136931</td>\n",
       "      <td>7489.434700</td>\n",
       "      <td>-2.423074</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-05</th>\n",
       "      <td>52.240002</td>\n",
       "      <td>50.480000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>50.180943</td>\n",
       "      <td>-2.423074</td>\n",
       "      <td>9800.666667</td>\n",
       "      <td>0.139700</td>\n",
       "      <td>0.055250</td>\n",
       "      <td>0.835417</td>\n",
       "      <td>0.109333</td>\n",
       "      <td>-1935.955642</td>\n",
       "      <td>-4.447978</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-06</th>\n",
       "      <td>50.360001</td>\n",
       "      <td>48.099998</td>\n",
       "      <td>50.360001</td>\n",
       "      <td>47.589809</td>\n",
       "      <td>-4.447978</td>\n",
       "      <td>30960.096774</td>\n",
       "      <td>0.048803</td>\n",
       "      <td>0.100806</td>\n",
       "      <td>0.796677</td>\n",
       "      <td>0.102484</td>\n",
       "      <td>5775.218777</td>\n",
       "      <td>-1.817800</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-07</th>\n",
       "      <td>47.860001</td>\n",
       "      <td>45.410000</td>\n",
       "      <td>47.860001</td>\n",
       "      <td>46.472260</td>\n",
       "      <td>-1.817800</td>\n",
       "      <td>2621.500000</td>\n",
       "      <td>0.307390</td>\n",
       "      <td>0.045350</td>\n",
       "      <td>0.823650</td>\n",
       "      <td>0.130950</td>\n",
       "      <td>1406.704325</td>\n",
       "      <td>3.127712</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-08</th>\n",
       "      <td>47.560001</td>\n",
       "      <td>45.730000</td>\n",
       "      <td>46.040001</td>\n",
       "      <td>46.956860</td>\n",
       "      <td>3.127712</td>\n",
       "      <td>13547.103448</td>\n",
       "      <td>0.151948</td>\n",
       "      <td>0.048069</td>\n",
       "      <td>0.853172</td>\n",
       "      <td>0.098759</td>\n",
       "      <td>-2245.066659</td>\n",
       "      <td>3.418475</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-09</th>\n",
       "      <td>48.373334</td>\n",
       "      <td>46.373333</td>\n",
       "      <td>46.610001</td>\n",
       "      <td>47.675521</td>\n",
       "      <td>3.418475</td>\n",
       "      <td>702.916667</td>\n",
       "      <td>0.360208</td>\n",
       "      <td>0.036833</td>\n",
       "      <td>0.799750</td>\n",
       "      <td>0.163417</td>\n",
       "      <td>-34.028050</td>\n",
       "      <td>3.709237</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-10</th>\n",
       "      <td>49.186667</td>\n",
       "      <td>47.016666</td>\n",
       "      <td>47.180000</td>\n",
       "      <td>48.394182</td>\n",
       "      <td>3.709237</td>\n",
       "      <td>2524.800000</td>\n",
       "      <td>-0.288960</td>\n",
       "      <td>0.091500</td>\n",
       "      <td>0.887200</td>\n",
       "      <td>0.021300</td>\n",
       "      <td>34.763340</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-11</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>47.660000</td>\n",
       "      <td>47.750000</td>\n",
       "      <td>49.112843</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3704.083333</td>\n",
       "      <td>0.399446</td>\n",
       "      <td>0.024542</td>\n",
       "      <td>0.785167</td>\n",
       "      <td>0.190292</td>\n",
       "      <td>1394.352987</td>\n",
       "      <td>-1.086732</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-12</th>\n",
       "      <td>50.560001</td>\n",
       "      <td>48.750000</td>\n",
       "      <td>49.689999</td>\n",
       "      <td>48.608463</td>\n",
       "      <td>-1.086732</td>\n",
       "      <td>4934.961538</td>\n",
       "      <td>0.183062</td>\n",
       "      <td>0.045692</td>\n",
       "      <td>0.852692</td>\n",
       "      <td>0.101615</td>\n",
       "      <td>648.260250</td>\n",
       "      <td>-0.305246</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-13</th>\n",
       "      <td>49.470001</td>\n",
       "      <td>48.070000</td>\n",
       "      <td>49.139999</td>\n",
       "      <td>48.450226</td>\n",
       "      <td>-0.305246</td>\n",
       "      <td>2240.200000</td>\n",
       "      <td>0.235567</td>\n",
       "      <td>0.051667</td>\n",
       "      <td>0.796400</td>\n",
       "      <td>0.152000</td>\n",
       "      <td>744.901227</td>\n",
       "      <td>-2.098759</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-14</th>\n",
       "      <td>48.720001</td>\n",
       "      <td>47.150002</td>\n",
       "      <td>48.599998</td>\n",
       "      <td>47.055759</td>\n",
       "      <td>-2.098759</td>\n",
       "      <td>2547.928571</td>\n",
       "      <td>0.392025</td>\n",
       "      <td>0.012607</td>\n",
       "      <td>0.835821</td>\n",
       "      <td>0.151571</td>\n",
       "      <td>1281.700721</td>\n",
       "      <td>0.944380</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-15</th>\n",
       "      <td>49.130001</td>\n",
       "      <td>47.639999</td>\n",
       "      <td>47.650002</td>\n",
       "      <td>47.570026</td>\n",
       "      <td>0.944380</td>\n",
       "      <td>1986.500000</td>\n",
       "      <td>0.067900</td>\n",
       "      <td>0.064111</td>\n",
       "      <td>0.864333</td>\n",
       "      <td>0.071556</td>\n",
       "      <td>520.731389</td>\n",
       "      <td>0.698759</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-16</th>\n",
       "      <td>48.976667</td>\n",
       "      <td>47.703334</td>\n",
       "      <td>47.830001</td>\n",
       "      <td>47.632664</td>\n",
       "      <td>0.698759</td>\n",
       "      <td>710.428571</td>\n",
       "      <td>0.595800</td>\n",
       "      <td>0.017286</td>\n",
       "      <td>0.853286</td>\n",
       "      <td>0.129429</td>\n",
       "      <td>502.585871</td>\n",
       "      <td>0.453138</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-17</th>\n",
       "      <td>48.823332</td>\n",
       "      <td>47.766668</td>\n",
       "      <td>48.010000</td>\n",
       "      <td>47.695301</td>\n",
       "      <td>0.453138</td>\n",
       "      <td>527.625000</td>\n",
       "      <td>0.547662</td>\n",
       "      <td>0.032875</td>\n",
       "      <td>0.798625</td>\n",
       "      <td>0.168500</td>\n",
       "      <td>32.746600</td>\n",
       "      <td>0.207517</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-18</th>\n",
       "      <td>48.669998</td>\n",
       "      <td>47.830002</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>47.757938</td>\n",
       "      <td>0.207517</td>\n",
       "      <td>4943.724921</td>\n",
       "      <td>0.298596</td>\n",
       "      <td>0.037299</td>\n",
       "      <td>0.830733</td>\n",
       "      <td>0.131963</td>\n",
       "      <td>1111.553143</td>\n",
       "      <td>-1.724138</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-19</th>\n",
       "      <td>48.840000</td>\n",
       "      <td>47.720001</td>\n",
       "      <td>48.720001</td>\n",
       "      <td>47.352455</td>\n",
       "      <td>-1.724138</td>\n",
       "      <td>1701.100000</td>\n",
       "      <td>0.549080</td>\n",
       "      <td>0.042500</td>\n",
       "      <td>0.756800</td>\n",
       "      <td>0.200600</td>\n",
       "      <td>1048.109500</td>\n",
       "      <td>-1.400506</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-20</th>\n",
       "      <td>48.169998</td>\n",
       "      <td>46.560001</td>\n",
       "      <td>47.840000</td>\n",
       "      <td>46.650276</td>\n",
       "      <td>-1.400506</td>\n",
       "      <td>1040.625000</td>\n",
       "      <td>0.308138</td>\n",
       "      <td>0.024250</td>\n",
       "      <td>0.849938</td>\n",
       "      <td>0.125750</td>\n",
       "      <td>343.374687</td>\n",
       "      <td>8.167955</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-21</th>\n",
       "      <td>52.270000</td>\n",
       "      <td>47.869999</td>\n",
       "      <td>47.869999</td>\n",
       "      <td>51.209480</td>\n",
       "      <td>8.167955</td>\n",
       "      <td>4087.935065</td>\n",
       "      <td>0.290555</td>\n",
       "      <td>0.036558</td>\n",
       "      <td>0.832312</td>\n",
       "      <td>0.131117</td>\n",
       "      <td>1253.919414</td>\n",
       "      <td>-5.580489</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-22</th>\n",
       "      <td>51.450001</td>\n",
       "      <td>48.369999</td>\n",
       "      <td>51.250000</td>\n",
       "      <td>47.856834</td>\n",
       "      <td>-5.580489</td>\n",
       "      <td>3233.733333</td>\n",
       "      <td>0.276140</td>\n",
       "      <td>0.020133</td>\n",
       "      <td>0.875667</td>\n",
       "      <td>0.104200</td>\n",
       "      <td>1158.498953</td>\n",
       "      <td>-3.969031</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-23</th>\n",
       "      <td>50.583333</td>\n",
       "      <td>47.943333</td>\n",
       "      <td>50.250000</td>\n",
       "      <td>47.692004</td>\n",
       "      <td>-3.969031</td>\n",
       "      <td>2556.142857</td>\n",
       "      <td>0.240871</td>\n",
       "      <td>0.026857</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>0.141143</td>\n",
       "      <td>975.690900</td>\n",
       "      <td>-2.357573</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-24</th>\n",
       "      <td>49.716666</td>\n",
       "      <td>47.516666</td>\n",
       "      <td>49.250000</td>\n",
       "      <td>47.527173</td>\n",
       "      <td>-2.357573</td>\n",
       "      <td>1795.666667</td>\n",
       "      <td>0.301833</td>\n",
       "      <td>0.024000</td>\n",
       "      <td>0.857778</td>\n",
       "      <td>0.118333</td>\n",
       "      <td>664.875244</td>\n",
       "      <td>-0.746115</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-25</th>\n",
       "      <td>48.849998</td>\n",
       "      <td>47.090000</td>\n",
       "      <td>48.250000</td>\n",
       "      <td>47.362343</td>\n",
       "      <td>-0.746115</td>\n",
       "      <td>9106.875000</td>\n",
       "      <td>0.501550</td>\n",
       "      <td>0.026937</td>\n",
       "      <td>0.769750</td>\n",
       "      <td>0.203312</td>\n",
       "      <td>1562.571862</td>\n",
       "      <td>-2.303376</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-26</th>\n",
       "      <td>48.930000</td>\n",
       "      <td>46.869999</td>\n",
       "      <td>48.189999</td>\n",
       "      <td>46.561272</td>\n",
       "      <td>-2.303376</td>\n",
       "      <td>802.333333</td>\n",
       "      <td>0.319411</td>\n",
       "      <td>0.042222</td>\n",
       "      <td>0.836944</td>\n",
       "      <td>0.120833</td>\n",
       "      <td>84.944561</td>\n",
       "      <td>-3.916999</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-27</th>\n",
       "      <td>47.450001</td>\n",
       "      <td>44.939999</td>\n",
       "      <td>47.230000</td>\n",
       "      <td>44.880001</td>\n",
       "      <td>-3.916999</td>\n",
       "      <td>1880.969697</td>\n",
       "      <td>0.168927</td>\n",
       "      <td>0.035091</td>\n",
       "      <td>0.861788</td>\n",
       "      <td>0.103121</td>\n",
       "      <td>659.380088</td>\n",
       "      <td>1.847733</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-28</th>\n",
       "      <td>46.279999</td>\n",
       "      <td>44.759998</td>\n",
       "      <td>44.919998</td>\n",
       "      <td>45.750000</td>\n",
       "      <td>1.847733</td>\n",
       "      <td>10861.230769</td>\n",
       "      <td>0.400405</td>\n",
       "      <td>0.036846</td>\n",
       "      <td>0.764385</td>\n",
       "      <td>0.198718</td>\n",
       "      <td>3291.307072</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 High        Low       Open  Adj Close  Pct_change  \\\n",
       "2019-02-27  51.369999  48.500000  50.830002  48.005184   -4.505215   \n",
       "2019-02-28  50.520000  46.660000  48.150002  49.745789    4.465208   \n",
       "2019-03-01  51.950001  50.590000  51.020000  51.090805    1.254409   \n",
       "2019-03-02  52.433334  50.920001  51.436667  51.403983    1.053096   \n",
       "2019-03-03  52.916668  51.250001  51.853334  51.717161    0.851782   \n",
       "2019-03-04  53.400002  51.580002  52.270000  52.030338    0.650469   \n",
       "2019-03-05  52.240002  50.480000  52.000000  50.180943   -2.423074   \n",
       "2019-03-06  50.360001  48.099998  50.360001  47.589809   -4.447978   \n",
       "2019-03-07  47.860001  45.410000  47.860001  46.472260   -1.817800   \n",
       "2019-03-08  47.560001  45.730000  46.040001  46.956860    3.127712   \n",
       "2019-03-09  48.373334  46.373333  46.610001  47.675521    3.418475   \n",
       "2019-03-10  49.186667  47.016666  47.180000  48.394182    3.709237   \n",
       "2019-03-11  50.000000  47.660000  47.750000  49.112843    4.000000   \n",
       "2019-03-12  50.560001  48.750000  49.689999  48.608463   -1.086732   \n",
       "2019-03-13  49.470001  48.070000  49.139999  48.450226   -0.305246   \n",
       "2019-03-14  48.720001  47.150002  48.599998  47.055759   -2.098759   \n",
       "2019-03-15  49.130001  47.639999  47.650002  47.570026    0.944380   \n",
       "2019-03-16  48.976667  47.703334  47.830001  47.632664    0.698759   \n",
       "2019-03-17  48.823332  47.766668  48.010000  47.695301    0.453138   \n",
       "2019-03-18  48.669998  47.830002  48.189999  47.757938    0.207517   \n",
       "2019-03-19  48.840000  47.720001  48.720001  47.352455   -1.724138   \n",
       "2019-03-20  48.169998  46.560001  47.840000  46.650276   -1.400506   \n",
       "2019-03-21  52.270000  47.869999  47.869999  51.209480    8.167955   \n",
       "2019-03-22  51.450001  48.369999  51.250000  47.856834   -5.580489   \n",
       "2019-03-23  50.583333  47.943333  50.250000  47.692004   -3.969031   \n",
       "2019-03-24  49.716666  47.516666  49.250000  47.527173   -2.357573   \n",
       "2019-03-25  48.849998  47.090000  48.250000  47.362343   -0.746115   \n",
       "2019-03-26  48.930000  46.869999  48.189999  46.561272   -2.303376   \n",
       "2019-03-27  47.450001  44.939999  47.230000  44.880001   -3.916999   \n",
       "2019-03-28  46.279999  44.759998  44.919998  45.750000    1.847733   \n",
       "\n",
       "            follower count  compound       neg       neu       pos  \\\n",
       "2019-02-27     4943.724921  0.298596  0.037299  0.830733  0.131963   \n",
       "2019-02-28      916.600000  0.255800  0.023200  0.915200  0.061600   \n",
       "2019-03-01     6370.678571  0.378829  0.035893  0.805286  0.158893   \n",
       "2019-03-02     1557.916667  0.535575  0.021583  0.785583  0.192750   \n",
       "2019-03-03     5174.818182  0.296073  0.010545  0.883000  0.106455   \n",
       "2019-03-04    10528.862069  0.396938  0.011172  0.851897  0.136931   \n",
       "2019-03-05     9800.666667  0.139700  0.055250  0.835417  0.109333   \n",
       "2019-03-06    30960.096774  0.048803  0.100806  0.796677  0.102484   \n",
       "2019-03-07     2621.500000  0.307390  0.045350  0.823650  0.130950   \n",
       "2019-03-08    13547.103448  0.151948  0.048069  0.853172  0.098759   \n",
       "2019-03-09      702.916667  0.360208  0.036833  0.799750  0.163417   \n",
       "2019-03-10     2524.800000 -0.288960  0.091500  0.887200  0.021300   \n",
       "2019-03-11     3704.083333  0.399446  0.024542  0.785167  0.190292   \n",
       "2019-03-12     4934.961538  0.183062  0.045692  0.852692  0.101615   \n",
       "2019-03-13     2240.200000  0.235567  0.051667  0.796400  0.152000   \n",
       "2019-03-14     2547.928571  0.392025  0.012607  0.835821  0.151571   \n",
       "2019-03-15     1986.500000  0.067900  0.064111  0.864333  0.071556   \n",
       "2019-03-16      710.428571  0.595800  0.017286  0.853286  0.129429   \n",
       "2019-03-17      527.625000  0.547662  0.032875  0.798625  0.168500   \n",
       "2019-03-18     4943.724921  0.298596  0.037299  0.830733  0.131963   \n",
       "2019-03-19     1701.100000  0.549080  0.042500  0.756800  0.200600   \n",
       "2019-03-20     1040.625000  0.308138  0.024250  0.849938  0.125750   \n",
       "2019-03-21     4087.935065  0.290555  0.036558  0.832312  0.131117   \n",
       "2019-03-22     3233.733333  0.276140  0.020133  0.875667  0.104200   \n",
       "2019-03-23     2556.142857  0.240871  0.026857  0.832000  0.141143   \n",
       "2019-03-24     1795.666667  0.301833  0.024000  0.857778  0.118333   \n",
       "2019-03-25     9106.875000  0.501550  0.026937  0.769750  0.203312   \n",
       "2019-03-26      802.333333  0.319411  0.042222  0.836944  0.120833   \n",
       "2019-03-27     1880.969697  0.168927  0.035091  0.861788  0.103121   \n",
       "2019-03-28    10861.230769  0.400405  0.036846  0.764385  0.198718   \n",
       "\n",
       "            Compound_multiplied  Predicted_change  Buy/Sell  \n",
       "2019-02-27          1111.553143          4.465208       1.0  \n",
       "2019-02-28           369.520540          1.254409       1.0  \n",
       "2019-03-01          2136.179221          1.053096       1.0  \n",
       "2019-03-02           886.068175          0.851782       1.0  \n",
       "2019-03-03          1072.697936          0.650469       1.0  \n",
       "2019-03-04          7489.434700         -2.423074      -1.0  \n",
       "2019-03-05         -1935.955642         -4.447978      -1.0  \n",
       "2019-03-06          5775.218777         -1.817800      -1.0  \n",
       "2019-03-07          1406.704325          3.127712       1.0  \n",
       "2019-03-08         -2245.066659          3.418475       1.0  \n",
       "2019-03-09           -34.028050          3.709237       1.0  \n",
       "2019-03-10            34.763340          4.000000       1.0  \n",
       "2019-03-11          1394.352987         -1.086732      -1.0  \n",
       "2019-03-12           648.260250         -0.305246      -1.0  \n",
       "2019-03-13           744.901227         -2.098759      -1.0  \n",
       "2019-03-14          1281.700721          0.944380       1.0  \n",
       "2019-03-15           520.731389          0.698759       1.0  \n",
       "2019-03-16           502.585871          0.453138       1.0  \n",
       "2019-03-17            32.746600          0.207517       1.0  \n",
       "2019-03-18          1111.553143         -1.724138      -1.0  \n",
       "2019-03-19          1048.109500         -1.400506      -1.0  \n",
       "2019-03-20           343.374687          8.167955       1.0  \n",
       "2019-03-21          1253.919414         -5.580489      -1.0  \n",
       "2019-03-22          1158.498953         -3.969031      -1.0  \n",
       "2019-03-23           975.690900         -2.357573      -1.0  \n",
       "2019-03-24           664.875244         -0.746115      -1.0  \n",
       "2019-03-25          1562.571862         -2.303376      -1.0  \n",
       "2019-03-26            84.944561         -3.916999      -1.0  \n",
       "2019-03-27           659.380088          1.847733       1.0  \n",
       "2019-03-28          3291.307072               NaN       NaN  "
      ]
     },
     "execution_count": 421,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full.head(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Save 'df_full' as excel file with the $cashtag as file name\n",
    "The files created with this notebook are used when testing how well the binary classifiers trained on data from 2016 predict stock price movements today, i.e. during March 2019!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Change the name of the excel-file that will be created below!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# import xlsxwriter\n",
    "# import os\n",
    "\n",
    "writer_df = pd.ExcelWriter('df_full.xlsx', engine='xlsxwriter')\n",
    "df_full.to_excel(writer_df)\n",
    "writer_df.save()\n",
    "\n",
    "os.rename('df_full.xlsx', '$WDC_prediction.xlsx') # UPDATE THE $CASHTAG BEFORE RUNNING THE CELL!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}