Quantcon-RL-Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "from datetime import datetime, timedelta\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense\n",
    "from keras.optimizers import sgd\n",
    "import talib\n",
    "import traceback\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse a data file into Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dateparse = lambda x,y: pd.datetime.strptime(x+' '+y, '%m/%d/%Y %H:%M')\n",
    "df = pd.read_csv('data.txt',parse_dates=[[0,1]],index_col=0,skiprows=0,date_parser=dateparse)\n",
    "df.index.rename('Time',inplace=True)\n",
    "df.columns = ['open','high','low','close','volume']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['close'].plot()\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a synthetic test series\n",
    "\n",
    "Here we create a sine wave and a trend line. \n",
    "The parameter mult determines the amount of noise in our system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_synth_series(x,y,mult=1):    \n",
    "    op = pd.Series(y+mult*np.random.randn(len(df)),index=df.index)\n",
    "    cl = pd.Series(y+mult*np.random.randn(len(df)),index=df.index)\n",
    "    hi = pd.Series(y+1+mult*np.abs(np.random.randn(len(df))),index=df.index)\n",
    "    lo = pd.Series(y-1-mult*np.abs(np.random.randn(len(df))),index=df.index)\n",
    "    dfs = pd.DataFrame([op,hi,lo,cl]).T\n",
    "    dfs.index.rename('Time',inplace=True)\n",
    "    dfs.columns = ['open','high','low','close']\n",
    "    return dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df.iloc[:80000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldf = len(df)\n",
    "mid= int(ldf/2)\n",
    "x = np.arange(len(df))\n",
    "y = np.zeros(ldf)\n",
    "y[mid:] = 0.01*x[:mid]+100\n",
    "y[:mid] = 10*np.sin(0.01*x[:mid])+100#+y[mid-1]\n",
    "dft = create_synth_series(x,y,mult=0.00)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dft['close'].plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This is the Reinforcement Learning part of the system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ExperienceReplay(object):\n",
    "    '''This class gathers and delivers the experience'''\n",
    "    def __init__(self, max_memory=100, discount=.9):\n",
    "        self.max_memory = max_memory\n",
    "        self.memory = list()\n",
    "        self.discount = discount\n",
    "\n",
    "    def remember(self, states, game_over):\n",
    "        # memory[i] = [[state_t, action_t, reward_t, state_t+1], game_over?]\n",
    "        self.memory.append([states, game_over])\n",
    "        if len(self.memory) > self.max_memory:\n",
    "            del self.memory[0]\n",
    "\n",
    "    def get_batch(self, model, batch_size=10):\n",
    "        len_memory = len(self.memory)\n",
    "        num_actions = model.output_shape[-1]\n",
    "        env_dim = self.memory[0][0][0].shape[1]\n",
    "        inputs = np.zeros((min(len_memory, batch_size), env_dim))\n",
    "        targets = np.zeros((inputs.shape[0], num_actions))\n",
    "        for i, idx in enumerate(np.random.randint(0, len_memory, size=inputs.shape[0])):\n",
    "            state_t, action_t, reward_t, state_tp1 = self.memory[idx][0]\n",
    "            game_over = self.memory[idx][1]\n",
    "\n",
    "            inputs[i:i+1] = state_t\n",
    "            # There should be no target values for actions not taken.\n",
    "            # Thou shalt not correct actions not taken #deep\n",
    "            targets[i] = model.predict(state_t)[0]\n",
    "            Q_sa = np.max(model.predict(state_tp1)[0])\n",
    "            if game_over:  # if game_over is True\n",
    "                targets[i, action_t] = reward_t\n",
    "            else:\n",
    "                # reward_t + gamma * max_a' Q(s', a')\n",
    "                targets[i, action_t] = reward_t + self.discount * Q_sa\n",
    "        return inputs, targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the \"Trading Game\"\n",
    "\n",
    "Here we define our state, reward and policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidePrompt": true
   },
   "outputs": [],
   "source": [
    "class Game(object):\n",
    "    '''This is the game. It starts, then takes an action (buy or sell) at some point and finally the reverse\n",
    "    action, at which point it is game over. This is where the reward is given. The state consists of a vector\n",
    "    with different bar sizes for OLHC. They are just concatenated. \n",
    "    lkbk: determines how many bars to use - larger lkbk - bigger state\n",
    "    '''\n",
    "    def __init__(self, df, lkbk=20, max_game_len=1000, run_mode='sequential', init_idx=None):\n",
    "        self.df = df\n",
    "        self.lkbk = lkbk\n",
    "        self.max_game_len = max_game_len\n",
    "        \n",
    "        self.is_over = False\n",
    "        self.reward = 0\n",
    "        self.run_mode =  run_mode\n",
    "        self.pnl_sum = 0\n",
    "        if run_mode == 'sequential' and init_idx == None:\n",
    "            print('------No init_idx set for \"sequential\": stopping------')\n",
    "            return\n",
    "        else:\n",
    "            self.init_idx = init_idx\n",
    "        self.reset()\n",
    "        \n",
    "    def _update_state(self, action):\n",
    "        \n",
    "        '''Here we update our state'''\n",
    "        self.curr_idx += 1\n",
    "        self.curr_time = self.df.index[self.curr_idx]\n",
    "        self.curr_price = self.df['close'][self.curr_idx]\n",
    "        self.pnl = (-self.entry + self.curr_price)*self.position/self.entry\n",
    "        self._assemble_state()\n",
    "        _k = list(map(float,str(self.curr_time.time()).split(':')[:2]))\n",
    "        self._time_of_day = (_k[0]*60 + _k[1])/(24*60) \n",
    "        self._day_of_week  = self.curr_time.weekday()/6\n",
    "        self.norm_epoch = (df.index[self.curr_idx]-df.index[0]).total_seconds()/self.t_in_secs\n",
    "        \n",
    "        '''This is where we define our policy and update our position'''\n",
    "        if action == 0:  \n",
    "            pass\n",
    "        \n",
    "        elif action == 2:\n",
    "            if self.position == -1:\n",
    "                self.is_over = True\n",
    "                self._get_reward()\n",
    "                self.trade_len = self.curr_idx - self.start_idx\n",
    "   \n",
    "            elif self.position == 0:\n",
    "                self.position = 1\n",
    "                self.entry = self.curr_price\n",
    "                self.start_idx = self.curr_idx\n",
    "            else: \n",
    "                pass\n",
    "            \n",
    "        elif action == 1:\n",
    "            if self.position == 1:\n",
    "                self.is_over = True\n",
    "                self._get_reward()\n",
    "                self.trade_len = self.curr_idx - self.start_idx\n",
    "\n",
    "            elif self.position == 0:\n",
    "                self.position = -1\n",
    "                self.entry = self.curr_price\n",
    "                self.start_idx = self.curr_idx\n",
    "            else:\n",
    "                pass\n",
    "        \n",
    "    \n",
    "    def _assemble_state(self):\n",
    "        '''Here we can add other things such as indicators and times'''\n",
    "        self._get_last_N_timebars()\n",
    "        bars = [self.last5m,self.last1h,self.last1d]\n",
    "        state = []\n",
    "        candles = {j:{k:np.array([]) for k in ['open','high','low','close']} for j in range(len(bars))}\n",
    "        for j,bar in enumerate(bars):\n",
    "            for col in ['open','high','low','close']:\n",
    "                candles[j][col] = np.asarray(bar[col])\n",
    "                state += (list(np.asarray(bar[col]))[-10:])\n",
    "\n",
    "        \n",
    "        self.state = np.array([])\n",
    "        self.state = np.append(self.state,state)\n",
    "        self.state = np.append(self.state,self.position)\n",
    "        np.append(self.state,np.sign(self.pnl_sum))\n",
    "        self.state = np.append(self.state,self._time_of_day)\n",
    "        self.state = np.append(self.state,self._day_of_week)\n",
    "        \n",
    "        for c in candles:\n",
    "            try:\n",
    "                sma1 = talib.SMA(candles[c]['close'],self.lkbk-1)[-1]\n",
    "                sma2 = talib.SMA(candles[c]['close'],self.lkbk-8)[-1]\n",
    "                self.state = np.append(self.state,(sma1-sma2)/sma2)\n",
    "                self.state = np.append(self.state,sma1)\n",
    "                self.state = np.append(self.state,talib.RSI(candles[c]['close'],self.lkbk-1)[-1])\n",
    "                self.state = np.append(self.state,talib.MOM(candles[c]['close'],self.lkbk-1)[-1])\n",
    "                #self.state = np.append(self.state,talib.MACD(candles[c]['close'],fastperiod=11, slowperiod=22, signalperiod=9)[0][0])\n",
    "                self.state = np.append(self.state,talib.BOP(candles[c]['open'],\n",
    "                                               candles[c]['high'],\n",
    "                                               candles[c]['low'],\n",
    "                                               candles[c]['close'])[-1])\n",
    "                #self.state = np.append(self.state,talib.ADXR(candles[c]['high'],\n",
    "                #                               candles[c]['low'],\n",
    "                #                               candles[c]['close'],\n",
    "                #                               self.lkbk-3)[-1]) \n",
    "                #self.state = np.append(self.state,talib.STOCH(candles[c]['high'],\n",
    "                #                               candles[c]['low'],\n",
    "                #                               candles[c]['close'],5,3,0,3,0)[-1][0])\n",
    "                self.state = np.append(self.state,talib.AROONOSC(candles[c]['high'],\n",
    "                                               candles[c]['low'],\n",
    "                                               self.lkbk-3)[-1])\n",
    "            except: print(traceback.format_exc())\n",
    "        #print('-->',self.state)\n",
    "        self.state = (np.array(self.state)-np.mean(self.state))/np.std(self.state)\n",
    "        \n",
    "\n",
    "\n",
    "        \n",
    "        \n",
    "    def _get_last_N_timebars(self):\n",
    "        '''The lengths of the time windows are currently hardcoded.'''\n",
    "        # TODO: find better way to calculate window lengths\n",
    "        wdw5m = 9\n",
    "        wdw1h = np.ceil(self.lkbk*15/24.)\n",
    "        wdw1d = np.ceil(self.lkbk*15)\n",
    "        \n",
    "        self.last5m = self.df[self.curr_time-timedelta(wdw5m):self.curr_time].iloc[-self.lkbk:]\n",
    "        self.last1h = self.bars1h[self.curr_time-timedelta(wdw1h):self.curr_time].iloc[-self.lkbk:]\n",
    "        self.last1d = self.bars1d[self.curr_time-timedelta(wdw1d):self.curr_time].iloc[-self.lkbk:]\n",
    "        \n",
    "        '''Making sure that window lengths are sufficient'''\n",
    "        try:\n",
    "            assert(len(self.last5m)==self.lkbk)\n",
    "            assert(len(self.last1h)==self.lkbk)\n",
    "            assert(len(self.last1d)==self.lkbk)\n",
    "        except:\n",
    "            print('****Window length too short****')\n",
    "            print(len(self.last5m),len(self.last1h),len(self.last1d))\n",
    "            if self.run_mode == 'sequential':\n",
    "                self.init_idx = self.curr_idx\n",
    "                self.reset()\n",
    "            else:\n",
    "                self.reset()\n",
    "\n",
    "\n",
    "    def _get_reward(self):\n",
    "        if self.position == 1 and self.is_over:\n",
    "            pnl = (self.curr_price - self.entry)/self.entry\n",
    "            self.reward = np.sign(pnl)#-(self.curr_idx - self.start_idx)/1000.\n",
    "        elif self.position == -1 and self.is_over:\n",
    "            pnl = (-self.curr_price + self.entry)/self.entry\n",
    "            self.reward = np.sign(pnl)#-(self.curr_idx - self.start_idx)/1000.\n",
    "        #print('entry:',self.entry,'exit:',self.curr_price,'pos:',self.position,'pnl:',pnl,self.reward)\n",
    "        return self.reward\n",
    "            \n",
    "    def observe(self):\n",
    "        return np.array([self.state])\n",
    "\n",
    "    def act(self, action):\n",
    "        self._update_state(action)\n",
    "        reward = self.reward\n",
    "        game_over = self.is_over\n",
    "        return self.observe(), reward, game_over\n",
    "\n",
    "    def reset(self):\n",
    "        self.pnl = 0\n",
    "        self.entry = 0\n",
    "        self._time_of_day = 0\n",
    "        self._day_of_week = 0\n",
    "        \n",
    "        if self.run_mode == 'random':\n",
    "            self.curr_idx = np.random.randint(0,len(df)-3000)\n",
    "            \n",
    "        elif self.run_mode == 'sequential':\n",
    "            self.curr_idx = self.init_idx\n",
    "            \n",
    "        self.t_in_secs = (df.index[-1]-df.index[0]).total_seconds()\n",
    "        self.start_idx = self.curr_idx\n",
    "        self.curr_time = self.df.index[self.curr_idx]\n",
    "        self.bars1h = df['close'].resample('1H',label='right',closed='right').ohlc().dropna()\n",
    "        self.bars1d = df['close'].resample('1D',label='right',closed='right').ohlc().dropna()\n",
    "        self._get_last_N_timebars()\n",
    "        self.state = []\n",
    "        self.position = 0\n",
    "        self._update_state(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running the \"Game\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def run(df,fname):\n",
    "    # parameters\n",
    "    epsilon_0 = .001\n",
    "    num_actions = 3 \n",
    "    epoch = 11500\n",
    "    max_memory = 10000\n",
    "    \n",
    "    batch_size = 500\n",
    "    lkbk = 25\n",
    "    START_IDX = 3000\n",
    "\n",
    "    env = Game(df, lkbk=lkbk, max_game_len=1000,init_idx=START_IDX,run_mode='sequential')\n",
    "    hidden_size = len(env.state)*2\n",
    "    model = Sequential()\n",
    "    model.add(Dense(hidden_size, input_shape=(len(env.state),), activation='relu'))\n",
    "    model.add(Dense(hidden_size, activation='relu'))\n",
    "    model.add(Dense(num_actions))\n",
    "    model.compile(sgd(lr=.005), \"mse\")\n",
    "\n",
    "    # If you want to continue training from a previous model, just uncomment the line bellow\n",
    "    model.load_weights(\"indicator_model.h5\")\n",
    "\n",
    "    # Initialize experience replay object\n",
    "    exp_replay = ExperienceReplay(max_memory=max_memory)\n",
    "\n",
    "    # Train\n",
    "    win_cnt = 0\n",
    "    loss_cnt = 0\n",
    "    wins = []\n",
    "    losses = []\n",
    "    pnls = []\n",
    "    for e in range(epoch):\n",
    "        epsilon = epsilon_0**(np.log10(e))\n",
    "        env = Game(df, lkbk=lkbk, max_game_len=1000,init_idx=env.curr_idx,run_mode='sequential')\n",
    "        loss = 0.\n",
    "        env.reset()\n",
    "        game_over = False\n",
    "        # get initial input\n",
    "        input_t = env.observe()\n",
    "\n",
    "        cnt = 0\n",
    "        while not game_over:\n",
    "            cnt += 1\n",
    "            input_tm1 = input_t\n",
    "            # get next action\n",
    "\n",
    "            if np.random.rand() <= epsilon:\n",
    "                action = np.random.randint(0, num_actions, size=1)[0]\n",
    "                if env.position == 0:\n",
    "                    if action == 2:\n",
    "                        exit_action = 1\n",
    "                    elif action == 1:\n",
    "                        exit_action = 2\n",
    "                #if env.position and action == exit_action:\n",
    "                #    print('***random exit***',env.position)\n",
    "                #elif not env.position and action:\n",
    "                #    print('***random entry***',env.position)\n",
    "                    \n",
    "            elif env.position == 0:\n",
    "                q = model.predict(input_tm1)\n",
    "                action = np.argmax(q[0])\n",
    "                if action:\n",
    "                    #print(cnt)\n",
    "                    exit_action = np.argmin(q[0][1:])+1\n",
    "                \n",
    "            elif cnt > 500:\n",
    "                #print('***Time Exit***')\n",
    "                action = exit_action\n",
    "                \n",
    "            elif env.position:\n",
    "                q = model.predict(input_tm1)\n",
    "                action = np.argmax(q[0])\n",
    "\n",
    "            # apply action, get rewards and new state\n",
    "            input_t, reward, game_over = env.act(action)\n",
    "            if reward > 0:\n",
    "                win_cnt += 1\n",
    "            elif reward < 0:\n",
    "                loss_cnt += 1\n",
    "\n",
    "            # store experience\n",
    "            if action or len(exp_replay.memory)<20 or np.random.rand() < 0.1:\n",
    "                exp_replay.remember([input_tm1, action, reward, input_t], game_over)\n",
    "\n",
    "            inputs, targets = exp_replay.get_batch(model, batch_size=batch_size)\n",
    "            env.pnl_sum = sum(pnls)\n",
    "\n",
    "            zz = model.train_on_batch(inputs, targets)\n",
    "            loss += zz\n",
    "        prt_str = (\"Epoch {:03d} | Loss {:.2f} | pos {} | len {} | pnl {:.2f}% @ {:.2f}% | eps {:,.4f} | {}\".format(e, \n",
    "                                                                                      loss, \n",
    "                                                                                      env.position, \n",
    "                                                                                      env.trade_len,\n",
    "                                                                                      sum(pnls)*100,\n",
    "                                                                                      env.pnl*100,\n",
    "                                                                                      epsilon,\n",
    "                                                                                      env.curr_time))\n",
    "        print(prt_str)\n",
    "        fid = open(fname,'a')\n",
    "        fid.write(prt_str+'\\n')\n",
    "        fid.close()\n",
    "        pnls.append(env.pnl)\n",
    "        if not e%10:\n",
    "            print('----saving weights-----')\n",
    "            model.save_weights(\"indicator_model.h5\", overwrite=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fname = 'output1.dat'\n",
    "fid = open(fname,'w')\n",
    "fid.close()\n",
    "run(df,fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}