Character_level_lm_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Important Note\n",
    "This exercise draws a lot of influence from the coding exercises in Course 5 on Sequence Models (DeepLearning Learning Specialization on coursera by DeepLearng.ai)<br>\n",
    "The specialization is taught by none other than Andrew Ng.<br>Course 5 is one of the best online resources available on the Sequence Models.<br>\n",
    "We (Rahee and Vishal) have ourselves used the course for our own learning and recommend it to everyone interested in learning more about Neural Networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "wCiq3swdYICh",
    "outputId": "5da4643b-1ae8-4519-a0d8-ed80906121af"
   },
   "outputs": [],
   "source": [
    "#use this only when running the notebook on google colab\n",
    "'''\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "filepath = '/content/drive/My Drive/rnn-workshop/bird-genera.txt'\n",
    "'''\n",
    "#use this when running locally:\n",
    "filepath = './bird-genera.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bVswruHmZZLA"
   },
   "outputs": [],
   "source": [
    "# Read input data and find basic info like vocab_size, data_size etc\n",
    "data = open(filepath, 'r').read()\n",
    "data= data.lower()\n",
    "chars = list(set(data))\n",
    "data_size, vocab_size = len(data), len(chars)\n",
    "print('There are %d total characters and %d unique characters in your data.' % (data_size, vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Utilities that making sampling easier\n",
    "char_to_index = {ch:i for i, ch in enumerate(sorted(chars))}\n",
    "index_to_char = {i:ch for i, ch in enumerate(sorted(chars))}\n",
    "print(\"Index to character mapping:\\n\",index_to_char)\n",
    "print(\"Character to Index mapping (Reverse of Index to Character mapping):\\n\",char_to_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## These are utility functions. Don't change anything here\n",
    "def softmax(x):\n",
    "    e_x = np.exp(x - np.max(x))\n",
    "    return e_x / e_x.sum(axis=0)\n",
    "\n",
    "def smooth(loss, cur_loss):\n",
    "     return loss * 0.999 + cur_loss * 0.001\n",
    "    \n",
    "def get_initial_loss(vocab_size, seq_length):\n",
    "    return -np.log(1.0/vocab_size)*seq_length\n",
    "\n",
    "def sample(parameters, char_to_ix, seed):\n",
    "    # Retrieve parameters and relevant shapes from \"parameters\" dictionary\n",
    "    Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']\n",
    "    vocab_size = by.shape[0]\n",
    "    n_a = Waa.shape[1]\n",
    "    \n",
    "    # Step 1: Create the one-hot vector x for the first character (initializing the sequence generation).\n",
    "    x = np.zeros((vocab_size,1))\n",
    "    # Step 1': Initialize a_prev as zeros \n",
    "    a_prev = np.zeros((n_a,1))\n",
    "    \n",
    "    # Create an empty list of indices, this is the list which will contain the list of indices of the characters to generate \n",
    "    indices = []\n",
    "    \n",
    "    # Idx is a flag to detect a newline character, we initialize it to -1\n",
    "    idx = -1 \n",
    "    \n",
    "    # Loop over time-steps t. At each time-step, sample a character from a probability distribution and append \n",
    "    # its index to \"indices\". We'll stop if we reach 50 characters (which should be very unlikely with a well \n",
    "    # trained model), which helps debugging and prevents entering an infinite loop. \n",
    "    counter = 0\n",
    "    newline_character = char_to_ix['\\n']\n",
    "    \n",
    "    while (idx != newline_character and counter != 50):\n",
    "        \n",
    "        # Step 2: Forward propagate x using the equations for an rnn unit\n",
    "        a = np.tanh(np.dot(Wax, x) + np.dot(Waa,a_prev) + b)\n",
    "        z = np.dot(Wya,a)+by\n",
    "        y = softmax(z)\n",
    "        \n",
    "        # Set Seed\n",
    "        np.random.seed(counter+seed) \n",
    "        \n",
    "        # Step 3: Sample the index of a character within the vocabulary from the probability distribution y\n",
    "        #p = np.array([0.1, 0.0, 0.7, 0.2])\n",
    "        idx = np.random.choice(list(range(vocab_size)), p = y.ravel()) \n",
    "\n",
    "        # Append the index to \"indices\"\n",
    "        indices.append(idx)\n",
    "        \n",
    "        # Step 4: Overwrite the input character as the one corresponding to the sampled index.\n",
    "        x = np.zeros((vocab_size,1))\n",
    "        x[idx] = 1\n",
    "        \n",
    "        # Update \"a_prev\" to be \"a\"\n",
    "        a_prev = a\n",
    "        \n",
    "        # update seed and counter\n",
    "        seed += 1\n",
    "        counter +=1\n",
    "\n",
    "    if (counter == 50):\n",
    "        indices.append(char_to_ix['\\n'])\n",
    "    \n",
    "    return indices\n",
    "\n",
    "def print_sample(sample_index, index_to_char):\n",
    "    txt = ''.join(index_to_char[index] for index in sample_index)\n",
    "    txt = txt[0].upper() + txt[1:]  # capitalize first character \n",
    "    print ('%s' % (txt, ), end='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#A quick check to see if the sampling is working as expected\n",
    "np.random.seed(2)\n",
    "n_a = 100\n",
    "Wax, Waa, Wya = np.random.randn(n_a, vocab_size), np.random.randn(n_a, n_a), np.random.randn(vocab_size, n_a)\n",
    "b, by = np.random.randn(n_a, 1), np.random.randn(vocab_size, 1)\n",
    "parameters = {\"Wax\": Wax, \"Waa\": Waa, \"Wya\": Wya, \"b\": b, \"by\": by}\n",
    "\n",
    "\n",
    "indices = sample(parameters, char_to_index, 0)\n",
    "print(\"Sampling:\")\n",
    "print(\"list of sampled indices:\\n\", indices,\"\\n\")\n",
    "#print(\"list of sampled characters:\\n\", [index_to_char[i] for i in indices])\n",
    "print_sample(indices, index_to_char)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Output\n",
    "Sampling:\n",
    "list of sampled indices:\n",
    "[12, 17, 24, 14, 13, 9, 10, 22, 24, 6, 13, 11, 12, 6, 21, 15, 21, 14, 3, 2, 1, 21, 18, 24, 7, 25, 6, 25, 18, 10, 16, 2, 3, 8, 15, 12, 11, 7, 1, 12, 10, 2, 7, 7, 11, 3, 6, 12, 7, 12, 0] \n",
    "\n",
    "Lqxnmijvxfmklfuouncbaurxgyfyrjpbcholkgaljbggkcflgl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 496
    },
    "colab_type": "code",
    "id": "UCrqGuxbT9gq",
    "outputId": "2009b2fb-f6f7-43bc-9d7b-59c2ee0b6cd1"
   },
   "outputs": [],
   "source": [
    "#Complete this function \n",
    "def one_element_forward_through_rnn(parameters, a_prev, x):\n",
    "    \n",
    "    Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']\n",
    "    a_next = # complete this line\n",
    "    p_t = softmax(np.dot(Wya, a_next) + by) # unnormalized log probabilities for next chars\n",
    "                                            # probabilities for next chars\n",
    "    return a_next, p_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_forward(X, Y, a0, parameters, vocab_size = 27):\n",
    "    \n",
    "    # Initialize x, a and y_hat as empty dictionaries\n",
    "    x, a, y_hat = {}, {}, {}\n",
    "    a[-1] = np.copy(a0)\n",
    "    # initialize your loss to 0\n",
    "    loss = 0\n",
    "    \n",
    "    for t in range(len(X)):\n",
    "        # Set x[t] to be the one-hot vector representation of the t'th character in X.\n",
    "        # if X[t] == None, we just have x[t]=0. This is used to set the input for the first timestep to the zero vector. \n",
    "        x[t] = np.zeros((vocab_size,1)) \n",
    "        if (X[t] != None):\n",
    "            x[t][X[t]] = 1\n",
    "        \n",
    "        # Run one step forward of the RNN\n",
    "        a[t], y_hat[t] = one_element_forward_through_rnn(parameters, a[t - 1], x[t])\n",
    "        \n",
    "        # Update the loss by substracting the cross-entropy term of this time-step from it.\n",
    "        loss -= np.log(y_hat[t][Y[t],0])\n",
    "        \n",
    "    cache = (y_hat, a, x)\n",
    "        \n",
    "    return loss, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_step_backward(dy, gradients, parameters, x, a, a_prev):\n",
    "    \n",
    "    gradients['dWya'] += np.dot(dy, a.T)\n",
    "    gradients['dby'] += dy\n",
    "    da = np.dot(parameters['Wya'].T, dy) + gradients['da_next'] # backprop into h\n",
    "    daraw = (1 - a * a) * da # backprop through tanh nonlinearity\n",
    "    gradients['db'] += daraw\n",
    "    gradients['dWax'] += np.dot(daraw, x.T)\n",
    "    gradients['dWaa'] += np.dot(daraw, a_prev.T)\n",
    "    gradients['da_next'] = np.dot(parameters['Waa'].T, daraw)\n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_backward(X, Y, parameters, cache):\n",
    "    # Initialize gradients as an empty dictionary\n",
    "    gradients = {}\n",
    "    \n",
    "    # Retrieve from cache and parameters\n",
    "    (y_hat, a, x) = cache\n",
    "    Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']\n",
    "    \n",
    "    # each one should be initialized to zeros of the same dimension as its corresponding parameter\n",
    "    gradients['dWax'], gradients['dWaa'], gradients['dWya'] = np.zeros_like(Wax), np.zeros_like(Waa), np.zeros_like(Wya)\n",
    "    gradients['db'], gradients['dby'] = np.zeros_like(b), np.zeros_like(by)\n",
    "    gradients['da_next'] = np.zeros_like(a[0])\n",
    "    \n",
    "    # Backpropagate through time\n",
    "    for t in reversed(range(len(X))):\n",
    "        dy = np.copy(y_hat[t])\n",
    "        dy[Y[t]] -= 1\n",
    "        gradients = rnn_step_backward(dy, gradients, parameters, x[t], a[t], a[t-1])\n",
    "    return gradients, a\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Complete this function\n",
    "def clip(gradients, maxValue):\n",
    "    \n",
    "    dWaa, dWax, dWya, db, dby = gradients['dWaa'], gradients['dWax'], gradients['dWya'], gradients['db'], gradients['dby']\n",
    "    # clip to mitigate exploding gradients, loop over [dWax, dWaa, dWya, db, dby]\n",
    "    for gradient in [dWax, dWaa, dWya, db, dby]:\n",
    "        gradient = # complete this line  . use np.clip()\n",
    "    \n",
    "    gradients = {\"dWaa\": dWaa, \"dWax\": dWax, \"dWya\": dWya, \"db\": db, \"dby\": dby}\n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TWcpNTbhT9gw"
   },
   "outputs": [],
   "source": [
    "def initialize_parameters(n_a, n_x, n_y):\n",
    "    np.random.seed(1)\n",
    "    Wax = np.random.randn(n_a, n_x)*0.01 # input to hidden\n",
    "    Waa = np.random.randn(n_a, n_a)*0.01 # hidden to hidden\n",
    "    Wya = np.random.randn(n_y, n_a)*0.01 # hidden to output\n",
    "    b = np.zeros((n_a, 1)) # hidden bias\n",
    "    by = np.zeros((n_y, 1)) # output bias\n",
    "    \n",
    "    parameters = {\"Wax\": Wax, \"Waa\": Waa, \"Wya\": Wya, \"b\": b,\"by\": by}\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Complete this function\n",
    "def update_parameters(parameters, gradients, lr):\n",
    "\n",
    "    parameters['Wax'] += # complete this line \n",
    "    parameters['Waa'] += # complete this line \n",
    "    parameters['Wya'] += # complete this line \n",
    "    parameters['b']  += # complete this line \n",
    "    parameters['by']  += # complete this line \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize(X, Y, a_prev, parameters, learning_rate = 0.01):\n",
    "    # Forward propagate through time\n",
    "    loss, cache = rnn_forward(X, Y, a_prev, parameters)\n",
    "    \n",
    "    # Backpropagate through time \n",
    "    gradients, a = rnn_backward(X, Y, parameters, cache)\n",
    "    \n",
    "    # Clip your gradients between -5 (min) and 5 (max) \n",
    "    gradients = {key:np.clip(gradient, -5, 5, gradient) for key,gradient in gradients.items() }\n",
    "    \n",
    "    # Update parameters\n",
    "    parameters = update_parameters(parameters, gradients, learning_rate)\n",
    "    return loss, gradients, a[len(X)-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(data, index_to_char, char_to_index, num_iterations = 2000, n_a = 50, bird_names = 15, vocab_size = 27):\n",
    "    # Retrieve n_x and n_y from vocab_size\n",
    "    n_x, n_y = vocab_size, vocab_size\n",
    "    \n",
    "    # Initialize parameters\n",
    "    parameters = initialize_parameters(n_a, n_x, n_y)\n",
    "    \n",
    "    # Initialize loss\n",
    "    loss = get_initial_loss(vocab_size, bird_names)\n",
    "    \n",
    "    # Build list of all bird names (training examples).\n",
    "    with open(filepath) as f:    \n",
    "        examples = f.readlines()\n",
    "    examples = [x.lower().strip() for x in examples]\n",
    "    \n",
    "    # Shuffle list of all bird names\n",
    "    np.random.seed(0)\n",
    "    np.random.shuffle(examples)\n",
    "    \n",
    "    # Initialize the hidden state of your RNN\n",
    "    a_prev = np.zeros((n_a, 1))\n",
    "    \n",
    "    # Optimization loop\n",
    "    for j in range(num_iterations):\n",
    "        # Use the hint above to define one training example (X,Y)\n",
    "        index = j % len(examples)\n",
    "        X = [None] + [char_to_index[ch] for ch in examples[index]]\n",
    "        Y = X[1:] + [char_to_index[\"\\n\"]]\n",
    "        \n",
    "        # Perform one optimization step: Forward-prop -> Backward-prop -> Clip -> Update parameters\n",
    "        # Choose a learning rate of 0.01\n",
    "        curr_loss, gradients, a_prev = optimize(X, Y, a_prev, parameters, learning_rate = 0.01)\n",
    "        \n",
    "        \n",
    "        # Use a latency trick to keep the loss smooth. It happens here to accelerate the training.\n",
    "        loss = smooth(loss, curr_loss)\n",
    "\n",
    "        # Every 2000 Iteration, generate \"n\" characters thanks to sample() to check if the model is learning properly\n",
    "        if j % 3000 == 0:\n",
    "            print('Iteration: %d, Loss: %f' % (j, loss) + '\\n')\n",
    "            # The number of bird names to print\n",
    "            seed = 1\n",
    "            for name in range(bird_names):\n",
    "                # Sample indices and print them\n",
    "                sampled_indices = sample(parameters, char_to_index, seed)\n",
    "                print_sample(sampled_indices, index_to_char)\n",
    "                seed+=1\n",
    "            print('\\n')\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = model(data, index_to_char, char_to_index, num_iterations=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_sample(sample(parameters, char_to_index, 569874), index_to_char)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Character_level_lm_rnn.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}