Data_Pre-Processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Process Dataset \n",
    "\n",
    "Dataset source: https://www.kaggle.com/snap/amazon-fine-food-reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing data # 0\n",
      "Processing data # 10000\n",
      "Processing data # 20000\n",
      "Processing data # 30000\n",
      "Processing data # 40000\n",
      "Processing data # 50000\n",
      "Processing data # 60000\n",
      "Processing data # 70000\n",
      "Processing data # 80000\n",
      "Processing data # 90000\n",
      "Processing data # 100000\n",
      "Processing data # 110000\n",
      "Processing data # 120000\n",
      "\n",
      "# of Data: 121799\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "from nltk import word_tokenize\n",
    "import string\n",
    "\n",
    "summaries = []\n",
    "texts = []\n",
    "\n",
    "def clean(text):\n",
    "    text = text.lower()\n",
    "    printable = set(string.printable)\n",
    "    text = \"\".join(list(filter(lambda x: x in printable, text))) #filter funny characters, if any.\n",
    "    return text\n",
    "\n",
    "text_max_len = 300\n",
    "text_min_len = 25\n",
    "summary_max_len = 15\n",
    "vocab2idx = {}\n",
    "\n",
    "#Data from https://www.kaggle.com/snap/amazon-fine-food-reviews\n",
    "with open('Dataset/Reviews.csv') as csvfile: \n",
    "    \n",
    "    Reviews = csv.DictReader(csvfile)\n",
    "    \n",
    "    i=0\n",
    "    \n",
    "    for row in Reviews:\n",
    "        \n",
    "        text = row['Text']\n",
    "        summary = row['Summary']\n",
    "        \n",
    "        if len(text) <= text_max_len and len(text) >= text_min_len and len(summary) <= summary_max_len:\n",
    "            #print(i)\n",
    "\n",
    "            clean_text = clean(text)\n",
    "            clean_summary = clean(summary)\n",
    "            \n",
    "            tokenized_summary = word_tokenize(clean_summary)\n",
    "            tokenized_text = word_tokenize(clean_text)\n",
    "            \n",
    "            # BUILD VOCABULARY\n",
    "            \n",
    "            for word in tokenized_text:\n",
    "                if word not in vocab2idx:\n",
    "                    vocab2idx[word]=len(vocab2idx)\n",
    "                    \n",
    "            for word in tokenized_summary:\n",
    "                if word not in vocab2idx:\n",
    "                    vocab2idx[word]=len(vocab2idx)\n",
    "                    \n",
    "            ## ________________\n",
    "\n",
    "            summaries.append(tokenized_summary)\n",
    "            texts.append(tokenized_text)\n",
    "\n",
    "            if i%10000==0:\n",
    "                print(\"Processing data # {}\".format(i))\n",
    "\n",
    "            i+=1\n",
    "\n",
    "print(\"\\n# of Data: {}\".format(len(texts)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SAMPLE CLEANED & TOKENIZED TEXT: \n",
      "\n",
      "['i', 'have', 'bought', 'this', 'same', 'bag', 'right', 'here', 'for', '$', '38.00', 'a', 'few', 'times', ',', 'now', 'its', '$', '108.00', '.', 'what', 'did', 'they', 'do', ',', 'accidentally', 'mix', 'in', 'superman', \"'s\", 'food', '?']\n",
      "\n",
      "\n",
      "SAMPLE CLEANED & TOKENIZED SUMMARY: \n",
      "\n",
      "['rip-off']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "index = random.randint(0,len(texts)-1)\n",
    "\n",
    "print(\"SAMPLE CLEANED & TOKENIZED TEXT: \\n\\n{}\\n\\n\".format(texts[index]))\n",
    "print(\"SAMPLE CLEANED & TOKENIZED SUMMARY: \\n\\n{}\\n\".format(summaries[index]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Embeddings\n",
    "\n",
    "Loading pre-trained GloVe embeddings. Source of Data: https://nlp.stanford.edu/projects/glove/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding Loaded.\n",
      "Vocabulary Size: 26052\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "vocab = []\n",
    "embd = []\n",
    "special_tags = ['<UNK>','<PAD>','<EOS>']\n",
    "\n",
    "\n",
    "def loadEmbeddings(filename):\n",
    "    vocab2embd = {}\n",
    "    \n",
    "    with open(filename) as infile:     \n",
    "        for line in infile:\n",
    "            row = line.strip().split(' ')\n",
    "            word = row[0].lower()\n",
    "            if word not in vocab2embd:\n",
    "                vocab2embd[word]=np.asarray(row[1:],np.float32)\n",
    "\n",
    "    print('Embedding Loaded.')\n",
    "    return vocab2embd\n",
    "\n",
    "vocab2embd = loadEmbeddings('Embeddings/glove.6B.100d.txt')\n",
    "\n",
    "for word in vocab2idx:\n",
    "    if word in vocab2embd:\n",
    "        vocab.append(word)\n",
    "        embd.append(vocab2embd[word])\n",
    "        \n",
    "for special_tag in special_tags:\n",
    "    vocab.append(special_tag)\n",
    "    embd.append(np.random.rand(len(embd[0]),))\n",
    "    \n",
    "vocab2idx = {word:idx for idx,word in enumerate(vocab)}\n",
    "embd = np.asarray(embd,np.float32)\n",
    "\n",
    "print(\"Vocabulary Size: {}\".format(len(vocab2idx)))\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26051\n"
     ]
    }
   ],
   "source": [
    "print(vocab2idx['<EOS>'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorize Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec_texts=[]\n",
    "vec_summaries=[]\n",
    "\n",
    "for text,summary in zip(texts,summaries):\n",
    "    # Replace out of vocab words with index for '<UNK>' tag\n",
    "    vec_texts.append([vocab2idx.get(word,vocab2idx['<UNK>']) for word in text])\n",
    "    vec_summaries.append([vocab2idx.get(word,vocab2idx['<UNK>']) for word in summary])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shuffle Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed(101)\n",
    "\n",
    "texts_idx = [idx for idx in range(len(vec_texts))]\n",
    "random.shuffle(texts_idx)\n",
    "\n",
    "vec_texts = [vec_texts[idx] for idx in texts_idx]\n",
    "vec_summaries = [vec_summaries[idx] for idx in texts_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split Data into train, validation, and test sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use first 10000 data for testing, the next 10000 data for validation, and rest for training\n",
    "\n",
    "test_summaries = vec_summaries[0:10000]\n",
    "test_texts = vec_texts[0:10000]\n",
    "\n",
    "val_summaries = vec_summaries[10000:20000]\n",
    "val_texts = vec_texts[10000:20000]\n",
    "\n",
    "train_summaries = vec_summaries[20000:]\n",
    "train_texts = vec_texts[20000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bucket And Batch Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bucket_and_batch(texts,summaries,batch_size=32):\n",
    "    \n",
    "    # Sort summaries and texts according to the length of text\n",
    "    # (So that texts with similar lengths tend to remain in the same batch and thus require less padding)\n",
    "    \n",
    "    text_lens = [len(text) for text in texts]\n",
    "    sortedidx = np.flip(np.argsort(text_lens),axis=0)\n",
    "    texts=[texts[idx] for idx in sortedidx]\n",
    "    summaries=[summaries[idx] for idx in sortedidx]\n",
    "    \n",
    "    batches_text=[]\n",
    "    batches_summary=[]\n",
    "    batches_true_text_len = []\n",
    "    batches_true_summary_len = []\n",
    "    \n",
    "    i=0\n",
    "    while i < (len(texts)-batch_size):\n",
    "        \n",
    "        max_len = len(texts[i])\n",
    "        \n",
    "        batch_text=[]\n",
    "        batch_summary=[]\n",
    "        batch_true_text_len=[]\n",
    "        batch_true_summary_len=[]\n",
    "        \n",
    "        for j in range(batch_size):\n",
    "            \n",
    "            padded_text = texts[i+j]\n",
    "            padded_summary = summaries[i+j]\n",
    "            \n",
    "            batch_true_text_len.append(len(texts[i+j]))\n",
    "            batch_true_summary_len.append(len(summaries[i+j])+1)\n",
    "     \n",
    "            while len(padded_text) < max_len:\n",
    "                padded_text.append(vocab2idx['<PAD>'])\n",
    "\n",
    "            padded_summary.append(vocab2idx['<EOS>']) #End of Sentence Marker\n",
    "            while len(padded_summary) < summary_max_len+1:\n",
    "                padded_summary.append(vocab2idx['<PAD>'])\n",
    "            \n",
    "        \n",
    "            batch_text.append(padded_text)\n",
    "            batch_summary.append(padded_summary)\n",
    "        \n",
    "        batches_text.append(batch_text)\n",
    "        batches_summary.append(batch_summary)\n",
    "        batches_true_text_len.append(batch_true_text_len)\n",
    "        batches_true_summary_len.append(batch_true_summary_len)\n",
    "        \n",
    "        i+=batch_size\n",
    "        \n",
    "    return batches_text, batches_summary, batches_true_text_len, batches_true_summary_len"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches_text, train_batches_summary, train_batches_true_text_len, train_batches_true_summary_len \\\n",
    "= bucket_and_batch(train_texts, train_summaries)\n",
    "\n",
    "val_batches_text, val_batches_summary, val_batches_true_text_len, val_batches_true_summary_len \\\n",
    "= bucket_and_batch(val_texts, val_summaries)\n",
    "\n",
    "test_batches_text, test_batches_summary, test_batches_true_text_len, test_batches_true_summary_len \\\n",
    "= bucket_and_batch(test_texts, test_summaries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "d = {}\n",
    "\n",
    "d[\"vocab\"] = vocab2idx\n",
    "d[\"embd\"] = embd.tolist()\n",
    "d[\"train_batches_text\"] = train_batches_text\n",
    "d[\"test_batches_text\"] = test_batches_text\n",
    "d[\"val_batches_text\"] = val_batches_text\n",
    "d[\"train_batches_summary\"] = train_batches_summary\n",
    "d[\"test_batches_summary\"] = test_batches_summary\n",
    "d[\"val_batches_summary\"] = val_batches_summary\n",
    "d[\"train_batches_true_text_len\"] = train_batches_true_text_len\n",
    "d[\"val_batches_true_text_len\"] = val_batches_true_text_len\n",
    "d[\"test_batches_true_text_len\"] = test_batches_true_text_len\n",
    "d[\"train_batches_true_summary_len\"] = train_batches_true_summary_len\n",
    "d[\"val_batches_true_summary_len\"] = val_batches_true_summary_len\n",
    "d[\"test_batches_true_summary_len\"] = test_batches_true_summary_len\n",
    "\n",
    "with open('Processed_Data/Amazon_Reviews_Processed.json', 'w') as outfile:\n",
    "    json.dump(d, outfile)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}