query-retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8qZ-TMc9SVHl"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import numpy as np\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "import re\n",
    "import wordninja \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "####### After importing nltk, run the following only once ######\n",
    "# nltk.download('averaged_perceptron_tagger')\n",
    "# nltk.download('wordnet')\n",
    "# nltk.download('stopwords')\n",
    "### pip install wordninja ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_htmlcodes (document):\n",
    "    \n",
    "    '''Removes HTML entity codes such as &amp from document and returns the clean document'''\n",
    "    \n",
    "    replacement = {\n",
    "                    \"&ampnbsp\": ' ',\n",
    "                    \"&ampamp\": '&',\n",
    "                    \"&ampquot\": '\\'',\n",
    "                    \"&ampldquo\": '\\\"',\n",
    "                    \"&amprdquo\": '\\\"',\n",
    "                    \"&amplsquo\": '\\'',\n",
    "                    \"&amprsquo\": '\\'',\n",
    "                    \"&amphellip\": '...',\n",
    "                    \"&ampndash\": '-',\n",
    "                    \"&ampmdash\": '-'\n",
    "                  }\n",
    "    \n",
    "    for str in replacement:\n",
    "        document = document.replace(str, replacement[str])\n",
    "        \n",
    "    return document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LpQkEVQOSVHr"
   },
   "outputs": [],
   "source": [
    "def get_wordnet_pos (word):\n",
    "    \n",
    "    '''Returns the tag of usage of word depending on context'''\n",
    "    \n",
    "    tag=nltk.pos_tag([word])[0][1][0].upper()\n",
    "    tag_dict={\"J\": wordnet.ADJ, \n",
    "              \"N\": wordnet.NOUN,\n",
    "              \"V\": wordnet.VERB,\n",
    "              \"R\": wordnet.ADV}\n",
    "    \n",
    "    return tag_dict.get(tag,wordnet.NOUN)\n",
    "\n",
    "def lemma_stop (str):\n",
    "    \n",
    "    '''Returns the lemmatized document after tokenization and stop word removal'''\n",
    "    \n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    tokenizer = RegexpTokenizer('\\w+|\\$]\\d\\[+|\\S+,-')\n",
    "    tokenized = tokenizer.tokenize(str)\n",
    "    lemmatized = [lemmatizer.lemmatize(w,get_wordnet_pos(w)) for w in tokenized]\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    filtered_sentence = [w for w in lemmatized if w.lower() not in stop_words]\n",
    "    after_lemma_stop = ' '.join(w for w in filtered_sentence)\n",
    "    \n",
    "    return filtered_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_not_credible (text):\n",
    "    \n",
    "    '''Returns true if text has no special characters, else returns false'''\n",
    "    \n",
    "    match = re.search(r'[!@#?&{}()]', text)\n",
    "    \n",
    "    if match:\n",
    "        return TrueF\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrub_words (text):\n",
    "    \n",
    "    '''Removes special characters from text and returns a clean string'''\n",
    "    \n",
    "    text = re.sub('[!@#?&{}()]', '', text)\n",
    "    text=re.sub(r'[^\\x00-\\x7F]',\" \",text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_document (document_string):\n",
    "    \n",
    "    '''Cleans document_string by splitting very long strings and identifying garbage JSON and HTML and discarding'''\n",
    "    \n",
    "    cleaned_doc = document_string\n",
    "    for word in document_string.split():\n",
    "                if is_not_credible(word):\n",
    "                    temp= scrub_words(word)\n",
    "                    split=wordninja.split(temp)\n",
    "                    if len(split)>7:\n",
    "                          cleaned_doc = cleaned_doc.replace(word,'')\n",
    "                    else:\n",
    "                        replace_with=' '.join(word for word in split)\n",
    "                        cleaned_doc = cleaned_doc.replace(word, replace_with)\n",
    "    return cleaned_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "count_dates = []\n",
    "\n",
    "def replace_dates(documentString, docID):\n",
    "    \n",
    "    '''Replaces dates of the format MM/DD and MM/DD/YYYY with DDmmmYYYY inside documentString'''\n",
    "    \n",
    "    regEx = '(([0-9]+(/)[0-9]+(/)[0-9]+)|([0-9]+(/)[0-9]+))'\n",
    "    iterator = re.finditer(regEx, documentString)\n",
    "    listOfDates = [(m.start(0), m.end(0)) for m in iterator]\n",
    "    tmp = []\n",
    "    replace_with = []\n",
    "    for indices in listOfDates:\n",
    "        date = documentString[indices[0]:indices[1]]\n",
    "        tmp.append(date)\n",
    "        count = date.count('/')\n",
    "        newDate = ''\n",
    "        if count == 2:\n",
    "            check_year = date[-3]\n",
    "            \n",
    "            if check_year == '/':\n",
    "                YY = date[-2:]\n",
    "                \n",
    "                if int(YY) <= 19:\n",
    "                    proper_date = date[:-2] + '20' + YY\n",
    "                    date = date.replace(date,proper_date)\n",
    "                else:\n",
    "                    proper_date = date[:-2] + '19' + YY\n",
    "                    date = date.replace(YY,('19'+YY))\n",
    "                    \n",
    "            try:\n",
    "                newDate = datetime.strptime(date, '%m/%d/%Y').strftime('%d %b %Y')\n",
    "            except ValueError as ve:\n",
    "                newDate = date\n",
    "        else:\n",
    "            try:\n",
    "                newDate = datetime.strptime(date, '%m/%d').strftime('%d %b')\n",
    "            except ValueError as ve:\n",
    "                newDate = date\n",
    "                \n",
    "        count_dates.append([docID, date])\n",
    "        newDate = newDate.replace(' ', '')\n",
    "        replace_with.append(newDate)\n",
    "        \n",
    "    for i in range(len(tmp)):\n",
    "        documentString = documentString.replace(tmp[i], replace_with[i])\n",
    "    \n",
    "    return documentString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fr-HGZhDrJfT"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000\n"
     ]
    }
   ],
   "source": [
    "# Reading persistent files\n",
    "\n",
    "import pickle\n",
    "import trie\n",
    "\n",
    "get_docID = {}\n",
    "get_index = {}\n",
    "\n",
    "data = np.load(\"datan.npy\", allow_pickle = True)\n",
    "\n",
    "print(len(data))\n",
    "\n",
    "for i in range(0, len(data)) :\n",
    "    get_docID[i] = data[i][0]\n",
    "    get_index[data[i][0]] = i\n",
    "collection = None\n",
    "documentRoot = {}\n",
    "max_tf = {}\n",
    "\n",
    "with open('collection.pickle', 'rb') as handle:\n",
    "    collection = pickle.load(handle)\n",
    "with open('documentRoot.pickle', 'rb') as handle:\n",
    "    documentRoot = pickle.load(handle)\n",
    "with open('max_tf.pickle', 'rb') as handle:\n",
    "    max_tf = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Go9w6bB9rJfg",
    "outputId": "2e5b7bc8-2e68-4de8-f81d-a142a989c902"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['messi', 'india']\n"
     ]
    }
   ],
   "source": [
    "# Processing query\n",
    "\n",
    "import unidecode\n",
    "\n",
    "query = \"messi india\"\n",
    "final_query = replace_dates(query, -1)\n",
    "final_query = lemma_stop(final_query)\n",
    "\n",
    "for i in range(len(final_query)):\n",
    "    final_query[i] = unidecode.unidecode(final_query[i])\n",
    "    # case-folding\n",
    "    final_query[i] = final_query[i].lower()\n",
    "print(final_query)\n",
    "\n",
    "tf_query = {}\n",
    "for w in final_query:\n",
    "    if w not in tf_query:\n",
    "        tf_query[w] = 1\n",
    "    else:\n",
    "        tf_query[w] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rBHDg-lJrJfo"
   },
   "source": [
    "***Ranked Retrieval based on TF-IDF Score :***\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lfT-kdUqrJfp",
    "outputId": "3c95efd0-76c9-4dfe-e425-e79c089931ee",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "Term in query =  messi\n",
      "\n",
      "df =  607\n",
      "idf =  1.61475131759678\n",
      "\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607523111-21620\n",
      "Keywords:\n",
      "\n",
      "Copa America 2019: Brazil still wary of Lionel Messi, says Thiago Silva\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Lionel Messi has not been at his best at the Copa America but Brazil defender Thiago Silva says his side will not take their eyes off the Argentina captain for a minute in their semi-final on Tuesday.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607520074-9575\n",
      "Keywords:\n",
      "\n",
      "Lionel Messi scores twice as Barcelona beat Real Madrid 3-2 in thrilling El Clasico\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Lionel Messi's last minute goal sealed a win for Barcelona in a thriller against Real Madrid.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607523725-24636\n",
      "Keywords:\n",
      "\n",
      "Luis Suarez agrees deal to join Juventus: Reports\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "... place, the 33-year-old striker will combine with Portugal star Cristiano Ronaldo from the next season instead of his rival Lionel Messi at the Camp Nou.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607519002-5479\n",
      "Keywords:\n",
      "\n",
      "Raul Gonzalez, Lionel Messi among La Liga’s five best players\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Raul Gonzalez, Cesar Rodriguez, Telmo Zarra, Lionel Messi and Enrique Castro 'Quini' led the list of the best footballers in La Liga's history.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607520979-13107\n",
      "Keywords:\n",
      "\n",
      "Lionel Messi double helps Barcelona beat Las Palmas in empty Nou Camp\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Lionel Messi then struck his 10th league goal of the season to double Barca's advantage in the 70th.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607518766-4530\n",
      "Keywords:\n",
      "\n",
      "Barcelona angry over Messi, Suarez being overlooked for UEFA award\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Five-time World Footballer of the Year Messi was placed fifth, behind team-mate Suarez, leading to Mestre's diatribe against UEFA.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607517675-347\n",
      "Keywords:\n",
      "\n",
      "FIFA World Cup: Knocking on heaven’s door\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Reluctant superstar Lionel Messi prepares for the grandest stage as pantheon of greats beckons.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607522922-20745\n",
      "Keywords:\n",
      "\n",
      "La Liga Roundup: Real Betis fans cheer as Lionel Messi scores hat-trick for Barcelona\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Lionel Messi scored a hat trick with his brilliant third goal applauded by opposition fans as Barcelona beat Real Betis 4-1 to open up a commanding 10-point lead of the Spanish league  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607522577-19137\n",
      "Keywords:\n",
      "\n",
      "Lionel Messi back against Real Betis, Ousmane Dembele excluded from Barcelona squad\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Messi has been training in the last week and travelled with the team to Tuesday's Champions League game against Inter Milan but did not make the bench for the 1-1 draw.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  1607521877-16603\n",
      "Keywords:\n",
      "\n",
      "Lionel Messi stunner sinks Atletico Madrid\n",
      "\n",
      "title score =  0\n",
      "messi -1.61475131759678 1\n",
      "\n",
      "\n",
      "Barcelona beat nearest challengers Atletico Madrid 1-0 thanks to an outstanding free kick from Lionel Messi on Sunday to move eight points clear at the top of La Liga and closer to a 25th Spanish title.  ... \n",
      "\n",
      "tf-idf score= 2.607421817680537\n",
      "\n",
      "\n",
      "============================================\n"
     ]
    }
   ],
   "source": [
    "import queue\n",
    "\n",
    "# scores[i] stores the dot product of the tf-idf score vectors of the query and document of docID i in the corpus\n",
    "scores = {}\n",
    "title_score = {}\n",
    "\n",
    "# N is the total number of documents in the corpus\n",
    "N = len(documentRoot)\n",
    "\n",
    "# wordsInDoc[i] is a sorted list of (word, score) tuples where\n",
    "# score is the tf-idf score for the (word, <ith doc>) pair\n",
    "wordsInDoc = {}\n",
    "\n",
    "factor = {}\n",
    "\n",
    "import math\n",
    "import bisect\n",
    "\n",
    "for query_term in tf_query:\n",
    "    \n",
    "    docs_having_query_term = collection.get_doc_list(query_term, 0)\n",
    "    df = len(docs_having_query_term)\n",
    "    idf = 0\n",
    "    \n",
    "    print('-------------------------------------')\n",
    "    print('Term in query = ', query_term)\n",
    "    print()\n",
    "    \n",
    "    if df == 0:\n",
    "        idf = 0\n",
    "    else:\n",
    "        idf = math.log10(N/df)\n",
    "        \n",
    "    docs_having_query_term_in_title = collection.get_title_list(query_term,0)\n",
    "    \n",
    "    for docID in docs_having_query_term_in_title:\n",
    "        if docID in title_score:\n",
    "            title_score[docID] += idf\n",
    "        else:\n",
    "            title_score[docID] = idf\n",
    "        \n",
    "    print('df = ',df)\n",
    "    print('idf = ',idf)\n",
    "    \n",
    "    tfidf_query = tf_query[query_term] * idf\n",
    "        \n",
    "    for docID in docs_having_query_term:\n",
    "        \n",
    "        tf_doc = documentRoot[docID].count_words(query_term, 0)\n",
    "        tf_doc = 0.5 + 0.5*tf_doc/max_tf[docID]\n",
    "        tfidf_doc = (tf_doc)\n",
    "        \n",
    "        if docID not in scores:\n",
    "            scores[docID] = (tfidf_query * tfidf_doc)\n",
    "            wordsInDoc[docID] = []\n",
    "            bisect.insort(wordsInDoc[docID], [-tfidf_query * tfidf_doc, query_term])\n",
    "            factor[docID] = idf\n",
    "        else:\n",
    "            scores[docID] += (tfidf_query * tfidf_doc)\n",
    "            bisect.insort(wordsInDoc[docID], [-tfidf_query * tfidf_doc, query_term])\n",
    "            factor[docID] += idf\n",
    "            \n",
    "# print(title_score)\n",
    "\n",
    "for docID in scores:\n",
    "    \n",
    "    #if documentLength[docID] != 0:\n",
    "    scores[docID] *= factor[docID]\n",
    "    if docID in title_score:\n",
    "        scores[docID] *= 1 + title_score[docID]\n",
    "\n",
    "sorted_scores = sorted(scores.items(), key = lambda kv : kv[1] , reverse = True)\n",
    "\n",
    "maxshow = min(10, len(scores))\n",
    "print('\\n\\n')\n",
    "print('============================================')\n",
    "\n",
    "for i in range(maxshow):\n",
    "    \n",
    "    print()\n",
    "    docID = sorted_scores[i][0]\n",
    "    print('doc ID = ', docID)\n",
    "    cnt = 0\n",
    "    print('Keywords:')\n",
    "    print()\n",
    "    print(data[get_index[sorted_scores[i][0]]][2])\n",
    "    print()\n",
    "    if sorted_scores[i][0] not in title_score:\n",
    "        print('title score = ',0)\n",
    "    else:\n",
    "        print('title score = ',title_score[sorted_scores[i][0]])\n",
    "    for j in range(len(wordsInDoc[docID])):\n",
    "        print(wordsInDoc[docID][j][1], wordsInDoc[docID][j][0], end = ' ')\n",
    "        print(documentRoot[docID].count_words(wordsInDoc[docID][j][1], 0))\n",
    "    print()\n",
    "    print()\n",
    "    count = 0\n",
    "    found = 0\n",
    "    words_before=queue.Queue()\n",
    "    at_start = 1\n",
    "    display = \"\"\n",
    "    \n",
    "    for word in data[get_index[docID]][4].split():\n",
    "            \n",
    "        check_with=replace_dates(word, -1)\n",
    "        check_with = check_with.lower()\n",
    "        if len(lemma_stop(check_with)) > 0:\n",
    "            check_with=lemma_stop(check_with)[0]\n",
    "        else:\n",
    "            check_with=word\n",
    "        \n",
    "        if check_with == wordsInDoc[docID][0][1]:\n",
    "            found=1\n",
    "            \n",
    "        if found == 1:\n",
    "            display = display + word + \" \"\n",
    "            count += 1\n",
    "            if count == 50:\n",
    "                break\n",
    "        if found == 0:\n",
    "            words_before.put(word)\n",
    "            if words_before.qsize()>20:\n",
    "                remove=words_before.get()\n",
    "                at_start=0\n",
    "                \n",
    "    if not at_start:\n",
    "        print('...', end = ' ')\n",
    "    while words_before.qsize() > 0:\n",
    "        print(words_before.get(), end = ' ')\n",
    "    print(display, end = ' ')\n",
    "    print('...', end = ' ')\n",
    "    print('\\n')\n",
    "    print('tf-idf score=', sorted_scores[i][1])\n",
    "    print('\\n')\n",
    "    print('============================================')\n",
    "#print(sorted_scores)\n",
    "dates = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "preprocessing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}