2Text-Rep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SECTION 2: Text Representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = ['text text mining is interesting',\n",
    "         'text mining is like data mining',\n",
    "         'text and data mining have few differences']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (binary = 'true') gives binary freq\n",
    "# (max_df=2) is when we want to ignore stop words that occur in > 2 docs\n",
    "# (min_df = 2) if word occurs in < 2 docs then we don't want it; noise words\n",
    "# (max_features = 4) gives me best 4 features\n",
    "# ngram_range(2,2) gives bigrams like 'text text', 'text mining', 'mining is'\n",
    "# ngram_range(2,3) gives bigrams and trigrams, min 2 max 3, like 'text text' 'text text mining'\n",
    "\n",
    "vec = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vec.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x10 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 16 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rows, cols\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 1, 1, 0, 1, 2],\n",
       "       [0, 1, 0, 0, 0, 0, 1, 1, 2, 1],\n",
       "       [1, 1, 1, 1, 1, 0, 0, 0, 1, 1]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# makes array with count of tokens\n",
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['and',\n",
       " 'data',\n",
       " 'differences',\n",
       " 'few',\n",
       " 'have',\n",
       " 'interesting',\n",
       " 'is',\n",
       " 'like',\n",
       " 'mining',\n",
       " 'text']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gives unique tokens\n",
    "vec.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vec.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 9,\n",
       " 'mining': 8,\n",
       " 'is': 6,\n",
       " 'interesting': 5,\n",
       " 'like': 7,\n",
       " 'data': 1,\n",
       " 'and': 0,\n",
       " 'have': 4,\n",
       " 'few': 3,\n",
       " 'differences': 2}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gives ids in array\n",
    "vec.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF Factorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min_df, max_df\n",
    "vec = TfidfVectorizer(min_df=1, max_df=2, max_features = 20, ngram_range=(1,1))\n",
    "# vec = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vec.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x8 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.79596054, 0.60534851, 0.        ],\n",
       "       [0.        , 0.51785612, 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.51785612, 0.68091856],\n",
       "       [0.46735098, 0.35543247, 0.46735098, 0.46735098, 0.46735098,\n",
       "        0.        , 0.        , 0.        ]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vec.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'is': 6,\n",
       " 'interesting': 5,\n",
       " 'like': 7,\n",
       " 'data': 1,\n",
       " 'and': 0,\n",
       " 'have': 4,\n",
       " 'few': 3,\n",
       " 'differences': 2}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "#################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = open('dataset.txt').read()\n",
    "# corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = corpus.strip().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x for docs, y for labels\n",
    "documents, labels = [], []\n",
    "\n",
    "for doc in docs:\n",
    "    i, l = doc.split(':')\n",
    "    documents.append(i.strip())\n",
    "    labels.append(l.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = vec.fit_transform(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<6x41 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 51 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , 0.        , 0.3170431 ,\n",
       "        0.        , 0.        , 0.        , 0.3170431 , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.3170431 , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.38663118, 0.38663118, 0.        , 0.38663118,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.38663118, 0.        , 0.        , 0.3170431 ,\n",
       "        0.        ],\n",
       "       [0.        , 0.37393382, 0.        , 0.37393382, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.30663108,\n",
       "        0.37393382, 0.37393382, 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.37393382,\n",
       "        0.        , 0.        , 0.25887903, 0.37393382, 0.        ,\n",
       "        0.        ],\n",
       "       [0.36102204, 0.        , 0.        , 0.        , 0.29604324,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.36102204, 0.36102204, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.36102204, 0.        ,\n",
       "        0.        , 0.        , 0.36102204, 0.        , 0.36102204,\n",
       "        0.        , 0.        , 0.        , 0.36102204, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.35750457, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.35750457,\n",
       "        0.        , 0.35750457, 0.        , 0.35750457, 0.        ,\n",
       "        0.29315886, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.29315886, 0.        , 0.35750457, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.24750486, 0.        , 0.        ,\n",
       "        0.35750457],\n",
       "       [0.        , 0.        , 0.31876546, 0.        , 0.        ,\n",
       "        0.        , 0.31876546, 0.        , 0.26139224, 0.        ,\n",
       "        0.31876546, 0.        , 0.        , 0.26139224, 0.26139224,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.26139224, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.31876546, 0.        , 0.31876546, 0.        ,\n",
       "        0.31876546, 0.        , 0.22068529, 0.        , 0.26139224,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.44644324, 0.        , 0.        , 0.        , 0.44644324,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.44644324, 0.        , 0.        ,\n",
       "        0.36608984, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.36608984, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.36608984, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['and',\n",
       " 'but',\n",
       " 'care',\n",
       " 'cook',\n",
       " 'cooking',\n",
       " 'demand',\n",
       " 'diet',\n",
       " 'fit',\n",
       " 'for',\n",
       " 'hardwork',\n",
       " 'important',\n",
       " 'in',\n",
       " 'include',\n",
       " 'is',\n",
       " 'it',\n",
       " 'like',\n",
       " 'longer',\n",
       " 'lot',\n",
       " 'my',\n",
       " 'need',\n",
       " 'of',\n",
       " 'one',\n",
       " 'pastas',\n",
       " 'playing',\n",
       " 'priorities',\n",
       " 'professional',\n",
       " 'recipe',\n",
       " 'snacks',\n",
       " 'soup',\n",
       " 'special',\n",
       " 'sports',\n",
       " 'sportsmen',\n",
       " 'stay',\n",
       " 'take',\n",
       " 'takes',\n",
       " 'their',\n",
       " 'this',\n",
       " 'to',\n",
       " 'usually',\n",
       " 'very',\n",
       " 'while']"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'this': 36,\n",
       " 'recipe': 26,\n",
       " 'is': 13,\n",
       " 'very': 39,\n",
       " 'special': 29,\n",
       " 'for': 8,\n",
       " 'cooking': 4,\n",
       " 'snacks': 27,\n",
       " 'like': 15,\n",
       " 'to': 37,\n",
       " 'cook': 3,\n",
       " 'but': 1,\n",
       " 'it': 14,\n",
       " 'usually': 38,\n",
       " 'takes': 34,\n",
       " 'longer': 16,\n",
       " 'my': 18,\n",
       " 'priorities': 24,\n",
       " 'in': 11,\n",
       " 'include': 12,\n",
       " 'pastas': 22,\n",
       " 'and': 0,\n",
       " 'soup': 28,\n",
       " 'one': 21,\n",
       " 'need': 19,\n",
       " 'stay': 32,\n",
       " 'fit': 7,\n",
       " 'while': 40,\n",
       " 'playing': 23,\n",
       " 'professional': 25,\n",
       " 'sports': 30,\n",
       " 'important': 10,\n",
       " 'sportsmen': 31,\n",
       " 'take': 33,\n",
       " 'care': 2,\n",
       " 'of': 20,\n",
       " 'their': 35,\n",
       " 'diet': 6,\n",
       " 'demand': 5,\n",
       " 'lot': 17,\n",
       " 'hardwork': 9}"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = open('buttons_amazon_kindle.txt.data',encoding='utf-8',errors='ignore').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = corpus.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = vec.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<166x788 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 2700 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.263758  , ..., 0.        , 0.29434293,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assignment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = open('dataset2.csv', encoding='utf8',errors='ignore').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = corpus.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [],
   "source": [
    "info = vector.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<653x2480 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 14783 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector = TfidfVectorizer(max_df = 200, min_df = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "info = vector.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<653x126 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 6471 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.41527158, ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "info.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}