Lab12_TFIDF_BagofWords.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LAB- 12: Write a python script to implement Bag of words and TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BAG OF WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import re\n",
    "import numpy as np\n",
    "  \n",
    "# execute the text here as :\n",
    "text = \"\"\"Please give it up for Amaury once again for that outstanding introduction. I have a bunch of good friends here today, including somebody who I served with, who is one of the finest senators in the country, and we’re lucky to have him, your Senator, Dick Durbin is here. I also noticed, by the way, former Governor Edgar here, who I haven’t seen in a long time, and somehow he has not aged and I have. And it’s great to see you, Governor. I want to thank President Killeen and everybody at the U of I System for making it possible for me to be here today. And I am deeply honored at the Paul Douglas Award that is being given to me. He is somebody who set the path for so much outstanding public service here in Illinois. Now, I want to start by addressing the elephant in the room. I know people are still wondering why I didn’t speak at the commencement.  \"\"\"\n",
    "dataset = nltk.sent_tokenize(text)\n",
    "for i in range(len(dataset)):\n",
    "    dataset[i] = dataset[i].lower()\n",
    "    dataset[i] = re.sub(r'\\W', ' ', dataset[i])\n",
    "    dataset[i] = re.sub(r'\\s+', ' ', dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2count = {}\n",
    "for data in dataset:\n",
    "    words = nltk.word_tokenize(data)\n",
    "    for word in words:\n",
    "        if word not in word2count.keys():\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import heapq\n",
    "freq_words = heapq.nlargest(100, word2count, key=word2count.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "for data in dataset:\n",
    "    vector = []\n",
    "    for word in freq_words:\n",
    "        if word in nltk.word_tokenize(data):\n",
    "            vector.append(1)\n",
    "        else:\n",
    "            vector.append(0)\n",
    "    X.append(vector)\n",
    "X = np.asarray(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 0 1 0 0 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 1 1 0 1 1 1 1 0 1 1 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1\n",
      "  1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 1 0 0 1 1 0 1 0 1 0 0 0 0 1 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 1 1 1 1 0 0 0 1 0 1 1 0 0 0 1 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1\n",
      "  1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 1 0 0 1 1 1 1 1 0 0 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 0 0 0 0]\n",
      " [1 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1]]\n"
     ]
    }
   ],
   "source": [
    "print(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    " from collections import Counter\n",
    " from tqdm import tqdm\n",
    " from scipy.sparse import csr_matrix\n",
    " import math\n",
    " import operator\n",
    " from sklearn.preprocessing import normalize\n",
    " import numpy as np "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    " corpus = [\n",
    "      'this is the first document',\n",
    "      'this document is the second document',\n",
    "      'and this is the third one',\n",
    "      'is this the first document',\n",
    " ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    " def IDF(corpus, unique_words):\n",
    "   idf_dict={}\n",
    "   N=len(corpus)\n",
    "   for i in unique_words:\n",
    "     count=0\n",
    "     for sen in corpus:\n",
    "       if i in sen.split():\n",
    "         count=count+1\n",
    "       idf_dict[i]=(math.log((1+N)/(count+1)))+1\n",
    "   return idf_dict "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    " def fit(whole_data):\n",
    "    unique_words = set()\n",
    "    if isinstance(whole_data, (list,)):\n",
    "      for x in whole_data:\n",
    "        for y in x.split():\n",
    "          if len(y)<2:\n",
    "            continue\n",
    "          unique_words.add(y)\n",
    "      unique_words = sorted(list(unique_words))\n",
    "      vocab = {j:i for i,j in enumerate(unique_words)}\n",
    "      Idf_values_of_all_unique_words=IDF(whole_data,unique_words)\n",
    "    return vocab, Idf_values_of_all_unique_words\n",
    " Vocabulary, idf_of_vocabulary=fit(corpus) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this']\n"
     ]
    }
   ],
   "source": [
    " print(list(Vocabulary.keys())) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.916290731874155, 1.2231435513142097, 1.5108256237659907, 1.0, 1.916290731874155, 1.916290731874155, 1.0, 1.916290731874155, 1.0]\n"
     ]
    }
   ],
   "source": [
    " print(list(idf_of_vocabulary.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NORM FORM\n",
      "   (0, 1)\t0.4697913855799205\n",
      "  (0, 2)\t0.580285823684436\n",
      "  (0, 3)\t0.3840852409148149\n",
      "  (0, 6)\t0.3840852409148149\n",
      "  (0, 8)\t0.3840852409148149\n",
      "  (1, 1)\t0.6876235979836937\n",
      "  (1, 3)\t0.2810886740337529\n",
      "  (1, 5)\t0.5386476208856762\n",
      "  (1, 6)\t0.2810886740337529\n",
      "  (1, 8)\t0.2810886740337529\n",
      "  (2, 0)\t0.511848512707169\n",
      "  (2, 3)\t0.267103787642168\n",
      "  (2, 4)\t0.511848512707169\n",
      "  (2, 6)\t0.267103787642168\n",
      "  (2, 7)\t0.511848512707169\n",
      "  (2, 8)\t0.267103787642168\n",
      "  (3, 1)\t0.4697913855799205\n",
      "  (3, 2)\t0.580285823684436\n",
      "  (3, 3)\t0.3840852409148149\n",
      "  (3, 6)\t0.3840852409148149\n",
      "  (3, 8)\t0.3840852409148149\n",
      "(4, 9)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\swati\\anaconda3\\lib\\site-packages\\scipy\\sparse\\_index.py:84: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_intXint(row, col, x.flat[0])\n"
     ]
    }
   ],
   "source": [
    " def transform(dataset,vocabulary,idf_values):\n",
    "     sparse_matrix= csr_matrix( (len(dataset), len(vocabulary)), dtype=np.float64)\n",
    "     for row  in range(0,len(dataset)):\n",
    "       number_of_words_in_sentence=Counter(dataset[row].split())\n",
    "       for word in dataset[row].split():\n",
    "           if word in  list(vocabulary.keys()):\n",
    "               tf_idf_value=(number_of_words_in_sentence[word]/len(dataset[row].split()))*(idf_values[word])\n",
    "               sparse_matrix[row,vocabulary[word]]=tf_idf_value\n",
    "     print(\"NORM FORM\\n\",normalize(sparse_matrix, norm='l2', axis=1, copy=True, return_norm=False))\n",
    "     output =normalize(sparse_matrix, norm='l2', axis=1, copy=True, return_norm=False)\n",
    "     return output\n",
    " final_output=transform(corpus,Vocabulary,idf_of_vocabulary)\n",
    " print(final_output.shape) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         0.46979139 0.58028582 0.38408524 0.         0.\n",
      "  0.38408524 0.         0.38408524]]\n"
     ]
    }
   ],
   "source": [
    " print(final_output[0].toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}