10-Day2-Text-Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "'''\r\n",
    "What is Text Classification ?\r\n",
    "A fairly popular text classification task is to identify a body of text as either spam or not spam,\r\n",
    "for things like email filters. In our case, we're going to try to create a sentiment analysis algorithm.\r\n",
    "'''"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import nltk\r\n",
    "import random\r\n",
    "from nltk.corpus import movie_reviews"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "documents = [(list(movie_reviews.words(fileid)), category)\r\n",
    "            for category in movie_reviews.categories()\r\n",
    "            for fileid in movie_reviews.fileids(category)]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "random.shuffle(documents)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "all_words = []\r\n",
    "\r\n",
    "for w in movie_reviews.words():\r\n",
    "    all_words.append(w.lower())\r\n",
    "    \r\n",
    "all_words = nltk.FreqDist(all_words)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "print(all_words.most_common(15))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[(',', 77717), ('the', 76529), ('.', 65876), ('a', 38106), ('and', 35576), ('of', 34123), ('to', 31937), (\"'\", 30585), ('is', 25195), ('in', 21822), ('s', 18513), ('\"', 17612), ('it', 16107), ('that', 15924), ('-', 15595)]\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "print(all_words[\"stupid\"])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "253\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Additional Example SMS Classification"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "import pandas as pd\r\n",
    "dataset = pd.read_csv('data.csv', encoding='ISO-8859-1');"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "import re\r\n",
    "import nltk\r\n",
    "\r\n",
    "nltk.download('punkt')\r\n",
    "from nltk.tokenize import word_tokenize as wt "
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "[nltk_data] Downloading package punkt to C:\\Users\\susan/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "!pip install autocorrect"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "nltk.download('stopwords')\r\n",
    "from nltk.corpus import stopwords\r\n",
    "\r\n",
    "from nltk.stem.porter import PorterStemmer\r\n",
    "stemmer = PorterStemmer()\r\n",
    "\r\n",
    "#spell correction\r\n",
    "from autocorrect import Speller\r\n",
    "eng_speller = Speller(lang='en')"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\susan/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "data = []\r\n",
    "\r\n",
    "for i in range(dataset.shape[0]):\r\n",
    "    sms = dataset.iloc[i, 1]\r\n",
    "\r\n",
    "    # remove non alphabatic characters\r\n",
    "    sms = re.sub('[^A-Za-z]', ' ', sms)\r\n",
    "\r\n",
    "    # make words lowercase, because Go and go will be considered as two words\r\n",
    "    sms = sms.lower()\r\n",
    "\r\n",
    "    # tokenising\r\n",
    "    tokenized_sms = wt(sms)\r\n",
    "\r\n",
    "    # remove stop words and stemming\r\n",
    " \r\n",
    "    sms_processed = []\r\n",
    "    for word in tokenized_sms:\r\n",
    "        if word not in set(stopwords.words('english')):\r\n",
    "            sms_processed.append(eng_speller(stemmer.stem(word)))\r\n",
    "\r\n",
    "    sms_text = \" \".join(sms_processed)\r\n",
    "    data.append(sms_text)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "# creating the feature matrix \r\n",
    "from sklearn.feature_extraction.text import CountVectorizer\r\n",
    "matrix = CountVectorizer(max_features=1000)\r\n",
    "X = matrix.fit_transform(data).toarray()\r\n",
    "y = dataset.iloc[:, 0]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "# split train and test data\r\n",
    "from sklearn.model_selection import train_test_split\r\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "# Naive Bayes \r\n",
    "from sklearn.naive_bayes import GaussianNB\r\n",
    "classifier = GaussianNB()\r\n",
    "classifier.fit(X_train, y_train)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "GaussianNB(priors=None, var_smoothing=1e-09)"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "# predict class\r\n",
    "y_pred = classifier.predict(X_test)\r\n",
    "\r\n",
    "# Confusion matrix\r\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\r\n",
    "cm = confusion_matrix(y_test, y_pred)\r\n",
    "cr = classification_report(y_test, y_pred)\r\n",
    "\r\n",
    "accuracy = accuracy_score(y_test, y_pred)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "print(\"Accuracy --->\",accuracy)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Accuracy ---> 0.7753050969131371\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.6.10 64-bit ('tf-gpu': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "interpreter": {
   "hash": "135e78ef6267b613ce7b86630936d470174b66187aad9f784a45e5cc3235687c"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}