Spam filtering using ML Algorithms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loading dataset\n",
    "sms=pd.read_table(\"C:\\Users\\Yatharth Saxena\\Downloads\\Compressed\\pycon-2016-tutorial-master\\pycon-2016-tutorial-master\\data\\sms.tsv\", header =None , names=['label','message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5572, 2)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of entries in xml\n",
    "sms.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>spam</td>\n",
       "      <td>FreeMsg Hey there darling it's been 3 week's n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ham</td>\n",
       "      <td>Even my brother is not like to speak with me. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ham</td>\n",
       "      <td>As per your request 'Melle Melle (Oru Minnamin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>spam</td>\n",
       "      <td>WINNER!! As a valued network customer you have...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>spam</td>\n",
       "      <td>Had your mobile 11 months or more? U R entitle...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...\n",
       "5  spam  FreeMsg Hey there darling it's been 3 week's n...\n",
       "6   ham  Even my brother is not like to speak with me. ...\n",
       "7   ham  As per your request 'Melle Melle (Oru Minnamin...\n",
       "8  spam  WINNER!! As a valued network customer you have...\n",
       "9  spam  Had your mobile 11 months or more? U R entitle..."
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#printing top 10 entries\n",
    "sms.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     4825\n",
       "spam     747\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#counting ham and spam entries\n",
    "sms.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a label, giving ham as 0(negative) and spam as 1(positive)\n",
    "sms['label_num'] = sms.label.map({'ham':0,'spam':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "      <th>label_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message  label_num\n",
       "0   ham  Go until jurong point, crazy.. Available only ...          0\n",
       "1   ham                      Ok lar... Joking wif u oni...          0\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...          1\n",
       "3   ham  U dun say so early hor... U c already then say...          0\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...          0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572,)\n",
      "(5572,)\n"
     ]
    }
   ],
   "source": [
    "#X is feature matrix and y is output matrix\n",
    "X=sms.message\n",
    "y=sms.label_num\n",
    "print(X.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4179,)\n",
      "(1393,)\n",
      "(4179,)\n",
      "(1393,)\n"
     ]
    }
   ],
   "source": [
    "#splitting training data and test data in 75% and 25% respectively\n",
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test= train_test_split(X,y, random_state=1)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#from nltk.stem.snowball import SnowballStemmer\n",
    "#stemmer =SnowballStemmer(\"english\")\n",
    "#for word in X:\n",
    "#   for j in word.split(\" \"):\n",
    "#      j=stemmer.stem(j)        \n",
    "#print X_train.head(5)\n",
    "#count vectorizer is tool used to learn vocabulary from the documents\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vect= CountVectorizer(stop_words='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_dtm= vect.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<4179x7204 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 32467 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sparse matrix\n",
    "X_train_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1393x7204 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 9780 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_dtm= vect.transform(X_test)\n",
    "X_test_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#naive bayes is used for vocabulary\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "nb= MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 7 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fitting dtraining data in naive bayes\n",
    "%time nb.fit(X_train_dtm, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#predicting output for test data\n",
    "y_pred_class= nb.predict(X_test_dtm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9877961234745154"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calculating accuracy of output of test data\n",
    "from sklearn import metrics\n",
    "metrics.accuracy_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1201,    7],\n",
       "       [  10,  175]], dtype=int64)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#confusion matricsshows number of wrong  and right predictions\n",
    "metrics.confusion_matrix(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    " upper left= true positive\n",
    " bottom right= true negative\n",
    " upper right= false positive\n",
    " bottom left= false negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#logistic reggression\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#comparing with logistic regression\n",
    "logreg=LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 133 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time logreg.fit(X_train_dtm, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_class = logreg.predict(X_test_dtm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97989949748743721"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.accuracy_score(y_test,y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7204"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tokens = vect.get_feature_names()\n",
    "len(X_train_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'00', u'000', u'008704050406', u'0121', u'01223585236', u'01223585334', u'0125698789', u'02', u'0207', u'02072069400', u'02073162414', u'02085076972', u'021', u'03', u'04', u'0430', u'05', u'050703', u'0578', u'06', u'07', u'07008009200', u'07090201529', u'07090298926', u'07123456789', u'07732584351', u'07734396839', u'07742676969', u'0776xxxxxxx', u'07781482378', u'07786200117', u'078', u'07801543489', u'07808', u'07808247860', u'07808726822', u'07815296484', u'07821230901', u'07880867867', u'0789xxxxxxx', u'07946746291', u'0796xxxxxx', u'07973788240', u'07xxxxxxxxx', u'08', u'0800', u'08000407165', u'08000776320', u'08000839402', u'08000930705']\n"
     ]
    }
   ],
   "source": [
    "print X_train_tokens[0:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'yelling', u'yellow', u'yen', u'yeovil', u'yep', u'yer', u'yes', u'yest', u'yesterday', u'yetunde', u'yijue', u'ym', u'ymca', u'yo', u'yoga', u'yogasana', u'yor', u'yorge', u'youdoing', u'youi', u'youphone', u'youre', u'yourjob', u'youwanna', u'yowifes', u'yoyyooo', u'yr', u'yrs', u'ything', u'yummmm', u'yummy', u'yun', u'yunny', u'yuo', u'yuou', u'yup', u'zac', u'zaher', u'zealand', u'zebra', u'zed', u'zeros', u'zhong', u'zindgi', u'zoe', u'zoom', u'zouk', u'zyada', u'\\xe8n', u'\\u3028ud']\n"
     ]
    }
   ],
   "source": [
    "print X_train_tokens[-50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.,   0.,   0., ...,   1.,   1.,   1.],\n",
       "       [  5.,  23.,   2., ...,   0.,   0.,   0.]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb.feature_count_\n",
    "#this matrix represents that how many times each of 7456 feature appeared in ham and in spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 7204)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb.feature_count_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0., ...,  1.,  1.,  1.])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ham_token_count = nb.feature_count_[0,:]\n",
    "ham_token_count "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  5.,  23.,   2., ...,   0.,   0.,   0.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_token_count = nb.feature_count_[1,:]\n",
    "spam_token_count "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens= pd.DataFrame({'token':X_train_tokens, 'ham':ham_token_count,'spam':spam_token_count})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ham</th>\n",
       "      <th>spam</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>008704050406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>01223585236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ham  spam         token\n",
       "0  0.0   5.0            00\n",
       "1  0.0  23.0           000\n",
       "2  0.0   2.0  008704050406\n",
       "3  0.0   1.0          0121\n",
       "4  0.0   1.0   01223585236"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ham</th>\n",
       "      <th>spam</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>iphone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>450ppw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3585</th>\n",
       "      <td>9.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>join</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2270</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>dontcha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1416</th>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>brings</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ham  spam    token\n",
       "3488  1.0   0.0   iphone\n",
       "488   0.0   2.0   450ppw\n",
       "3585  9.0  13.0     join\n",
       "2270  0.0   1.0  dontcha\n",
       "1416  4.0   1.0   brings"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens.sample(5, random_state=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3617.,   562.])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb.class_count_\n",
    "#3617 - span\n",
    "#562 - ham"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ham</th>\n",
       "      <th>spam</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>iphone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>450ppw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3585</th>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>join</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2270</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>dontcha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1416</th>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>brings</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ham  spam    token\n",
       "3488   2.0   1.0   iphone\n",
       "488    1.0   3.0   450ppw\n",
       "3585  10.0  14.0     join\n",
       "2270   1.0   2.0  dontcha\n",
       "1416   5.0   2.0   brings"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens['ham']=tokens.ham+1\n",
    "tokens['spam']=tokens.spam+1\n",
    "tokens.sample(5, random_state=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ham</th>\n",
       "      <th>spam</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>iphone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.005338</td>\n",
       "      <td>450ppw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3585</th>\n",
       "      <td>0.002765</td>\n",
       "      <td>0.024911</td>\n",
       "      <td>join</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2270</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>dontcha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1416</th>\n",
       "      <td>0.001382</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>brings</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ham      spam    token\n",
       "3488  0.000553  0.001779   iphone\n",
       "488   0.000276  0.005338   450ppw\n",
       "3585  0.002765  0.024911     join\n",
       "2270  0.000276  0.003559  dontcha\n",
       "1416  0.001382  0.003559   brings"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens['ham']=tokens.ham / nb.class_count_[0]\n",
    "tokens['spam']=tokens.spam / nb.class_count_[1]\n",
    "tokens.sample(5,random_state=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ham</th>\n",
       "      <th>spam</th>\n",
       "      <th>token</th>\n",
       "      <th>spam_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>iphone</td>\n",
       "      <td>3.217972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.005338</td>\n",
       "      <td>450ppw</td>\n",
       "      <td>19.307829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3585</th>\n",
       "      <td>0.002765</td>\n",
       "      <td>0.024911</td>\n",
       "      <td>join</td>\n",
       "      <td>9.010320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2270</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>dontcha</td>\n",
       "      <td>12.871886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1416</th>\n",
       "      <td>0.001382</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>brings</td>\n",
       "      <td>2.574377</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ham      spam    token  spam_ratio\n",
       "3488  0.000553  0.001779   iphone    3.217972\n",
       "488   0.000276  0.005338   450ppw   19.307829\n",
       "3585  0.002765  0.024911     join    9.010320\n",
       "2270  0.000276  0.003559  dontcha   12.871886\n",
       "1416  0.001382  0.003559   brings    2.574377"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calculating spamyness of each word\n",
    "tokens['spam_ratio']=tokens.spam / tokens.ham\n",
    "tokens.sample(5,random_state=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ham</th>\n",
       "      <th>spam</th>\n",
       "      <th>token</th>\n",
       "      <th>spam_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1718</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.158363</td>\n",
       "      <td>claim</td>\n",
       "      <td>572.798932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5043</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.135231</td>\n",
       "      <td>prize</td>\n",
       "      <td>489.131673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.087189</td>\n",
       "      <td>150p</td>\n",
       "      <td>315.361210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6470</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.085409</td>\n",
       "      <td>tone</td>\n",
       "      <td>308.925267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3048</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.076512</td>\n",
       "      <td>guaranteed</td>\n",
       "      <td>276.745552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.069395</td>\n",
       "      <td>18</td>\n",
       "      <td>251.001779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1971</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.065836</td>\n",
       "      <td>cs</td>\n",
       "      <td>238.129893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7110</th>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.129893</td>\n",
       "      <td>www</td>\n",
       "      <td>234.911922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.056940</td>\n",
       "      <td>1000</td>\n",
       "      <td>205.950178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1101</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.053381</td>\n",
       "      <td>awarded</td>\n",
       "      <td>193.078292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.051601</td>\n",
       "      <td>150ppm</td>\n",
       "      <td>186.642349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6614</th>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.099644</td>\n",
       "      <td>uk</td>\n",
       "      <td>180.206406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>521</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.048043</td>\n",
       "      <td>500</td>\n",
       "      <td>173.770463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5389</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.044484</td>\n",
       "      <td>ringtone</td>\n",
       "      <td>160.898577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4244</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.042705</td>\n",
       "      <td>mob</td>\n",
       "      <td>154.462633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.042705</td>\n",
       "      <td>000</td>\n",
       "      <td>154.462633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1785</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.039146</td>\n",
       "      <td>collection</td>\n",
       "      <td>141.590747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.037367</td>\n",
       "      <td>2000</td>\n",
       "      <td>135.154804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6726</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.037367</td>\n",
       "      <td>valid</td>\n",
       "      <td>135.154804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.037367</td>\n",
       "      <td>10p</td>\n",
       "      <td>135.154804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.037367</td>\n",
       "      <td>800</td>\n",
       "      <td>135.154804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>620</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.035587</td>\n",
       "      <td>8007</td>\n",
       "      <td>128.718861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.067616</td>\n",
       "      <td>16</td>\n",
       "      <td>122.282918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6922</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.033808</td>\n",
       "      <td>weekly</td>\n",
       "      <td>122.282918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6471</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.032028</td>\n",
       "      <td>tones</td>\n",
       "      <td>115.846975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3740</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.032028</td>\n",
       "      <td>land</td>\n",
       "      <td>115.846975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3298</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.032028</td>\n",
       "      <td>http</td>\n",
       "      <td>115.846975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.030249</td>\n",
       "      <td>5000</td>\n",
       "      <td>109.411032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4391</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.030249</td>\n",
       "      <td>national</td>\n",
       "      <td>109.411032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5471</th>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.030249</td>\n",
       "      <td>sae</td>\n",
       "      <td>109.411032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2170</th>\n",
       "      <td>0.011335</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>didn</td>\n",
       "      <td>0.156974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1800</th>\n",
       "      <td>0.011612</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>coming</td>\n",
       "      <td>0.153237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6876</th>\n",
       "      <td>0.023500</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>wat</td>\n",
       "      <td>0.151434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2667</th>\n",
       "      <td>0.011888</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>fine</td>\n",
       "      <td>0.149673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2970</th>\n",
       "      <td>0.012165</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>gonna</td>\n",
       "      <td>0.146271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3760</th>\n",
       "      <td>0.012165</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>late</td>\n",
       "      <td>0.146271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4573</th>\n",
       "      <td>0.024882</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>oh</td>\n",
       "      <td>0.143021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7134</th>\n",
       "      <td>0.012441</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>ya</td>\n",
       "      <td>0.143021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6940</th>\n",
       "      <td>0.012718</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>went</td>\n",
       "      <td>0.139912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3887</th>\n",
       "      <td>0.052530</td>\n",
       "      <td>0.007117</td>\n",
       "      <td>ll</td>\n",
       "      <td>0.135494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2620</th>\n",
       "      <td>0.013824</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>feel</td>\n",
       "      <td>0.128719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6460</th>\n",
       "      <td>0.013824</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>told</td>\n",
       "      <td>0.128719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3050</th>\n",
       "      <td>0.014100</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>gud</td>\n",
       "      <td>0.126195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1901</th>\n",
       "      <td>0.014929</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>cos</td>\n",
       "      <td>0.119184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>903</th>\n",
       "      <td>0.015206</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>amp</td>\n",
       "      <td>0.117017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6165</th>\n",
       "      <td>0.015206</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>sure</td>\n",
       "      <td>0.117017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4578</th>\n",
       "      <td>0.061100</td>\n",
       "      <td>0.007117</td>\n",
       "      <td>ok</td>\n",
       "      <td>0.116488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5475</th>\n",
       "      <td>0.016312</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>said</td>\n",
       "      <td>0.109084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4289</th>\n",
       "      <td>0.016865</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>morning</td>\n",
       "      <td>0.105507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3910</th>\n",
       "      <td>0.017694</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>lol</td>\n",
       "      <td>0.100562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7149</th>\n",
       "      <td>0.017694</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>yeah</td>\n",
       "      <td>0.100562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2259</th>\n",
       "      <td>0.019077</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>doing</td>\n",
       "      <td>0.093275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1032</th>\n",
       "      <td>0.019630</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>ask</td>\n",
       "      <td>0.090647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6891</th>\n",
       "      <td>0.019630</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>way</td>\n",
       "      <td>0.090647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1794</th>\n",
       "      <td>0.048936</td>\n",
       "      <td>0.003559</td>\n",
       "      <td>come</td>\n",
       "      <td>0.072723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3762</th>\n",
       "      <td>0.030688</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>later</td>\n",
       "      <td>0.057981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015</th>\n",
       "      <td>0.032900</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>da</td>\n",
       "      <td>0.054084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3932</th>\n",
       "      <td>0.032900</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>lor</td>\n",
       "      <td>0.054084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td>0.064142</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>lt</td>\n",
       "      <td>0.027741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3044</th>\n",
       "      <td>0.064971</td>\n",
       "      <td>0.001779</td>\n",
       "      <td>gt</td>\n",
       "      <td>0.027387</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7204 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           ham      spam       token  spam_ratio\n",
       "1718  0.000276  0.158363       claim  572.798932\n",
       "5043  0.000276  0.135231       prize  489.131673\n",
       "293   0.000276  0.087189        150p  315.361210\n",
       "6470  0.000276  0.085409        tone  308.925267\n",
       "3048  0.000276  0.076512  guaranteed  276.745552\n",
       "307   0.000276  0.069395          18  251.001779\n",
       "1971  0.000276  0.065836          cs  238.129893\n",
       "7110  0.000553  0.129893         www  234.911922\n",
       "246   0.000276  0.056940        1000  205.950178\n",
       "1101  0.000276  0.053381     awarded  193.078292\n",
       "297   0.000276  0.051601      150ppm  186.642349\n",
       "6614  0.000553  0.099644          uk  180.206406\n",
       "521   0.000276  0.048043         500  173.770463\n",
       "5389  0.000276  0.044484    ringtone  160.898577\n",
       "4244  0.000276  0.042705         mob  154.462633\n",
       "1     0.000276  0.042705         000  154.462633\n",
       "1785  0.000276  0.039146  collection  141.590747\n",
       "336   0.000276  0.037367        2000  135.154804\n",
       "6726  0.000276  0.037367       valid  135.154804\n",
       "254   0.000276  0.037367         10p  135.154804\n",
       "617   0.000276  0.037367         800  135.154804\n",
       "620   0.000276  0.035587        8007  128.718861\n",
       "302   0.000553  0.067616          16  122.282918\n",
       "6922  0.000276  0.033808      weekly  122.282918\n",
       "6471  0.000276  0.032028       tones  115.846975\n",
       "3740  0.000276  0.032028        land  115.846975\n",
       "3298  0.000276  0.032028        http  115.846975\n",
       "522   0.000276  0.030249        5000  109.411032\n",
       "4391  0.000276  0.030249    national  109.411032\n",
       "5471  0.000276  0.030249         sae  109.411032\n",
       "...        ...       ...         ...         ...\n",
       "2170  0.011335  0.001779        didn    0.156974\n",
       "1800  0.011612  0.001779      coming    0.153237\n",
       "6876  0.023500  0.003559         wat    0.151434\n",
       "2667  0.011888  0.001779        fine    0.149673\n",
       "2970  0.012165  0.001779       gonna    0.146271\n",
       "3760  0.012165  0.001779        late    0.146271\n",
       "4573  0.024882  0.003559          oh    0.143021\n",
       "7134  0.012441  0.001779          ya    0.143021\n",
       "6940  0.012718  0.001779        went    0.139912\n",
       "3887  0.052530  0.007117          ll    0.135494\n",
       "2620  0.013824  0.001779        feel    0.128719\n",
       "6460  0.013824  0.001779        told    0.128719\n",
       "3050  0.014100  0.001779         gud    0.126195\n",
       "1901  0.014929  0.001779         cos    0.119184\n",
       "903   0.015206  0.001779         amp    0.117017\n",
       "6165  0.015206  0.001779        sure    0.117017\n",
       "4578  0.061100  0.007117          ok    0.116488\n",
       "5475  0.016312  0.001779        said    0.109084\n",
       "4289  0.016865  0.001779     morning    0.105507\n",
       "3910  0.017694  0.001779         lol    0.100562\n",
       "7149  0.017694  0.001779        yeah    0.100562\n",
       "2259  0.019077  0.001779       doing    0.093275\n",
       "1032  0.019630  0.001779         ask    0.090647\n",
       "6891  0.019630  0.001779         way    0.090647\n",
       "1794  0.048936  0.003559        come    0.072723\n",
       "3762  0.030688  0.001779       later    0.057981\n",
       "2015  0.032900  0.001779          da    0.054084\n",
       "3932  0.032900  0.001779         lor    0.054084\n",
       "3974  0.064142  0.001779          lt    0.027741\n",
       "3044  0.064971  0.001779          gt    0.027387\n",
       "\n",
       "[7204 rows x 4 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#printing spammy words in decresing order\n",
    "tokens.sort_values('spam_ratio',ascending=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}