Govind_ATML_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Govind_ATML .ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "18gQgtrEFPOf1PK9R26X0FFebSxGz2GyU",
      "authorship_tag": "ABX9TyPLdQYMBA6MAb0RXreEEQMU",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aniruddh-shukla/ATML-proj/blob/main/Govind_ATML_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QHxVSOeHNJk9"
      },
      "source": [
        "import re\n",
        "import numpy as np \n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from tabulate import tabulate\n",
        "\n",
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "import string\n",
        "\n",
        "# dataset\n",
        "from sklearn.datasets import fetch_20newsgroups\n",
        "\n",
        "### Pre-processing\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction import DictVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from collections import Counter\n",
        "\n",
        "## Classifiers\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.svm import SVC\n",
        "\n",
        "##Evaluation\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f1LuDGQINYIx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "293675cd-865d-4540-cc41-e84ce39053cf"
      },
      "source": [
        "news_group = fetch_20newsgroups(remove=('headers', 'footers', 'quotes'))\n",
        "news_group_data = news_group.data\n",
        "news_group_target_names = news_group.target_names\n",
        "news_group_target = news_group.target\n",
        "\n",
        "# Creating a dataframe from the loaded data\n",
        "news_df = pd.DataFrame({'desc': news_group_data, \n",
        "                        'class': news_group_target})"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading 20news dataset. This may take a few minutes.\n",
            "Downloading dataset from https://ndownloader.figshare.com/files/5975967 (14 MB)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zOegtuwsOkxp"
      },
      "source": [
        "###EDA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UyBYgwqblnit",
        "outputId": "c7b56b1b-0d11-43e1-de76-67f72d196332"
      },
      "source": [
        "print(news_group_target_names)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uAqFzO__Ns9h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fdc3a718-06f4-43f9-a4f9-8044c55bc6b3"
      },
      "source": [
        "news_df.info()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 11314 entries, 0 to 11313\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   desc    11314 non-null  object\n",
            " 1   class   11314 non-null  int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 176.9+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LlI_u_6oOTIX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "outputId": "891706a1-9f61-4e3e-fee5-981a87ab49a6"
      },
      "source": [
        "#Class distribution Visualization\n",
        "x = \"class\"\n",
        "\n",
        "fig, ax = plt.subplots()\n",
        "fig.suptitle(x, fontsize=12)\n",
        "news_df[x].reset_index().groupby(x).count().sort_values(by= \n",
        "       \"index\").plot(kind=\"barh\", legend=False, \n",
        "        ax=ax).grid(axis='x')\n",
        "plt.show()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEVCAYAAADn6Y5lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZt0lEQVR4nO3df5QdZX3H8feHhB8SfiQkIVIIDTGQlqIGpBRUaMBqI1JptVJSrVDTprbaolUp1BZqezwHrKi09mhTiIhiEBBaSq1AlZByDiIbiJCQIKhBNiCLYgihKCT59o95FpfN3d27yz4zOzOf1zl79t5n5t7n+Yble+c+M893FBGYmVl77FL1AMzMrFxO/GZmLePEb2bWMk78ZmYt48RvZtYyTvxmZi3jxG+WSDpT0m1Vj8MsNyd+M7OWceI3M2sZJ35rJUmzJV0r6XFJP5b06Q77XCzpYUlbJK2WdPyAbcdI6knbHpP0idS+h6QvpvfcLOlOSbPKjM1sJE781jqSJgE3AA8Bc4ADgSs77HonsADYD/gScLWkPdK2i4GLI2If4GXAVan9DGBfYDYwHXg38EyWQMzGyInf2ugY4BeAD0XE0xHx04jY6aRuRHwxIn4cEdsi4iJgd2B+2vwcME/SjIjYGhHfHNA+HZgXEdsjYnVEbCkhJrOuOfFbG80GHoqIbcPtJOmDktZLelLSZooj+Rlp8xLgMGBDms45JbV/AbgRuFLSI5I+JmnXTHGYjYkTv7XRw8DBkiYPtUOazz8bOA2YFhFTgScBAUTEAxGxGNgfuBC4RtKUiHguIj4SEYcDrwZOAd6ZNxyz0XHitzb6FvAocIGkKemE7GsG7bM3sA14HJgs6Txgn/6Nkt4haWZE7AA2p+Ydkk6U9PJ0HmELxdTPjtwBmY2GE7+1TkRsB34LmAf8AOgFfm/QbjcCXwO+Q3ES+KcU3xT6LQLWSdpKcaL39Ih4BngpcA1F0l8P3Eox/WM2Ycg3YjEzaxcf8ZuZtYwTv5lZyzjxm5m1jBO/mVnLOPGbmbWME7+ZWcs48ZuZtYwTv5lZyzjxm5m1jBO/mVnLOPGbmbWME7+ZWcs48ZuZtYwTv5lZyzjxm5m1jBO/mVnLOPGbmbXMkDebnkimTp0a8+bNq3oY4+rpp59mypQpVQ9jXDmm+mhiXI5pZ6tXr/5RRMwc3F6LxD9r1ix6enqqHsa4WrlyJQsXLqx6GOPKMdVHE+NyTDuT9FCn9mxTPZKWS+qTtHZA236Sbpb0QPo9LVf/ZmbWWbabrUs6AdgKXB4RR6S2jwFPRMQFks4BpkXEX430XgfPnRe7nHZxlnFW5QMv38ZF99biC1fXHFN9NDGuJsZ02aIpL/aIf3VEHD24PdsRf0SsAp4Y1Hwq8Pn0+PPAb+fq38zMOiv7qp5ZEfFoevxDYFbJ/ZuZtV5ll3NGMcc05DyTpKWSeiT1bN2ypcSRmZk1W9mJ/zFJBwCk331D7RgRyyLi6Ig4eq999iltgGZmTVd24r8eOCM9PgP4j5L7NzNrvWynwCWtABYCMyT1AucDFwBXSVoCPASc1s17vWTXSdx/wZtyDbUSK1euZOPbF1Y9jHHlmOqjiXE1NaYcsiX+iFg8xKbX5erTzMxG5lo9ZmYt48RvZtYyZZds+DtJmyStST8n5+rfzMw6K7tkw98BWyPi46N5L5dsqAfHVB9NjKtpMW284E3jUaRtQpRsMDOzilUxx/9eSfekqSBX5zQzK1nZif8zwMuABcCjwEVD7eiSDWZmeZSa+CPisYjYHhE7gH8DjhlmX5dsMDPLoNQzIZIOGFCd83eAtcPt388rd+vBMdVHE+NqYky5lF2yYaGkBRRVOTcCf5KrfzMz66zskg2X5urPzMy645W7ZmYt48RvZtYylSR+Se+XtE7SWkkrJO1RxTjMzNooW8mGITuUDgRuAw6PiGckXQV8NSIuG+o1LtlQD46pPpoYV9NiqmXJhhFMBl4iaTKwJ/BIReMwM2ud0hN/RGwCPg78gGL17pMRcVPZ4zAza6vSE3+qz3MqcAjwC8AUSe/osJ9LNpiZZVDFVM9vAN+PiMcj4jngWuDVg3dyyQYzszyqOBPyA+BYSXsCz1Dcg7dnuBe4ZEM9OKb6aGJcTYwplyrm+O8ArgHuAu5NY1hW9jjMzNqqkmufIuJ8ito9ZmZWMq/cNTNrGSd+M7OWqapkw1mpXMM6Se+rYgxmZm1VRcmGI4ArKe6+9SzwNeDdEfHgUK9xyYZ6cEz10cS4mhTTxnQVY5NKNvwycEdE/F9EbANuBd5SwTjMzFqpisS/Fjhe0vR0Lf/JwOzBO3nlrplZHqV/L4qI9ZIuBG4CngbWANs77LeMdH3/wXPnlTsfZWbWYJWc3I2ISyPiVRFxAvAT4DtVjMPMrI0qORMiaf+I6JN0MMX8/rHD7e+SDfXgmOqjiXE1MaZcqjoF/hVJ04HngPdExOaKxmFm1jpVlWw4vop+zczMK3fNzFrHid/MrGWyJX5JyyX1SVo7oO1tqUzDDkk7rSYzM7P8spVskHQCsBW4PCKOSG2/DOwA/hX4YEQMewOWfi7ZUA+OqT6aGFdTYto44ArGXCUbsv0rRcQqSXMGta1Pg8nVrZmZjWDCzvG7ZIOZWR4TNvH7ZutmZnlM2MRvZmZ51OJMiEs21INjqo8mxtXEmHLJeTnnCuB2YL6kXklLJP2OpF7gOOC/JN2Yq38zM+ss51U9i4fYdF2uPs3MbGSe4zczaxknfjOzlsm5cnc5cArQN2Dl7j8Ap1Ks3u0DzoyIR0Z6L6/crQfHVB9NjKspMZWxcjfnEf9lwKJBbf8YEa+IiAXADcB5Gfs3M7MOsiX+iFgFPDGobeAS3CmA76VrZlay0r8XSfoo8E7gSeDEYfZbCiwFmDZ9Jl67a2Y2Pko/uRsRH46I2cAVwHuH2c8lG8zMMqjyqp4rgLdW2L+ZWSuVOtUj6dCIeCA9PRXY0M3rXLKhHhxTfTQxribGlEu2xJ9KNiwEZqQyDecDJ0uaT3E550PAu3P1b2ZmnZVdsuHSXP2ZmVl3vHLXzKxlnPjNzFomW8mGITss5vi/PKBpLnBeRHxqqNe4ZEM9OKb6aGJcTYmp1jdbH0pE3A8sSIOaBGzCpZrNzEpT9VTP64DvRsRDFY/DzKw1qk78pwMrOm2QtFRSj6SerVu2dNrFzMzGoLLEL2k34M3A1Z22u2SDmVkeVR7xvxG4KyIeq3AMZmatU+Up8MUMMc0zmEs21INjqo8mxtXEmHKp5Ihf0hTg9cC1VfRvZtZmlRzxR8TTwPQq+jYza7uqr+oxM7OSOfGbmbVMtpINkpYDpwB9EXHEoG0fAD4OzIyIH430Xi7ZUA+OqT6aGFcTYto46CKWXCUbch7xXwYs6jCQ2cAbgB9k7NvMzIaQLfFHxCrgiQ6bPgmcDZRbHc7MzICS5/glnQpsiohvd7GvSzaYmWVQ2oSYpD2Bv6aY5hlRRCwDlkExx59xaGZmrVLmEf/LgEOAb0vaCBwE3CXppSWOwcys9Uo74o+Ie4H9+5+n5H90N1f1uGRDPTim+mhiXE2MKZdsR/ySVgC3A/Ml9UpakqsvMzPrXrYj/ohYPML2Obn6NjOzoXnlrplZy1SyzC3N7z8FbAe2dVpZZmZmeWQr2TBsp6M4sQsu2VAXjqk+mhhXE2JqQskGMzObgKpK/AHcJGm1pKUVjcHMrJWq+l702ojYJGl/4GZJG1Jtn+elD4SlANOmz8S3WzczGx+VHPFHxKb0uw+4Djimwz7LIuLoiDh6r32c9s3MxkvpR/zpfru7RMRT6fEbgL8f7jVeuVsPjqk+mhhXE2PKpYqpnlnAdZL6+/9SRHytgnGYmbVS6Yk/Ir4HvLLsfs3MrODLOc3MWqarxC/pZZJ2T48XSvoLSVPzDs3MzHLo9oj/K8B2SfMobo4yG/jScC+QtFxSn6S1A9r+UdIGSfdIus4fHmZm5euqZIOkuyLiKEkfAn4aEf8s6e6IOHKY15wAbAUuj4gjUtsbgG9ExDZJFwJExF+N1L9LNtSDY6qPJsZV95gGl2uA6ks2PCdpMXAGcENq23W4F3S62XpE3BQR29LTb1LchcvMzErUbeL/Q+A44KMR8X1JhwBfeJF9vwv47xf5HmZmNkpdfS+KiPuAvwCQNA3YOyIuHGunkj4MbAOuGGYfl2wwM8ug26t6VkraR9J+wF3Av0n6xFg6lHQmcArw9hjmBINLNpiZ5dHtmZB9I2KLpD+iOFl7vqR7RtuZpEXA2cCvR8T/dfs6l2yoB8dUH02Mq4kx5dLtHP9kSQcAp/Hzk7vDGuJm658G9qaoyLlG0mfHMmgzMxu7bo/4/x64EbgtIu6UNBd4YLgXDHGz9UtHOT4zMxtn3Z7cvRq4esDz7wFvzTUoMzPLp6vEL2kPYAnwK8Ae/e0R8a5M4zIzs0y6neP/AvBS4DeBWykWXj01lg4lzZZ0i6T7JK2TdNZY3sfMzMam25INd0fEkZLuiYhXSNoV+N+IOHbUHRYniQ+IiLsk7Q2sBn47rRXoyCUb6sEx1UcT46p7TBOyZEP6vVnSEcC+wP5jGUhEPBoRd6XHTwHrgQPH8l5mZjZ63X48Lksrdv8WuB7YCzjvxXYuaQ5wJHDHi30vMzPrTrdX9VySHt4KzB2PjiXtRVHu+X0RsaXDdpdsMDPLYNjEL+kvh9seEWMt27ArRdK/IiKuHeK9l1HU/ufgufNGPhFhZmZdGemIf+/0OwAN2jamZKziLuuXAuu7/eBwyYZ6cEz10cS4mhhTLsMm/oj4CICkzwNnRcTm9HwacNEY+3wN8AfAvZLWpLa/joivjvH9zMxsFLo9ufuK/qQPEBE/kTTk3beGExG3sfO3BzMzK0m3l3Puko7yAUjlmet7wayZWYt1m7wvAm6X1F+v523AR/MMyczMcur2cs7LJfUAJ6Wmtwy30nY4qe7PKmD31P81EXH+WN7LzMxGr6uSDePaYXFVz5SI2Jou67yN4sTxN4d6jUs21INjqo8mxlXnmDqVa4B8JRtK/1dKt1vcmp7umn58nb6ZWUm6Pbk7riRNSpdy9gE3R8ROJRskLZXUI6ln65adFvaamdkYVZL4I2J7RCygKO98TCr8Nngf32zdzCyDShJ/v7Q24BZgUZXjMDNrk9Ln+CXNBJ6LiM2SXgK8HrhwuNe4ZEM9OKb6aGJcTYwplypOgR8AfF7SJIpvHFdFxA0VjMPMrJWquKrnHooa/GZmVoFK5/jNzKx8TvxmZi1T1XX8UyVdI2mDpPWSjqtiHGZmbVR6yQZ4vr7//0bEJZJ2A/YcWPZ5MJdsqAfHVB9NjKuOMQ1VqqFfY0o2SNoXOAE4EyAingWeLXscZmZtVcVUzyHA48DnJN0t6RJJUwbv5JINZmZ5VJH4JwNHAZ+JiCOBp4FzBu/kkg1mZnlUkfh7gd4BhdmuofggMDOzElSxgOuHkh6WND8i7gdeBwx7UxeXbKgHx1QfTYyriTHlUtUp8D8HrkhX9HwP+MOKxmFm1jqVJP6IWAPsdImRmZnl55W7ZmYt48RvZtYyVa3cXQ6cAvRFxE533xrMK3frwTHVRxPjqltMI63ahXwrd6s64r8M33XLzKwSVd1zdxXwRBV9m5m13YSd43fJBjOzPCZs4nfJBjOzPCZs4jczszxqcQrcJRvqwTHVRxPjamJMuVR1B64VwO3AfEm9kpZUMQ4zszaqqmTD4ir6NTMzz/GbmbWOE7+ZWctUUrIBQNIkoAfYFBGnDLevSzbUg2OqjybGVYeYuinTMFDTSjYAnAWsr7B/M7NWquqqnoOANwGXVNG/mVmbVXXE/yngbGDHUDu4ZIOZWR6lJ35J/eWYVw+3n0s2mJnlUcUR/2uAN0vaCFwJnCTpixWMw8yslUo/BR4R5wLnAkhaCHwwIt4x3GtcsqEeHFN9NDGuJsaUi6/jNzNrmUoveo2IlcDKKsdgZtY2PuI3M2sZJ34zs5bJVrJB0nKg/9LNI1Lbl4H5aZepwOaIWDDSe7lkQz04pvpoYlwTJabRlmUYTq6SDTn/lS4DPg1c3t8QEb83YEAXAU9m7N/MzDrIlvgjYpWkOZ22SRJwGnBSrv7NzKyzqub4jwcei4gHhtrBJRvMzPKoKvEvBlYMt4NLNpiZ5VH6mRBJk4G3AK8qu28zM6tmAddvABsiorfbF7hkQz04pvpoYlxNjCmXbFM9klYAtwPzJfVKWpI2nc4I0zxmZpZPzqt6Fg/RfmauPs3MbGReuWtm1jI5p3qWS+qTtHZQ+59L2iBpnaSP5erfzMw6y1my4QRgK3D5gJINJwIfBt4UET+TtH9E9I30Xi7ZUA+OqT6aGFfumMazFEO3cpVsyHbEHxGrgCcGNf8pcEFE/CztM2LSNzOz8VX2HP9hwPGS7pB0q6RfLbl/M7PWK/u73mRgP+BY4FeBqyTNjQ7zTZKWAksBpk2fidfumpmNj7KP+HuBa6PwLWAHMKPTji7ZYGaWR9lH/P8OnAjcIukwYDfgRyO9yCt368Ex1UcT42piTLlkS/xp5e5CYIakXuB8YDmwPF3i+SxwRqdpHjMzy6f0lbvAO3L1aWZmI/PKXTOzlnHiNzNrmUoSv6RFku6X9KCkc6oYg5lZW1VxI5ZJwL8Ar6e4vPNOSddHxH1DveaZ57Yz55z/KmuIpfjAy7dxpmOa8JoYE9QnrirKJLRBFUf8xwAPRsT3IuJZ4Erg1ArGYWbWSlUk/gOBhwc8701tZmZWggl7clfSUkk9knq2btlS9XDMzBqjisS/CZg94PlBqe0FXLLBzCyPKgpy3wkcKukQioR/OvD7w73AJRvqwTHVR1Pjsu6UnvgjYpuk9wI3ApOA5RGxruxxmJm1VSW34ImIrwJfraJvM7O2m7And83MLA8nfjOzlsmW+CUtl9SXSjD3ty2Q9E1Ja9Klmsfk6t/MzDrLOcd/GfBp4PIBbR8DPhIR/y3p5PR84Uhv5JIN9eCY6mOixOWSDNXIdsQfEauAJwY3w/O3z90XeCRX/2Zm1lnZV/W8D7hR0scpPnRePdSOvtm6mVkeZZ/c/VPg/RExG3g/cOlQO3rlrplZHmUn/jOAa9PjqykqdZqZWYnKnup5BPh1YCVwEvBANy9yyYZ6cEz10dS4rDvZEr+kFRRX7MyQ1AucD/wxcLGkycBPSXP4ZmZWnmyJPyIWD7HpVbn6NDOzkXnlrplZyzjxm5m1TNklG14p6XZJ90r6T0m+TtPMrGRll2y4BPhgRNwq6V3Ah4C/HemNXLKhHhxTNVz2wEar7JINhwGr0uObgbfm6t/MzDore45/HXBqevw2Xnjv3RfwzdbNzPIoO/G/C/gzSauBvYFnh9rRJRvMzPIodeVuRGwA3gAg6TDAk5NmZiUrNfFL2j8i+iTtAvwN8NluXueSDfXgmMzqIeflnCuA24H5knolLQEWS/oOsIGibs/ncvVvZmadVVGy4eJcfZqZ2cgUEVWPYUSSngLur3oc42wG8KOqBzHOHFN9NDEux7SzX4yImYMbyy7LPFb3R8TRVQ9iPEnqcUwTXxNjgmbG5Zi651o9ZmYt48RvZtYydUn8y6oeQAaOqR6aGBM0My7H1KVanNw1M7PxU5cjfjMzGycTOvFLWiTpfkkPSjqn6vGMxhD3I9hP0s2SHki/p6V2SfqnFOc9ko6qbuRDkzRb0i2S7pO0TtJZqb22cUnaQ9K3JH07xfSR1H6IpDvS2L8sabfUvnt6/mDaPqfK8Q9H0iRJd0u6IT2vdUySNqZ7eayR1JPaavu3ByBpqqRrJG2QtF7ScWXENGETv6RJwL8AbwQOp1j1e3i1oxqVy4BFg9rOAb4eEYcCX0/PoYjx0PSzFPhMSWMcrW3AByLicOBY4D3pv0md4/oZcFJEvBJYACySdCxwIfDJiJgH/ARYkvZfAvwktX8y7TdRnQWsH/C8CTGdGBELBlziWOe/PSgWtH4tIn4JeCXFf6/8MUXEhPwBjgNuHPD8XODcqsc1yhjmAGsHPL8fOCA9PoBifQLAvwKLO+03kX+A/wBe35S4gD2Bu4Bfo1g0Mzm1P/+3CNwIHJceT077qeqxd4jloJQ0TgJuANSAmDYCMwa11fZvD9gX+P7gf+syYpqwR/zAgcDDA573prY6mxURj6bHPwRmpce1izVNBxwJ3EHN40pTImuAPoobBH0X2BwR29IuA8f9fExp+5PA9HJH3JVPAWcDO9Lz6dQ/pgBukrRa0tLUVue/vUOAx4HPpSm5SyRNoYSYJnLib7QoPrJreUmVpL2ArwDvi4gX3CWnjnFFxPaIWEBxlHwM8EsVD+lFkXQK0BcRq6seyzh7bUQcRTHl8R5JJwzcWMO/vcnAUcBnIuJI4Gl+Pq0D5ItpIif+TbzwDl0HpbY6e0zSAQDpd19qr02sknalSPpXRMS1qbn2cQFExGbgFoppkKmS+kuaDBz38zGl7fsCPy55qCN5DfBmSRuBKymmey6m3jEREZvS7z7gOooP6Tr/7fUCvRFxR3p+DcUHQfaYJnLivxM4NF2JsBtwOnB9xWN6sa4HzkiPz6CYI+9vf2c6a38s8OSAr3oThiQBlwLrI+ITAzbVNi5JMyVNTY9fQnHOYj3FB8Dvpt0Gx9Qf6+8C30hHZRNGRJwbEQdFxByK/2++ERFvp8YxSZoiae/+xxQ3dFpLjf/2IuKHwMOS5qem1wH3UUZMVZ/gGOHkx8nAdyjmXD9c9XhGOfYVwKPAcxSf7Eso5k2/DjwA/A+wX9pXFFcwfRe4Fzi66vEPEdNrKb523gOsST8n1zku4BXA3SmmtcB5qX0u8C3gQeBqYPfUvkd6/mDaPrfqGEaIbyFwQ91jSmP/dvpZ158P6vy3l8a5AOhJf3//DkwrIyav3DUza5mJPNVjZmYZOPGbmbWME7+ZWcs48ZuZtYwTv5lZyzjxm5m1jBO/mVnLOPGbmbXM/wNMqIYj9QZvcAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VkCVhMFvPiFV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "9d4a5029-f636-4d04-ff4b-33bd03ca836b"
      },
      "source": [
        "news_df['desc'][0]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'I was wondering if anyone out there could enlighten me on this car I saw\\nthe other day. It was a 2-door sports car, looked to be from the late 60s/\\nearly 70s. It was called a Bricklin. The doors were really small. In addition,\\nthe front bumper was separate from the rest of the body. This is \\nall I know. If anyone can tellme a model name, engine specs, years\\nof production, where this car is made, history, or whatever info you\\nhave on this funky looking car, please e-mail.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AEGLMBnFPRIm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "ae60a0e5-d642-4511-c028-2c0f3b8c0973"
      },
      "source": [
        "# Dictionary of English Contractions\n",
        "contractions_dict = { \"ain't\": \"are not\",\"'s\":\" is\",\"aren't\": \"are not\",\n",
        "                     \"can't\": \"cannot\",\"can't've\": \"cannot have\",\n",
        "                     \"'cause\": \"because\",\"could've\": \"could have\",\"couldn't\": \"could not\",\n",
        "                     \"couldn't've\": \"could not have\", \"didn't\": \"did not\",\"doesn't\": \"does not\",\n",
        "                     \"don't\": \"do not\",\"hadn't\": \"had not\",\"hadn't've\": \"had not have\",\n",
        "                     \"hasn't\": \"has not\",\"haven't\": \"have not\",\"he'd\": \"he would\",\n",
        "                     \"he'd've\": \"he would have\",\"he'll\": \"he will\", \"he'll've\": \"he will have\",\n",
        "                     \"how'd\": \"how did\",\"how'd'y\": \"how do you\",\"how'll\": \"how will\",\n",
        "                     \"I'd\": \"I would\", \"I'd've\": \"I would have\",\"I'll\": \"I will\",\n",
        "                     \"I'll've\": \"I will have\",\"I'm\": \"I am\",\"I've\": \"I have\", \"isn't\": \"is not\",\n",
        "                     \"it'd\": \"it would\",\"it'd've\": \"it would have\",\"it'll\": \"it will\",\n",
        "                     \"it'll've\": \"it will have\", \"let's\": \"let us\",\"ma'am\": \"madam\",\n",
        "                     \"mayn't\": \"may not\",\"might've\": \"might have\",\"mightn't\": \"might not\", \n",
        "                     \"mightn't've\": \"might not have\",\"must've\": \"must have\",\"mustn't\": \"must not\",\n",
        "                     \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n",
        "                     \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\"oughtn't\": \"ought not\",\n",
        "                     \"oughtn't've\": \"ought not have\",\"shan't\": \"shall not\",\"sha'n't\": \"shall not\",\n",
        "                     \"shan't've\": \"shall not have\",\"she'd\": \"she would\",\"she'd've\": \"she would have\",\n",
        "                     \"she'll\": \"she will\", \"she'll've\": \"she will have\",\"should've\": \"should have\",\n",
        "                     \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\"so've\": \"so have\",\n",
        "                     \"that'd\": \"that would\",\"that'd've\": \"that would have\", \"there'd\": \"there would\",\n",
        "                     \"there'd've\": \"there would have\", \"they'd\": \"they would\",\n",
        "                     \"they'd've\": \"they would have\",\"they'll\": \"they will\",\n",
        "                     \"they'll've\": \"they will have\", \"they're\": \"they are\",\"they've\": \"they have\",\n",
        "                     \"to've\": \"to have\",\"wasn't\": \"was not\",\"we'd\": \"we would\",\n",
        "                     \"we'd've\": \"we would have\",\"we'll\": \"we will\",\"we'll've\": \"we will have\",\n",
        "                     \"we're\": \"we are\",\"we've\": \"we have\", \"weren't\": \"were not\",\"what'll\": \"what will\",\n",
        "                     \"what'll've\": \"what will have\",\"what're\": \"what are\", \"what've\": \"what have\",\n",
        "                     \"when've\": \"when have\",\"where'd\": \"where did\", \"where've\": \"where have\",\n",
        "                     \"who'll\": \"who will\",\"who'll've\": \"who will have\",\"who've\": \"who have\",\n",
        "                     \"why've\": \"why have\",\"will've\": \"will have\",\"won't\": \"will not\",\n",
        "                     \"won't've\": \"will not have\", \"would've\": \"would have\",\"wouldn't\": \"would not\",\n",
        "                     \"wouldn't've\": \"would not have\",\"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n",
        "                     \"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\n",
        "                     \"y'all've\": \"you all have\", \"you'd\": \"you would\",\"you'd've\": \"you would have\",\n",
        "                     \"you'll\": \"you will\",\"you'll've\": \"you will have\", \"you're\": \"you are\",\n",
        "                     \"you've\": \"you have\"}\n",
        "# Regular expression for finding contractions\n",
        "contractions_re=re.compile('(%s)' % '|'.join(contractions_dict.keys()))\n",
        "\n",
        "# Function for expanding contractions\n",
        "def expand_contractions(text,contractions_dict=contractions_dict):\n",
        "  def replace(match):\n",
        "    return contractions_dict[match.group(0)]\n",
        "  return contractions_re.sub(replace, text)\n",
        "\n",
        "# Expanding Contractions in the reviews\n",
        "news_df['desc']=news_df['desc'].apply(lambda x:expand_contractions(x))\n",
        "news_df['desc'][0]"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'I was wondering if anyone out there could enlighten me on this car I saw\\nthe other day. It was a 2-door sports car, looked to be from the late 60s/\\nearly 70s. It was called a Bricklin. The doors were really small. In addition,\\nthe front bumper was separate from the rest of the body. This is \\nall I know. If anyone can tellme a model name, engine specs, years\\nof production, where this car is made, history, or whatever info you\\nhave on this funky looking car, please e-mail.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-cKtRK5OrVZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "9179db69-58b0-4028-a8b9-2ac254f3316e"
      },
      "source": [
        "## Remove '\\n' from the News description\n",
        "news_df['cleaned_desc']=news_df['desc'].apply(lambda x: x.replace('\\n',' '))\n",
        "## Converting every text to lowercase\n",
        "news_df['cleaned_desc']=news_df['cleaned_desc'].apply(lambda x: x.lower())\n",
        "\n",
        "## Removing digits and word containing digits\n",
        "news_df['cleaned_desc']=news_df['cleaned_desc'].apply(lambda x: re.sub('\\w*\\d\\w*',' ', x))\n",
        "\n",
        "## Removing Punctuation\n",
        "news_df['cleaned_desc']=news_df['cleaned_desc'].apply(lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x))\n",
        "\n",
        "## Removing multiple white spaces\n",
        "news_df['cleaned_desc']=news_df['cleaned_desc'].apply(lambda x: re.sub(' +',' ',x))\n",
        "\n",
        "news_df['cleaned_desc'][4]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'from article world std com by tombaker world std com tom a baker my understanding is that the expected errors are basically known bugs in the warning system software things are checked that do not have the right values in yet because they are not set till after launch and suchlike rather than fix the code and possibly introduce new bugs they just tell the crew ok if you see a warning no before liftoff ignore it '"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rfNPRbVLR1QV"
      },
      "source": [
        "# Importing spacy\n",
        "import spacy\n",
        "\n",
        "# Loading model\n",
        "nlp = spacy.load('en_core_web_sm',disable=['parser', 'ner'])\n",
        "\n",
        "# Lemmatization with stopwords removal\n",
        "news_df['lemmatized_desc']=news_df['cleaned_desc'].apply(lambda x: ' '.join([token.lemma_ for token in list(nlp(x)) if (token.is_stop==False)]))"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "uFdFk1JTj_66",
        "outputId": "207eac56-2323-4319-f52b-cd04889c6e13"
      },
      "source": [
        "news_df['lemmatized_desc'].iloc[4]"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'article world std com tombaker world std com tom baker understanding expect error basically know bug warning system software thing check right value set till launch suchlike fix code possibly introduce new bug tell crew ok warning liftoff ignore'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1O4E6BuaAZqT"
      },
      "source": [
        "train_df, test_df = train_test_split(news_df, test_size=0.1, random_state=42, stratify=news_df[\"class\"])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "01j7i6m5KBHk"
      },
      "source": [
        "news_sampled = train_df.sample(2000)\n",
        "news_sampled.reset_index(drop=True, inplace=True)\n",
        "\n",
        "news_sampled_test = test_df.sample(400)\n",
        "news_sampled_test.reset_index(drop=True, inplace=True)\n",
        "\n",
        "train_data = news_sampled[\"lemmatized_desc\"]\n",
        "train_target = news_sampled[\"class\"]\n",
        "test_data = news_sampled_test[\"lemmatized_desc\"]\n",
        "test_target = news_sampled_test[\"class\"]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RZTElMwYR7rR"
      },
      "source": [
        "train_df, test_df = train_test_split(news_df, test_size=0.1, random_state=42, stratify=news_df[\"class\"])\n",
        "train_data = train_df[\"lemmatized_desc\"]\n",
        "train_target = train_df[\"class\"]\n",
        "test_data = test_df[\"lemmatized_desc\"]\n",
        "test_target = test_df[\"class\"]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uKMmqTx8fgz3"
      },
      "source": [
        "mult_nb = Pipeline([(\"count_vectorizer\", CountVectorizer(analyzer=lambda x: x)), (\"multinomial nb\", MultinomialNB())])\n",
        "bern_nb = Pipeline([(\"count_vectorizer\", CountVectorizer(analyzer=lambda x: x)), (\"bernoulli nb\", BernoulliNB())])\n",
        "mult_nb_tfidf = Pipeline([(\"tfidf_vectorizer\", TfidfVectorizer(analyzer=lambda x: x)), (\"multinomial nb\", MultinomialNB())])\n",
        "bern_nb_tfidf = Pipeline([(\"tfidf_vectorizer\", TfidfVectorizer(analyzer=lambda x: x)), (\"bernoulli nb\", BernoulliNB())])\n",
        "svc = Pipeline([(\"count_vectorizer\", CountVectorizer(analyzer=lambda x: x)), (\"linear svc\", SVC(kernel=\"linear\"))])\n",
        "svc_tfidf = Pipeline([(\"tfidf_vectorizer\", TfidfVectorizer(analyzer=lambda x: x)), (\"linear svc\", SVC(kernel=\"linear\"))])"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gY_Kizf3f5Op"
      },
      "source": [
        "cv = StratifiedShuffleSplit( test_size=0.2, random_state=42)\n",
        "##BenchMark\n",
        "\n",
        "all_models = [\n",
        "    (\"mult_nb\", mult_nb),\n",
        "    (\"mult_nb_tfidf\", mult_nb_tfidf),\n",
        "    (\"bern_nb\", bern_nb),\n",
        "    (\"bern_nb_tfidf\", bern_nb_tfidf),\n",
        "    (\"svc\", svc),\n",
        "    (\"svc_tfidf\", svc_tfidf),\n",
        "#    (\"w2v\", etree_w2v),\n",
        "#    (\"w2v_tfidf\", etree_w2v_tfidf),\n",
        "]\n",
        "scores = sorted([(name, cross_val_score(model, train_data, train_target, scoring=\"accuracy\", cv=cv).mean()) for name, model in all_models])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ve4pgWSFiKqE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be93f051-89d7-4c76-e9fd-890441ddc025"
      },
      "source": [
        "print (tabulate(scores, floatfmt=\".4f\", headers=(\"model\", 'accuracy')))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "model            accuracy\n",
            "-------------  ----------\n",
            "bern_nb            0.0975\n",
            "bern_nb_tfidf      0.0975\n",
            "mult_nb            0.1705\n",
            "mult_nb_tfidf      0.0842\n",
            "svc                0.2075\n",
            "svc_tfidf          0.1155\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MM84zOlbjPOu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9cb064ac-987f-4796-8c16-5d49a3feb68e"
      },
      "source": [
        "from __future__ import print_function\n",
        "\n",
        "from pprint import pprint\n",
        "from time import time\n",
        "import logging\n",
        "\n",
        "from sklearn.datasets import fetch_20newsgroups\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "print(__doc__)\n",
        "\n",
        "# Display progress logs on stdout\n",
        "logging.basicConfig(level=logging.INFO,\n",
        "                    format='%(asctime)s %(levelname)s %(message)s')\n",
        "pipeline = Pipeline([\n",
        "    ('vect', CountVectorizer()),\n",
        "    ('tfidf', TfidfTransformer()),\n",
        "    ('clf', SGDClassifier()),\n",
        "])\n",
        "\n",
        "# uncommenting more parameters will give better exploring power but will\n",
        "# increase processing time in a combinatorial way\n",
        "parameters = {\n",
        "    'vect__max_df': (0.5, 0.75, 1.0),\n",
        "    'vect__max_features': (None, 5000, 10000, 50000),\n",
        "    'vect__ngram_range': ((1, 1), (1, 2)),  # unigrams or bigrams\n",
        "    #'tfidf__use_idf': (True, False),\n",
        "    #'tfidf__norm': ('l1', 'l2'),\n",
        "    'clf__alpha': (0.00001, 0.000001),\n",
        "    'clf__penalty': ('l2', 'elasticnet'),\n",
        "    #'clf__n_iter': (10, 50, 80),\n",
        "}\n",
        "if __name__ == \"__main__\":\n",
        "    # multiprocessing requires the fork to happen in a __main__ protected\n",
        "    # block\n",
        "\n",
        "    # find the best parameters for both the feature extraction and the\n",
        "    # classifier\n",
        "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
        "\n",
        "    print(\"Performing grid search...\")\n",
        "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
        "    print(\"parameters:\")\n",
        "    pprint(parameters)\n",
        "    t0 = time()\n",
        "    grid_search.fit(train_data, train_target)\n",
        "    print(\"done in %0.3fs\" % (time() - t0))\n",
        "    print()\n",
        "\n",
        "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
        "    print(\"Best parameters set:\")\n",
        "    best_parameters = grid_search.best_estimator_.get_params()\n",
        "    for param_name in sorted(parameters.keys()):\n",
        "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Automatically created module for IPython interactive environment\n",
            "Performing grid search...\n",
            "pipeline: ['vect', 'tfidf', 'clf']\n",
            "parameters:\n",
            "{'clf__alpha': (1e-05, 1e-06),\n",
            " 'clf__penalty': ('l2', 'elasticnet'),\n",
            " 'vect__max_df': (0.5, 0.75, 1.0),\n",
            " 'vect__max_features': (None, 5000, 10000, 50000),\n",
            " 'vect__ngram_range': ((1, 1), (1, 2))}\n",
            "Fitting 5 folds for each of 96 candidates, totalling 480 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  46 tasks      | elapsed:   17.7s\n",
            "[Parallel(n_jobs=-1)]: Done 196 tasks      | elapsed:  1.3min\n",
            "[Parallel(n_jobs=-1)]: Done 446 tasks      | elapsed:  3.2min\n",
            "[Parallel(n_jobs=-1)]: Done 480 out of 480 | elapsed:  3.4min finished\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "done in 207.832s\n",
            "\n",
            "Best score: 0.609\n",
            "Best parameters set:\n",
            "\tclf__alpha: 1e-05\n",
            "\tclf__penalty: 'elasticnet'\n",
            "\tvect__max_df: 0.5\n",
            "\tvect__max_features: None\n",
            "\tvect__ngram_range: (1, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uw7K0FHlkRiI"
      },
      "source": [
        "type(news_group.data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SWFijwHsA0Sw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7f1448bd-747b-4046-c3aa-d1514c08b17a"
      },
      "source": [
        "import spacy.cli\n",
        "spacy.cli.download(\"en_core_web_md\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_md')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CrASzXLNk5Uy"
      },
      "source": [
        "import spacy \n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "nlp = spacy.load(\"en_core_web_md\")  # this model will give you 300D\n",
        "\n",
        "class SpacyVectorTransformer(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self, nlp):\n",
        "        self.nlp = nlp\n",
        "        self.dim = 300\n",
        "\n",
        "    def fit(self, X, y):\n",
        "        return self\n",
        "\n",
        "    def transform(self, X):\n",
        "        # Doc.vector defaults to an average of the token vectors.\n",
        "        # https://spacy.io/api/doc#vector\n",
        "        return [self.nlp(text).vector for text in X]"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VsZQQ7068r8h"
      },
      "source": [
        "embeddings_pipeline = Pipeline(\n",
        "    steps=[\n",
        "        (\"mean_embeddings\", SpacyVectorTransformer(nlp)),\n",
        "        (\"reduce_dim\", TruncatedSVD(50)),\n",
        "        (\"classifier\", RandomForestClassifier()),\n",
        "    ])\n",
        "embeddings_pipeline.fit(train_data, train_target)\n",
        "y_pred = embeddings_pipeline.predict(test_data)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CvzPs1J98v9c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d0fa4e4a-2e11-4766-92e5-272a22873e78"
      },
      "source": [
        "print('Classification report:\\n\\n{}'.format(\n",
        "    classification_report(test_target, y_pred))\n",
        ")"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Classification report:\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.42      0.31      0.36        16\n",
            "           1       0.47      0.50      0.48        16\n",
            "           2       0.44      0.44      0.44        18\n",
            "           3       0.44      0.40      0.42        20\n",
            "           4       0.29      0.47      0.36        19\n",
            "           5       0.59      0.45      0.51        22\n",
            "           6       0.44      0.64      0.52        22\n",
            "           7       0.71      0.59      0.65        17\n",
            "           8       0.50      0.47      0.48        15\n",
            "           9       0.67      0.61      0.64        23\n",
            "          10       0.78      0.78      0.78        23\n",
            "          11       0.65      0.52      0.58        21\n",
            "          12       0.92      0.48      0.63        25\n",
            "          13       0.68      0.68      0.68        22\n",
            "          14       0.70      0.68      0.69        28\n",
            "          15       0.52      0.81      0.63        21\n",
            "          16       0.41      0.61      0.49        18\n",
            "          17       0.77      0.74      0.76        23\n",
            "          18       0.50      0.28      0.36        18\n",
            "          19       0.25      0.23      0.24        13\n",
            "\n",
            "    accuracy                           0.55       400\n",
            "   macro avg       0.56      0.54      0.54       400\n",
            "weighted avg       0.58      0.55      0.55       400\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KFSxw7gjFKsM"
      },
      "source": [
        "###Part of Speech(POS)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CVN9yz4qEl6r"
      },
      "source": [
        "class Cleaner():\n",
        "\n",
        "    def __init__(self):\n",
        "        self.stop_words = stopwords.words('english')\n",
        "\n",
        "    def clean_news(self, text):\n",
        "        text = re.sub(r'(From:\\s+[^\\n]+\\n)', '', text) # remove From\n",
        "        text = re.sub(r'(Subject:)', '', text) # remove the word \"Subject:\"\"\n",
        "        text = text.lower() # Convert to lowerCase\n",
        "        text = text.strip() # Strip terminal spaces\n",
        "        text = re.sub(self.re_url, '', text)\n",
        "        text = re.sub(self.re_email, '', text)       \n",
        "        text = re.sub(r'\\s+\\w{1}\\s+', ' ', text) #remove single char\n",
        "        #text = text.replace('\\n',' ')\n",
        "        text = re.sub(f'[{re.escape(string.punctuation)}]', '', text) # punctuations\n",
        "        text = re.sub(r'^\\d+\\s|\\s\\d+\\s|\\s\\d+$', ' ', text) # remove pure digits\n",
        "        text = re.sub(r'(\\s+)', ' ', text) # replace >1 whitespaces with single space\n",
        "\n",
        "        return text\n",
        "\n",
        "    def removeStopWords(self, text):\n",
        "        \n",
        "        x = text.split(' ')\n",
        "        for word in x:\n",
        "            if(word in self.stop_words):\n",
        "                x = list(filter((word).__ne__, x))\n",
        "        return ' '.join(x)\n",
        "\n",
        "    def fit(self, x, y=None):\n",
        "        return self\n",
        "    def transform(self, data):\n",
        "        cleaner = Cleaner()\n",
        "        \n",
        "        data_array = []\n",
        "        for d in data:\n",
        "            #s = cleaner.clean_news(d)\n",
        "            w = cleaner.removeStopWords(d)\n",
        "            data_array.append(w)\n",
        "        return data_array "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xplqt7PWGFbK"
      },
      "source": [
        "###BOW Vectorizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B2VpZbpFF5oV"
      },
      "source": [
        "class BOWVectorizer():\n",
        "    def __init__(self):\n",
        "        self.vectorize = None\n",
        "    def fit(self, x, y=None):\n",
        "        bowvec = TfidfVectorizer()\n",
        "        bowvec.fit(x)\n",
        "        self.vectorize = bowvec\n",
        "        return self.vectorize\n",
        "    \n",
        "    def transform(self, data):\n",
        "        x = self.vectorize.transform(data)\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WP9sE5Q5GJy3"
      },
      "source": [
        "class POSVectorizer():\n",
        "    def __init__(self):\n",
        "        pass\n",
        "\n",
        "    def creatingPOSTags(self, x):\n",
        "             \n",
        "        pos_family = {'NOUN' : ['NN','NNS','NNP','NNPS'],\n",
        "                    'PRON' : ['PRP','PRP$','WP','WP$'], \n",
        "                    'VERB' : ['VB','VBD','VBG','VBN','VBP','VBZ'],\n",
        "                    'ADJ'  : ['JJ','JJR','JJS'],\n",
        "                    'ADV'  : ['RB','RBR','RBS','WRB']\n",
        "                    }\n",
        "            \n",
        "        count_pos = {'NOUN':0,'PRON':0,'VERB':0,'ADJ':0,'ADV':0}\n",
        "        \n",
        "        blob  = TextBlob(x) #converts sentences to tokens\n",
        "        for tuple in blob.tags: #blob tags contains term and its pos\n",
        "            #print(tuple)\n",
        "            pos = list(tuple)[1]\n",
        "            if pos in pos_family['NOUN']:\n",
        "                count_pos['NOUN'] = count_pos.get('NOUN')+1\n",
        "            elif pos in pos_family['PRON']:\n",
        "                count_pos['PRON'] = count_pos.get('PRON')+1\n",
        "            elif pos in pos_family['VERB']:\n",
        "                count_pos['VERB'] = count_pos.get('VERB')+1\n",
        "            elif pos in pos_family['ADJ']:\n",
        "                count_pos['ADJ'] = count_pos.get('ADJ')+1\n",
        "            elif pos in pos_family['ADV']:\n",
        "                count_pos['ADV'] = count_pos.get('ADV')+1\n",
        "        return count_pos \n",
        "    \n",
        "    def fit(self, x, y=None):\n",
        "        return self\n",
        "    def transform(self, data):\n",
        "        posVector = POSVectorizer()\n",
        "        pos_vect = []\n",
        "        for d in data:\n",
        "            pos_vect.append(posVector.creatingPOSTags(d))\n",
        "        return pos_vect"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z13lxV4pGWNE"
      },
      "source": [
        "###Convert to Array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3Ol1v1dGPj_"
      },
      "source": [
        "class ToArray():\n",
        "\n",
        "    def transform(self, X):\n",
        "        return X.toarray()\n",
        "\n",
        "    def fit(self, X, y=None, **fit_params):        \n",
        "        return self"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0IBHeWMmHZO7"
      },
      "source": [
        "from sklearn.pipeline import Pipeline, FeatureUnion\n",
        "from sklearn.feature_selection import SelectKBest, chi2\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.feature_extraction import DictVectorizer\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn import preprocessing\n",
        "from nltk.corpus import stopwords\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S78YreXvGyuU"
      },
      "source": [
        "bow_transformer = Pipeline(\n",
        "    steps=[\n",
        "        (\"cleaner\", Cleaner()),\n",
        "        (\"bow\", BOWVectorizer()),\n",
        "        (\"toarray\", ToArray()), #converting toarray since minmax can't handle sparce matrix\n",
        "        (\"scale\", preprocessing.MinMaxScaler())\n",
        "    ]\n",
        ")\n",
        "\n",
        "\n",
        "pos_transformer = Pipeline(\n",
        "    steps=[\n",
        "        (\"cleaner\", Cleaner()),\n",
        "        (\"pos\", POSVectorizer()),\n",
        "        (\"dict_vect\", DictVectorizer()),\n",
        "        (\"toarray\", ToArray()), #converting toarray since minmax can't handle sparce matrix\n",
        "        (\"scale\", preprocessing.MinMaxScaler())\n",
        "    ]\n",
        ")\n",
        "\n",
        "combined_features = FeatureUnion(\n",
        "    transformer_list=[\n",
        "        (\"bow\", bow_transformer),\n",
        "        (\"pos\", pos_transformer),\n",
        "    ]\n",
        ")\n",
        "\n",
        "def fitFinalPipeline(classifier, X_train, Y_train, X_test, Y_test):\n",
        "    final_pipeline = Pipeline(\n",
        "        steps=[\n",
        "            (\"combined_features\", combined_features),\n",
        "            ('chi',  SelectKBest(chi2, k=100)),\n",
        "            (\"classifier\", classifier),\n",
        "        ]\n",
        "    )\n",
        "\n",
        "    final_pipeline.fit(train_data, train_target)\n",
        "    y_pred = final_pipeline.predict(test_data)\n",
        "    cr = classification_report(test_target, y_pred)\n",
        "    print(cr)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sUejCQGMLHQ3"
      },
      "source": [
        "!pip install -U pip setuptools wheel\n",
        "!pip install -U textstat\n",
        "!pip install textblob\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9dI7k6ZlHTM2"
      },
      "source": [
        "fitFinalPipeline (RandomForestClassifier(), train_data, train_target, test_data, test_target)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9u2-oED0IfgQ"
      },
      "source": [
        "import textstat\n",
        "import string\n",
        "import nltk\n",
        "from textblob import TextBlob"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4eOVgXBRLtoi"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}