Untitled0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "id": "3tzTGJ9VPQMb"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s9B5wybgbItj",
    "outputId": "7aad9d3c-9ea3-4310-c5b1-124dd0010073"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.8.1)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk) (8.1.7)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk) (1.4.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk) (2023.12.25)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk) (4.66.4)\n",
      "/usr/lib/python3.10/runpy.py:126: RuntimeWarning: 'nltk.downloader' found in sys.modules after import of package 'nltk', but prior to execution of 'nltk.downloader'; this may result in unpredictable behaviour\n",
      "  warn(RuntimeWarning(msg))\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk\n",
    "!python -m nltk.downloader punkt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "id": "qdVAebn3bNmQ"
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "PgtljwpMTpG0",
    "outputId": "117d82b6-45ca-454e-aec5-341d9609cf6d"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"df\",\n  \"rows\": 100,\n  \"fields\": [\n    {\n      \"column\": \"URL_ID\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"blackassign0084\",\n          \"blackassign0054\",\n          \"blackassign0071\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"URL\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"https://insights.blackcoffer.com/how-voice-search-makes-your-business-a-successful-business/\",\n          \"https://insights.blackcoffer.com/how-google-fit-measure-heart-and-respiratory-rates-using-a-phone/\",\n          \"https://insights.blackcoffer.com/how-to-overcome-your-fear-of-making-mistakes-2/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe",
       "variable_name": "df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-1d59d910-a54c-4043-a015-c39aa079c57c\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL_ID</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>blackassign0001</td>\n",
       "      <td>https://insights.blackcoffer.com/rising-it-cit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>blackassign0002</td>\n",
       "      <td>https://insights.blackcoffer.com/rising-it-cit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blackassign0003</td>\n",
       "      <td>https://insights.blackcoffer.com/internet-dema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>blackassign0004</td>\n",
       "      <td>https://insights.blackcoffer.com/rise-of-cyber...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>blackassign0005</td>\n",
       "      <td>https://insights.blackcoffer.com/ott-platform-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1d59d910-a54c-4043-a015-c39aa079c57c')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-1d59d910-a54c-4043-a015-c39aa079c57c button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-1d59d910-a54c-4043-a015-c39aa079c57c');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-1d45db88-1c06-4566-8b6b-1a80181197c1\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1d45db88-1c06-4566-8b6b-1a80181197c1')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-1d45db88-1c06-4566-8b6b-1a80181197c1 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "            URL_ID                                                URL\n",
       "0  blackassign0001  https://insights.blackcoffer.com/rising-it-cit...\n",
       "1  blackassign0002  https://insights.blackcoffer.com/rising-it-cit...\n",
       "2  blackassign0003  https://insights.blackcoffer.com/internet-dema...\n",
       "3  blackassign0004  https://insights.blackcoffer.com/rise-of-cyber...\n",
       "4  blackassign0005  https://insights.blackcoffer.com/ott-platform-..."
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel(\"/content/drive/MyDrive/Input.xlsx\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "N6-F-Z-GWK81",
    "outputId": "4198a074-87c9-4a30-aed3-8eceaf409fc6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 2)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "id": "kLoxeaiLWZUd"
   },
   "outputs": [],
   "source": [
    "def extract(url):\n",
    "    page = requests.get(url)\n",
    "    if(page.status_code!=200):\n",
    "        return None,None\n",
    "    soup = BeautifulSoup(page.text,'html')\n",
    "    title = soup.find('h1').text.strip()\n",
    "    content = [i.text.strip() for i in soup.find_all('p',class_=None)]\n",
    "    return title,content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FDtXtUe6WfSl",
    "outputId": "ee3e509f-fce2-4b15-f75f-ff4dbd4ebf30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blackassign0001\n",
      "blackassign0002\n",
      "blackassign0003\n",
      "blackassign0004\n",
      "blackassign0005\n",
      "blackassign0006\n",
      "blackassign0007\n",
      "blackassign0008\n",
      "blackassign0009\n",
      "blackassign0010\n",
      "blackassign0011\n",
      "blackassign0012\n",
      "blackassign0013\n",
      "blackassign0014\n",
      "blackassign0015\n",
      "blackassign0016\n",
      "blackassign0017\n",
      "blackassign0018\n",
      "blackassign0019\n",
      "blackassign0020\n",
      "blackassign0021\n",
      "blackassign0022\n",
      "blackassign0023\n",
      "blackassign0024\n",
      "blackassign0025\n",
      "blackassign0026\n",
      "blackassign0027\n",
      "blackassign0028\n",
      "blackassign0029\n",
      "blackassign0030\n",
      "blackassign0031\n",
      "blackassign0032\n",
      "blackassign0033\n",
      "blackassign0034\n",
      "blackassign0035\n",
      "blackassign0036\n",
      "blackassign0036 is not reachable\n",
      "blackassign0037\n",
      "blackassign0038\n",
      "blackassign0039\n",
      "blackassign0040\n",
      "blackassign0041\n",
      "blackassign0042\n",
      "blackassign0043\n",
      "blackassign0044\n",
      "blackassign0045\n",
      "blackassign0046\n",
      "blackassign0047\n",
      "blackassign0048\n",
      "blackassign0049\n",
      "blackassign0049 is not reachable\n",
      "blackassign0050\n",
      "blackassign0051\n",
      "blackassign0052\n",
      "blackassign0053\n",
      "blackassign0054\n",
      "blackassign0055\n",
      "blackassign0056\n",
      "blackassign0057\n",
      "blackassign0058\n",
      "blackassign0059\n",
      "blackassign0060\n",
      "blackassign0061\n",
      "blackassign0062\n",
      "blackassign0063\n",
      "blackassign0064\n",
      "blackassign0065\n",
      "blackassign0066\n",
      "blackassign0067\n",
      "blackassign0068\n",
      "blackassign0069\n",
      "blackassign0070\n",
      "blackassign0071\n",
      "blackassign0072\n",
      "blackassign0073\n",
      "blackassign0074\n",
      "blackassign0075\n",
      "blackassign0076\n",
      "blackassign0077\n",
      "blackassign0078\n",
      "blackassign0079\n",
      "blackassign0080\n",
      "blackassign0081\n",
      "blackassign0082\n",
      "blackassign0083\n",
      "blackassign0084\n",
      "blackassign0085\n",
      "blackassign0086\n",
      "blackassign0087\n",
      "blackassign0088\n",
      "blackassign0089\n",
      "blackassign0090\n",
      "blackassign0091\n",
      "blackassign0092\n",
      "blackassign0093\n",
      "blackassign0094\n",
      "blackassign0095\n",
      "blackassign0096\n",
      "blackassign0097\n",
      "blackassign0098\n",
      "blackassign0099\n",
      "blackassign0100\n"
     ]
    }
   ],
   "source": [
    "for index,row in df.iterrows():\n",
    "    url = row[\"URL\"]\n",
    "    j = row['URL_ID']\n",
    "    print(j)\n",
    "    title,text = extract(url)\n",
    "    if(title==None):\n",
    "        print(j,\"is not reachable\")\n",
    "        continue\n",
    "    file = f\"{j}.txt\"\n",
    "    f = open(file,'w+',encoding='utf-8')\n",
    "    f.write(title+'\\n')\n",
    "    for i in text:\n",
    "        f.write(i+'\\n')\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "id": "a8sfw7c7WleB"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "id": "D1ewiiP-XBCu"
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import RegexpTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 582
    },
    "id": "fFBuVTrvXJsB",
    "outputId": "94f35af2-eebc-4259-c8be-3ac40eb50d01"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "repr_error": "Out of range float values are not JSON compliant: nan",
       "type": "dataframe",
       "variable_name": "df_output"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-6638bb3b-0963-453e-9151-690bb4dd8ec6\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL_ID</th>\n",
       "      <th>URL</th>\n",
       "      <th>POSITIVE SCORE</th>\n",
       "      <th>NEGATIVE SCORE</th>\n",
       "      <th>POLARITY SCORE</th>\n",
       "      <th>SUBJECTIVITY SCORE</th>\n",
       "      <th>AVG SENTENCE LENGTH</th>\n",
       "      <th>PERCENTAGE OF COMPLEX WORDS</th>\n",
       "      <th>FOG INDEX</th>\n",
       "      <th>AVG NUMBER OF WORDS PER SENTENCE</th>\n",
       "      <th>COMPLEX WORD COUNT</th>\n",
       "      <th>WORD COUNT</th>\n",
       "      <th>SYLLABLE PER WORD</th>\n",
       "      <th>PERSONAL PRONOUNS</th>\n",
       "      <th>AVG WORD LENGTH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>blackassign0001</td>\n",
       "      <td>https://insights.blackcoffer.com/rising-it-cit...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>blackassign0002</td>\n",
       "      <td>https://insights.blackcoffer.com/rising-it-cit...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blackassign0003</td>\n",
       "      <td>https://insights.blackcoffer.com/internet-dema...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>blackassign0004</td>\n",
       "      <td>https://insights.blackcoffer.com/rise-of-cyber...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>blackassign0005</td>\n",
       "      <td>https://insights.blackcoffer.com/ott-platform-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>blackassign0096</td>\n",
       "      <td>https://insights.blackcoffer.com/what-is-the-r...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>blackassign0097</td>\n",
       "      <td>https://insights.blackcoffer.com/impact-of-cov...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>blackassign0098</td>\n",
       "      <td>https://insights.blackcoffer.com/contribution-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>blackassign0099</td>\n",
       "      <td>https://insights.blackcoffer.com/how-covid-19-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>blackassign0100</td>\n",
       "      <td>https://insights.blackcoffer.com/how-will-covi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 15 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6638bb3b-0963-453e-9151-690bb4dd8ec6')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-6638bb3b-0963-453e-9151-690bb4dd8ec6 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-6638bb3b-0963-453e-9151-690bb4dd8ec6');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-1ac371b7-38aa-4cc8-a997-ec3f6db9971a\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1ac371b7-38aa-4cc8-a997-ec3f6db9971a')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-1ac371b7-38aa-4cc8-a997-ec3f6db9971a button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "  <div id=\"id_cf6d5265-836a-4b5a-8011-ae1bd282f13d\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df_output')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_cf6d5265-836a-4b5a-8011-ae1bd282f13d button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('df_output');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "             URL_ID                                                URL  \\\n",
       "0   blackassign0001  https://insights.blackcoffer.com/rising-it-cit...   \n",
       "1   blackassign0002  https://insights.blackcoffer.com/rising-it-cit...   \n",
       "2   blackassign0003  https://insights.blackcoffer.com/internet-dema...   \n",
       "3   blackassign0004  https://insights.blackcoffer.com/rise-of-cyber...   \n",
       "4   blackassign0005  https://insights.blackcoffer.com/ott-platform-...   \n",
       "..              ...                                                ...   \n",
       "95  blackassign0096  https://insights.blackcoffer.com/what-is-the-r...   \n",
       "96  blackassign0097  https://insights.blackcoffer.com/impact-of-cov...   \n",
       "97  blackassign0098  https://insights.blackcoffer.com/contribution-...   \n",
       "98  blackassign0099  https://insights.blackcoffer.com/how-covid-19-...   \n",
       "99  blackassign0100  https://insights.blackcoffer.com/how-will-covi...   \n",
       "\n",
       "   POSITIVE SCORE NEGATIVE SCORE POLARITY SCORE SUBJECTIVITY SCORE  \\\n",
       "0             NaN            NaN            NaN                NaN   \n",
       "1             NaN            NaN            NaN                NaN   \n",
       "2             NaN            NaN            NaN                NaN   \n",
       "3             NaN            NaN            NaN                NaN   \n",
       "4             NaN            NaN            NaN                NaN   \n",
       "..            ...            ...            ...                ...   \n",
       "95            NaN            NaN            NaN                NaN   \n",
       "96            NaN            NaN            NaN                NaN   \n",
       "97            NaN            NaN            NaN                NaN   \n",
       "98            NaN            NaN            NaN                NaN   \n",
       "99            NaN            NaN            NaN                NaN   \n",
       "\n",
       "   AVG SENTENCE LENGTH PERCENTAGE OF COMPLEX WORDS FOG INDEX  \\\n",
       "0                  NaN                         NaN       NaN   \n",
       "1                  NaN                         NaN       NaN   \n",
       "2                  NaN                         NaN       NaN   \n",
       "3                  NaN                         NaN       NaN   \n",
       "4                  NaN                         NaN       NaN   \n",
       "..                 ...                         ...       ...   \n",
       "95                 NaN                         NaN       NaN   \n",
       "96                 NaN                         NaN       NaN   \n",
       "97                 NaN                         NaN       NaN   \n",
       "98                 NaN                         NaN       NaN   \n",
       "99                 NaN                         NaN       NaN   \n",
       "\n",
       "   AVG NUMBER OF WORDS PER SENTENCE COMPLEX WORD COUNT WORD COUNT  \\\n",
       "0                               NaN                NaN        NaN   \n",
       "1                               NaN                NaN        NaN   \n",
       "2                               NaN                NaN        NaN   \n",
       "3                               NaN                NaN        NaN   \n",
       "4                               NaN                NaN        NaN   \n",
       "..                              ...                ...        ...   \n",
       "95                              NaN                NaN        NaN   \n",
       "96                              NaN                NaN        NaN   \n",
       "97                              NaN                NaN        NaN   \n",
       "98                              NaN                NaN        NaN   \n",
       "99                              NaN                NaN        NaN   \n",
       "\n",
       "   SYLLABLE PER WORD PERSONAL PRONOUNS AVG WORD LENGTH  \n",
       "0                NaN               NaN             NaN  \n",
       "1                NaN               NaN             NaN  \n",
       "2                NaN               NaN             NaN  \n",
       "3                NaN               NaN             NaN  \n",
       "4                NaN               NaN             NaN  \n",
       "..               ...               ...             ...  \n",
       "95               NaN               NaN             NaN  \n",
       "96               NaN               NaN             NaN  \n",
       "97               NaN               NaN             NaN  \n",
       "98               NaN               NaN             NaN  \n",
       "99               NaN               NaN             NaN  \n",
       "\n",
       "[100 rows x 15 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_output = pd.DataFrame(columns=['URL_ID','URL','POSITIVE SCORE','NEGATIVE SCORE','POLARITY SCORE','SUBJECTIVITY SCORE','AVG SENTENCE LENGTH',\n",
    "'PERCENTAGE OF COMPLEX WORDS','FOG INDEX','AVG NUMBER OF WORDS PER SENTENCE','COMPLEX WORD COUNT','WORD COUNT','SYLLABLE PER WORD',\n",
    "'PERSONAL PRONOUNS','AVG WORD LENGTH'])\n",
    "df_output['URL_ID'] = df['URL_ID']\n",
    "df_output['URL'] = df['URL']\n",
    "df_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "id": "phpnZ1OAYG3h"
   },
   "outputs": [],
   "source": [
    "# returns Average Sentence Length(cleaned words) by default and Cleaned Word Count if check is True\n",
    "def sentence_len(file,check=False):\n",
    "    f = open(file,'r+',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    # only counting total words after removing stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    word_tokens = nltk.word_tokenize(text)\n",
    "    filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]\n",
    "    count_words=0\n",
    "    #punctuations have length of 1, so we can discard them\n",
    "    for i in filtered_sentence:\n",
    "        if(len(i)>1):\n",
    "            count_words+=1\n",
    "    #  sentence count\n",
    "    sen_count=0\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    sen_count = len(sentences)\n",
    "    if(check is True):\n",
    "        return count_words\n",
    "    return count_words/sen_count\n",
    "\n",
    "# returns Percentage of Complex words(with cleaned words) and Complex Word Count if check is True\n",
    "def complex(file,check=False):\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    word_tokens = nltk.word_tokenize(text)\n",
    "    filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]\n",
    "    count_complex_words = 0\n",
    "    vowels = 'aeiouy'\n",
    "    for word in filtered_sentence:\n",
    "        word = word.lower()\n",
    "        count = 0\n",
    "        if word[0] in vowels:\n",
    "            count += 1\n",
    "        for index in range(1, len(word)):\n",
    "            if word[index] in vowels and word[index - 1] not in vowels:\n",
    "                count += 1\n",
    "        if word.endswith(\"e\"):\n",
    "            count -= 1\n",
    "        if count == 0:\n",
    "            count += 1\n",
    "        if(count>2):\n",
    "            count_complex_words += 1\n",
    "\n",
    "    if(check==True):\n",
    "        return count_complex_words\n",
    "    return count_complex_words/sentence_len(file,True)\n",
    "\n",
    "def fog(file):\n",
    "    return 0.4*(sentence_len(file)*complex(file))\n",
    "\n",
    "#  returns Average Number of Words Per Sentence(Total words without cleaning)\n",
    "def total_avg(file):\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    # removing punctuations to count all the words\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    words = tokenizer.tokenize(text)\n",
    "    word_count = len(words)\n",
    "\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    sen_count = len(sentences)\n",
    "\n",
    "    return word_count/sen_count\n",
    "\n",
    "# returns Syllable Count Per Word\n",
    "def syllable_per_word(file):\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    # removing punctuations to count all the words\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    words = tokenizer.tokenize(text)\n",
    "\n",
    "    # syllables\n",
    "    vowels = 'aeiouy'\n",
    "    total_syllables = 0\n",
    "    for word in words:\n",
    "        word = word.lower()\n",
    "        count = 0\n",
    "        if word[0] in vowels:\n",
    "            count += 1\n",
    "        for index in range(1, len(word)):\n",
    "            if word[index] in vowels and word[index - 1] not in vowels:\n",
    "                count += 1\n",
    "        if word.endswith(\"e\"):\n",
    "            count -= 1\n",
    "        if count == 0:\n",
    "            count += 1\n",
    "        total_syllables += count\n",
    "\n",
    "    return total_syllables/len(words)\n",
    "\n",
    "# returns Count of personal pronouns\n",
    "def personal_pronouns(file):\n",
    "    pronouns = [\"i\", \"you\", \"he\", \"she\", \"it\", \"we\", \"they\", \"them\", \"us\", \"him\", \"her\", \"his\", \"hers\", \"its\", \"theirs\", \"our\", \"your\"]\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    # removing punctuations to count all the words\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    words = tokenizer.tokenize(text)\n",
    "    count = 0\n",
    "    for word in words:\n",
    "        if(word==\"US\"):\n",
    "            continue\n",
    "        elif(word.lower() in pronouns):\n",
    "            count += 1\n",
    "    return count\n",
    "\n",
    "# returns Average Word Length\n",
    "def average_word_length(file):\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    words = tokenizer.tokenize(text)\n",
    "    total_words = len(words)\n",
    "\n",
    "    word_len = 0\n",
    "    for word in words:\n",
    "        word_len += len(word)\n",
    "\n",
    "    return word_len/total_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "id": "d2ruxmVvZr6w"
   },
   "outputs": [],
   "source": [
    "def positive_score(file):\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    words = tokenizer.tokenize(text)\n",
    "\n",
    "    pos_file = open(\"/content/drive/MyDrive/positive-words.txt\",'r')\n",
    "    positive_words = nltk.word_tokenize(pos_file.read())\n",
    "\n",
    "    count = 0\n",
    "    for word in words:\n",
    "        word = word.lower()\n",
    "        if(word in positive_words):\n",
    "            count+=1\n",
    "\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "id": "0illxXcVaAN_"
   },
   "outputs": [],
   "source": [
    "def negative_score(file):\n",
    "    f = open(file,'r',encoding='utf-8')\n",
    "    text = f.read()\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    words = tokenizer.tokenize(text)\n",
    "\n",
    "    neg_file = open(\"/content/drive/MyDrive/negative-words_utf8.txt\",'r',)\n",
    "    negative_words = nltk.word_tokenize(neg_file.read())\n",
    "\n",
    "    count = 0\n",
    "    for word in words:\n",
    "        word = word.lower()\n",
    "        if(word in negative_words):\n",
    "            count-=1\n",
    "\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "id": "CgNkJJGfaG5u"
   },
   "outputs": [],
   "source": [
    "def polarity(file):\n",
    "    # Polarity Score = (Positive Score – Negative Score)/ ((Positive Score + Negative Score) + 0.000001)\n",
    "    return (positive_score(file)-negative_score(file))/((positive_score(file)+negative_score(file)) +  0.000001)\n",
    "\n",
    "def subjectivity(file):\n",
    "    #  Subjectivity Score = (Positive Score + Negative Score)/ ((Total Words after cleaning) + 0.000001)\n",
    "    return (positive_score(file)+negative_score(file))/(sentence_len(file,True) + 0.000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "id": "DAnwunr5aNYH"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "def calculate_set(id,index):\n",
    "    file = f\"{id}.txt\"\n",
    "    if(os.path.isfile(file)==False):\n",
    "        return\n",
    "    elif os.path.getsize(file) == 0:\n",
    "        return\n",
    "    elif(os.path.exists(file)):\n",
    "        df_output.iloc[index,2] = positive_score(file)\n",
    "        df_output.iloc[index,3] = negative_score(file)\n",
    "        df_output.iloc[index,4] = polarity(file)\n",
    "        df_output.iloc[index,5] = subjectivity(file)\n",
    "        df_output.iloc[index,6] = sentence_len(file,False) # Average Sentence Length(cleaned words)\n",
    "        df_output.iloc[index,7] = complex(file) # Percentage of Complex words(with cleaned words)\n",
    "        df_output.iloc[index,8] = fog(file) # Fog Index\n",
    "        df_output.iloc[index,9] = total_avg(file) # Average Number of Words Per Sentence(Total words without cleaning)\n",
    "        df_output.iloc[index,10] = complex(file,True) # Complex Word Count\n",
    "        df_output.iloc[index,11] = sentence_len(file,True) # Cleaned Word Count\n",
    "        df_output.iloc[index,12] = syllable_per_word(file) # Syllable Count Per Word\n",
    "        df_output.iloc[index,13] = personal_pronouns(file) # Counts personal pronouns\n",
    "        df_output.iloc[index,14] = average_word_length(file) # Average word length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rw7obo0faVLe",
    "outputId": "b6bd61a7-212e-401f-ffac-13e72ffc6a07"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blackassign0001\n",
      "blackassign0002\n",
      "blackassign0003\n",
      "blackassign0004\n",
      "blackassign0005\n",
      "blackassign0006\n",
      "blackassign0007\n",
      "blackassign0008\n",
      "blackassign0009\n",
      "blackassign0010\n",
      "blackassign0011\n",
      "blackassign0012\n",
      "blackassign0013\n",
      "blackassign0014\n",
      "blackassign0015\n",
      "blackassign0016\n",
      "blackassign0017\n",
      "blackassign0018\n",
      "blackassign0019\n",
      "blackassign0020\n",
      "blackassign0021\n",
      "blackassign0022\n",
      "blackassign0023\n",
      "blackassign0024\n",
      "blackassign0025\n",
      "blackassign0026\n",
      "blackassign0027\n",
      "blackassign0028\n",
      "blackassign0029\n",
      "blackassign0030\n",
      "blackassign0031\n",
      "blackassign0032\n",
      "blackassign0033\n",
      "blackassign0034\n",
      "blackassign0035\n",
      "blackassign0036\n",
      "blackassign0037\n",
      "blackassign0038\n",
      "blackassign0039\n",
      "blackassign0040\n",
      "blackassign0041\n",
      "blackassign0042\n",
      "blackassign0043\n",
      "blackassign0044\n",
      "blackassign0045\n",
      "blackassign0046\n",
      "blackassign0047\n",
      "blackassign0048\n",
      "blackassign0049\n",
      "blackassign0050\n",
      "blackassign0051\n",
      "blackassign0052\n",
      "blackassign0053\n",
      "blackassign0054\n",
      "blackassign0055\n",
      "blackassign0056\n",
      "blackassign0057\n",
      "blackassign0058\n",
      "blackassign0059\n",
      "blackassign0060\n",
      "blackassign0061\n",
      "blackassign0062\n",
      "blackassign0063\n",
      "blackassign0064\n",
      "blackassign0065\n",
      "blackassign0066\n",
      "blackassign0067\n",
      "blackassign0068\n",
      "blackassign0069\n",
      "blackassign0070\n",
      "blackassign0071\n",
      "blackassign0072\n",
      "blackassign0073\n",
      "blackassign0074\n",
      "blackassign0075\n",
      "blackassign0076\n",
      "blackassign0077\n",
      "blackassign0078\n",
      "blackassign0079\n",
      "blackassign0080\n",
      "blackassign0081\n",
      "blackassign0082\n",
      "blackassign0083\n",
      "blackassign0084\n",
      "blackassign0085\n",
      "blackassign0086\n",
      "blackassign0087\n",
      "blackassign0088\n",
      "blackassign0089\n",
      "blackassign0090\n",
      "blackassign0091\n",
      "blackassign0092\n",
      "blackassign0093\n",
      "blackassign0094\n",
      "blackassign0095\n",
      "blackassign0096\n",
      "blackassign0097\n",
      "blackassign0098\n",
      "blackassign0099\n",
      "blackassign0100\n"
     ]
    }
   ],
   "source": [
    "for index,row in df.iterrows():\n",
    "    j = row['URL_ID']\n",
    "    print(j)\n",
    "    calculate_set(j,index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 582
    },
    "id": "_NYl1caDMj94",
    "outputId": "06daca8a-3aed-401a-e295-b862c98ba6e1"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"df_output\",\n  \"rows\": 100,\n  \"fields\": [\n    {\n      \"column\": \"URL_ID\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"blackassign0084\",\n          \"blackassign0054\",\n          \"blackassign0071\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"URL\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"https://insights.blackcoffer.com/how-voice-search-makes-your-business-a-successful-business/\",\n          \"https://insights.blackcoffer.com/how-google-fit-measure-heart-and-respiratory-rates-using-a-phone/\",\n          \"https://insights.blackcoffer.com/how-to-overcome-your-fear-of-making-mistakes-2/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"POSITIVE SCORE\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0,\n        \"max\": 106,\n        \"num_unique_values\": 58,\n        \"samples\": [\n          8,\n          92,\n          27\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"NEGATIVE SCORE\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": -89,\n        \"max\": 0,\n        \"num_unique_values\": 52,\n        \"samples\": [\n          -15,\n          -25,\n          -45\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"POLARITY SCORE\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": -53.000053000053,\n        \"max\": 24.333325222224925,\n        \"num_unique_values\": 97,\n        \"samples\": [\n          3.5217389773156964,\n          2.058823489042677,\n          -10.142858591836942\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SUBJECTIVITY SCORE\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": -0.10236220452290905,\n        \"max\": 0.13852813792844962,\n        \"num_unique_values\": 97,\n        \"samples\": [\n          0.028465346499424077,\n          0.058285714219102044,\n          -0.013384321198117933\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AVG SENTENCE LENGTH\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 6.175438596491228,\n        \"max\": 102.44444444444444,\n        \"num_unique_values\": 96,\n        \"samples\": [\n          7.6,\n          15.233333333333333,\n          18.125\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"PERCENTAGE OF COMPLEX WORDS\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.22330097087378642,\n        \"max\": 0.5390781563126252,\n        \"num_unique_values\": 97,\n        \"samples\": [\n          0.29455445544554454,\n          0.34285714285714286,\n          0.28680688336520077\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"FOG INDEX\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.76,\n        \"max\": 12.044444444444444,\n        \"num_unique_values\": 95,\n        \"samples\": [\n          0.76,\n          2.5000000000000004,\n          1.5448275862068965\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AVG NUMBER OF WORDS PER SENTENCE\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 13.0,\n        \"max\": 204.77777777777777,\n        \"num_unique_values\": 97,\n        \"samples\": [\n          18.01219512195122,\n          19.7,\n          28.842105263157894\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"COMPLEX WORD COUNT\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 2,\n        \"max\": 580,\n        \"num_unique_values\": 87,\n        \"samples\": [\n          297,\n          49,\n          456\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"WORD COUNT\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 7,\n        \"max\": 2162,\n        \"num_unique_values\": 92,\n        \"samples\": [\n          303,\n          331,\n          103\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SYLLABLE PER WORD\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 1.4484536082474226,\n        \"max\": 2.1312169312169313,\n        \"num_unique_values\": 97,\n        \"samples\": [\n          1.5748138117806365,\n          1.6865482233502538,\n          1.5054744525547445\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"PERSONAL PRONOUNS\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 1,\n        \"max\": 148,\n        \"num_unique_values\": 51,\n        \"samples\": [\n          105,\n          73,\n          42\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AVG WORD LENGTH\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 4.319587628865979,\n        \"max\": 6.222222222222222,\n        \"num_unique_values\": 97,\n        \"samples\": [\n          4.758970886932972,\n          4.98984771573604,\n          4.617700729927007\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe",
       "variable_name": "df_output"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-8bbf0cf1-8cb3-4de2-93ac-2f4abc0497a7\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL_ID</th>\n",
       "      <th>URL</th>\n",
       "      <th>POSITIVE SCORE</th>\n",
       "      <th>NEGATIVE SCORE</th>\n",
       "      <th>POLARITY SCORE</th>\n",
       "      <th>SUBJECTIVITY SCORE</th>\n",
       "      <th>AVG SENTENCE LENGTH</th>\n",
       "      <th>PERCENTAGE OF COMPLEX WORDS</th>\n",
       "      <th>FOG INDEX</th>\n",
       "      <th>AVG NUMBER OF WORDS PER SENTENCE</th>\n",
       "      <th>COMPLEX WORD COUNT</th>\n",
       "      <th>WORD COUNT</th>\n",
       "      <th>SYLLABLE PER WORD</th>\n",
       "      <th>PERSONAL PRONOUNS</th>\n",
       "      <th>AVG WORD LENGTH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>blackassign0001</td>\n",
       "      <td>https://insights.blackcoffer.com/rising-it-cit...</td>\n",
       "      <td>8</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.285714</td>\n",
       "      <td>0.037037</td>\n",
       "      <td>7.56</td>\n",
       "      <td>0.259259</td>\n",
       "      <td>0.784</td>\n",
       "      <td>14.28</td>\n",
       "      <td>49</td>\n",
       "      <td>189</td>\n",
       "      <td>1.546218</td>\n",
       "      <td>22</td>\n",
       "      <td>4.515406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>blackassign0002</td>\n",
       "      <td>https://insights.blackcoffer.com/rising-it-cit...</td>\n",
       "      <td>62</td>\n",
       "      <td>-31</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.036471</td>\n",
       "      <td>11.038961</td>\n",
       "      <td>0.388235</td>\n",
       "      <td>1.714286</td>\n",
       "      <td>18.74026</td>\n",
       "      <td>330</td>\n",
       "      <td>850</td>\n",
       "      <td>1.837145</td>\n",
       "      <td>27</td>\n",
       "      <td>5.358281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blackassign0003</td>\n",
       "      <td>https://insights.blackcoffer.com/internet-dema...</td>\n",
       "      <td>41</td>\n",
       "      <td>-26</td>\n",
       "      <td>4.466666</td>\n",
       "      <td>0.022355</td>\n",
       "      <td>11.982143</td>\n",
       "      <td>0.508197</td>\n",
       "      <td>2.435714</td>\n",
       "      <td>19.446429</td>\n",
       "      <td>341</td>\n",
       "      <td>671</td>\n",
       "      <td>2.061524</td>\n",
       "      <td>28</td>\n",
       "      <td>5.986226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>blackassign0004</td>\n",
       "      <td>https://insights.blackcoffer.com/rise-of-cyber...</td>\n",
       "      <td>40</td>\n",
       "      <td>-75</td>\n",
       "      <td>-3.285714</td>\n",
       "      <td>-0.052239</td>\n",
       "      <td>13.137255</td>\n",
       "      <td>0.476119</td>\n",
       "      <td>2.501961</td>\n",
       "      <td>20.882353</td>\n",
       "      <td>319</td>\n",
       "      <td>670</td>\n",
       "      <td>1.981221</td>\n",
       "      <td>18</td>\n",
       "      <td>5.883568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>blackassign0005</td>\n",
       "      <td>https://insights.blackcoffer.com/ott-platform-...</td>\n",
       "      <td>26</td>\n",
       "      <td>-10</td>\n",
       "      <td>2.25</td>\n",
       "      <td>0.038741</td>\n",
       "      <td>10.589744</td>\n",
       "      <td>0.360775</td>\n",
       "      <td>1.528205</td>\n",
       "      <td>18.025641</td>\n",
       "      <td>149</td>\n",
       "      <td>413</td>\n",
       "      <td>1.738265</td>\n",
       "      <td>15</td>\n",
       "      <td>5.371266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>blackassign0096</td>\n",
       "      <td>https://insights.blackcoffer.com/what-is-the-r...</td>\n",
       "      <td>30</td>\n",
       "      <td>-57</td>\n",
       "      <td>-3.222222</td>\n",
       "      <td>-0.041602</td>\n",
       "      <td>12.98</td>\n",
       "      <td>0.391371</td>\n",
       "      <td>2.032</td>\n",
       "      <td>22.94</td>\n",
       "      <td>254</td>\n",
       "      <td>649</td>\n",
       "      <td>1.741935</td>\n",
       "      <td>8</td>\n",
       "      <td>5.12816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>blackassign0097</td>\n",
       "      <td>https://insights.blackcoffer.com/impact-of-cov...</td>\n",
       "      <td>32</td>\n",
       "      <td>-39</td>\n",
       "      <td>-10.142859</td>\n",
       "      <td>-0.013384</td>\n",
       "      <td>13.763158</td>\n",
       "      <td>0.286807</td>\n",
       "      <td>1.578947</td>\n",
       "      <td>28.842105</td>\n",
       "      <td>150</td>\n",
       "      <td>523</td>\n",
       "      <td>1.505474</td>\n",
       "      <td>38</td>\n",
       "      <td>4.617701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>blackassign0098</td>\n",
       "      <td>https://insights.blackcoffer.com/contribution-...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.020202</td>\n",
       "      <td>19.8</td>\n",
       "      <td>0.282828</td>\n",
       "      <td>2.24</td>\n",
       "      <td>29.6</td>\n",
       "      <td>28</td>\n",
       "      <td>99</td>\n",
       "      <td>1.756757</td>\n",
       "      <td>1</td>\n",
       "      <td>5.324324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>blackassign0099</td>\n",
       "      <td>https://insights.blackcoffer.com/how-covid-19-...</td>\n",
       "      <td>24</td>\n",
       "      <td>-3</td>\n",
       "      <td>1.285714</td>\n",
       "      <td>0.067961</td>\n",
       "      <td>11.035714</td>\n",
       "      <td>0.223301</td>\n",
       "      <td>0.985714</td>\n",
       "      <td>19.464286</td>\n",
       "      <td>69</td>\n",
       "      <td>309</td>\n",
       "      <td>1.546789</td>\n",
       "      <td>14</td>\n",
       "      <td>4.699083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>blackassign0100</td>\n",
       "      <td>https://insights.blackcoffer.com/how-will-covi...</td>\n",
       "      <td>37</td>\n",
       "      <td>-56</td>\n",
       "      <td>-4.894737</td>\n",
       "      <td>-0.037109</td>\n",
       "      <td>16.516129</td>\n",
       "      <td>0.361328</td>\n",
       "      <td>2.387097</td>\n",
       "      <td>30.741935</td>\n",
       "      <td>185</td>\n",
       "      <td>512</td>\n",
       "      <td>1.668416</td>\n",
       "      <td>9</td>\n",
       "      <td>5.011542</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 15 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8bbf0cf1-8cb3-4de2-93ac-2f4abc0497a7')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-8bbf0cf1-8cb3-4de2-93ac-2f4abc0497a7 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-8bbf0cf1-8cb3-4de2-93ac-2f4abc0497a7');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-dced0c3c-4e6d-4f40-b677-540e387fd4b7\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-dced0c3c-4e6d-4f40-b677-540e387fd4b7')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-dced0c3c-4e6d-4f40-b677-540e387fd4b7 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "  <div id=\"id_7596da27-2e8f-4332-b825-546999e669f0\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df_output')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_7596da27-2e8f-4332-b825-546999e669f0 button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('df_output');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "             URL_ID                                                URL  \\\n",
       "0   blackassign0001  https://insights.blackcoffer.com/rising-it-cit...   \n",
       "1   blackassign0002  https://insights.blackcoffer.com/rising-it-cit...   \n",
       "2   blackassign0003  https://insights.blackcoffer.com/internet-dema...   \n",
       "3   blackassign0004  https://insights.blackcoffer.com/rise-of-cyber...   \n",
       "4   blackassign0005  https://insights.blackcoffer.com/ott-platform-...   \n",
       "..              ...                                                ...   \n",
       "95  blackassign0096  https://insights.blackcoffer.com/what-is-the-r...   \n",
       "96  blackassign0097  https://insights.blackcoffer.com/impact-of-cov...   \n",
       "97  blackassign0098  https://insights.blackcoffer.com/contribution-...   \n",
       "98  blackassign0099  https://insights.blackcoffer.com/how-covid-19-...   \n",
       "99  blackassign0100  https://insights.blackcoffer.com/how-will-covi...   \n",
       "\n",
       "   POSITIVE SCORE NEGATIVE SCORE POLARITY SCORE SUBJECTIVITY SCORE  \\\n",
       "0               8             -1       1.285714           0.037037   \n",
       "1              62            -31            3.0           0.036471   \n",
       "2              41            -26       4.466666           0.022355   \n",
       "3              40            -75      -3.285714          -0.052239   \n",
       "4              26            -10           2.25           0.038741   \n",
       "..            ...            ...            ...                ...   \n",
       "95             30            -57      -3.222222          -0.041602   \n",
       "96             32            -39     -10.142859          -0.013384   \n",
       "97              2              0            1.0           0.020202   \n",
       "98             24             -3       1.285714           0.067961   \n",
       "99             37            -56      -4.894737          -0.037109   \n",
       "\n",
       "   AVG SENTENCE LENGTH PERCENTAGE OF COMPLEX WORDS FOG INDEX  \\\n",
       "0                 7.56                    0.259259     0.784   \n",
       "1            11.038961                    0.388235  1.714286   \n",
       "2            11.982143                    0.508197  2.435714   \n",
       "3            13.137255                    0.476119  2.501961   \n",
       "4            10.589744                    0.360775  1.528205   \n",
       "..                 ...                         ...       ...   \n",
       "95               12.98                    0.391371     2.032   \n",
       "96           13.763158                    0.286807  1.578947   \n",
       "97                19.8                    0.282828      2.24   \n",
       "98           11.035714                    0.223301  0.985714   \n",
       "99           16.516129                    0.361328  2.387097   \n",
       "\n",
       "   AVG NUMBER OF WORDS PER SENTENCE COMPLEX WORD COUNT WORD COUNT  \\\n",
       "0                             14.28                 49        189   \n",
       "1                          18.74026                330        850   \n",
       "2                         19.446429                341        671   \n",
       "3                         20.882353                319        670   \n",
       "4                         18.025641                149        413   \n",
       "..                              ...                ...        ...   \n",
       "95                            22.94                254        649   \n",
       "96                        28.842105                150        523   \n",
       "97                             29.6                 28         99   \n",
       "98                        19.464286                 69        309   \n",
       "99                        30.741935                185        512   \n",
       "\n",
       "   SYLLABLE PER WORD PERSONAL PRONOUNS AVG WORD LENGTH  \n",
       "0           1.546218                22        4.515406  \n",
       "1           1.837145                27        5.358281  \n",
       "2           2.061524                28        5.986226  \n",
       "3           1.981221                18        5.883568  \n",
       "4           1.738265                15        5.371266  \n",
       "..               ...               ...             ...  \n",
       "95          1.741935                 8         5.12816  \n",
       "96          1.505474                38        4.617701  \n",
       "97          1.756757                 1        5.324324  \n",
       "98          1.546789                14        4.699083  \n",
       "99          1.668416                 9        5.011542  \n",
       "\n",
       "[100 rows x 15 columns]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "id": "I2uSm26pMk6-"
   },
   "outputs": [],
   "source": [
    "df_output.to_excel(\"output.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "id": "LoJdh65eMyoq"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}