8-Day2-NLTK-Corpora.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "'''What is a corpus?\r\n",
    "A corpus can be defined as a collection of text documents. It can be thought as just a bunch of text files in a directory,\r\n",
    "often alongside many other directories of text files.\r\n",
    "\r\n",
    "How it is done ?\r\n",
    "NLTK already defines a list of data paths or directories in nltk.data.path. Our custom corpora must be present within any of \r\n",
    "these given paths so it can be found by NLTK.\r\n",
    "We can also create a custom nltk_data directory in our home directory and verify that it is in the list of known paths specified \r\n",
    "by nltk.data.path.\r\n",
    "'''"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import nltk\r\n",
    "from nltk.corpus import gutenberg\r\n",
    "from nltk.tokenize import sent_tokenize"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "sample = gutenberg.raw(\"bible-kjv.txt\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "tok = sent_tokenize(sample)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "print(tok[5:15])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['1:5 And God called the light Day, and the darkness he called Night.', 'And the evening and the morning were the first day.', '1:6 And God said, Let there be a firmament in the midst of the waters,\\nand let it divide the waters from the waters.', '1:7 And God made the firmament, and divided the waters which were\\nunder the firmament from the waters which were above the firmament:\\nand it was so.', '1:8 And God called the firmament Heaven.', 'And the evening and the\\nmorning were the second day.', '1:9 And God said, Let the waters under the heaven be gathered together\\nunto one place, and let the dry land appear: and it was so.', '1:10 And God called the dry land Earth; and the gathering together of\\nthe waters called he Seas: and God saw that it was good.', '1:11 And God said, Let the earth bring forth grass, the herb yielding\\nseed, and the fruit tree yielding fruit after his kind, whose seed is\\nin itself, upon the earth: and it was so.', '1:12 And the earth brought forth grass, and herb yielding seed after\\nhis kind, and the tree yielding fruit, whose seed was in itself, after\\nhis kind: and God saw that it was good.']\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "print(nltk.__file__)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "C:\\Users\\susan\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\nltk\\__init__.py\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Additional Stuff"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "# importing libraries\r\n",
    "import os, os.path\r\n",
    "  \r\n",
    "# using the given path\r\n",
    "path = os.path.expanduser('~/nltk_data')\r\n",
    "  \r\n",
    "# checking\r\n",
    "if not os.path.exists(path):\r\n",
    "    os.mkdir(path)\r\n",
    "      \r\n",
    "print (\"Does path exists : \", os.path.exists(path))\r\n",
    "  \r\n",
    "  \r\n",
    "import nltk.data\r\n",
    "print (\"\\nDoes path exists in nltk : \", \r\n",
    "       path in nltk.data.path)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Does path exists :  True\n",
      "\n",
      "Does path exists in nltk :  True\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# Web and Chat Text"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "from nltk.corpus import webtext\r\n",
    "for fileid in webtext.fileids():\r\n",
    "    print(fileid, webtext.raw(fileid)[:65], '...')"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "firefox.txt Cookie Manager: \"Don't allow sites that set removed cookies to se ...\n",
      "grail.txt SCENE 1: [wind] [clop clop clop] \n",
      "KING ARTHUR: Whoa there!  [clop ...\n",
      "overheard.txt White guy: So, do you have any plans for this evening?\n",
      "Asian girl ...\n",
      "pirates.txt PIRATES OF THE CARRIBEAN: DEAD MAN'S CHEST, by Ted Elliott & Terr ...\n",
      "singles.txt 25 SEXY MALE, seeks attrac older single lady, for discreet encoun ...\n",
      "wine.txt Lovely delicate, fragrant Rhone wine. Polished leather and strawb ...\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "from nltk.corpus import nps_chat\r\n",
    "chatroom = nps_chat.posts('10-19-20s_706posts.xml')\r\n",
    "chatroom[123]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['i',\n",
       " 'do',\n",
       " \"n't\",\n",
       " 'want',\n",
       " 'hot',\n",
       " 'pics',\n",
       " 'of',\n",
       " 'a',\n",
       " 'female',\n",
       " ',',\n",
       " 'I',\n",
       " 'can',\n",
       " 'look',\n",
       " 'in',\n",
       " 'a',\n",
       " 'mirror',\n",
       " '.']"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Brown Corpus\r\n",
    "'''The Brown Corpus was the first million-word electronic corpus of English, created in 1961 at Brown University.\r\n",
    " This corpus contains text from 500 sources, and the sources have been categorized by genre, such as news, editorial, and so on.'''"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "from nltk.corpus import brown\r\n",
    "brown.categories()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['adventure',\n",
       " 'belles_lettres',\n",
       " 'editorial',\n",
       " 'fiction',\n",
       " 'government',\n",
       " 'hobbies',\n",
       " 'humor',\n",
       " 'learned',\n",
       " 'lore',\n",
       " 'mystery',\n",
       " 'news',\n",
       " 'religion',\n",
       " 'reviews',\n",
       " 'romance',\n",
       " 'science_fiction']"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "brown.words(categories='news')"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', ...]"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "brown.words(fileids=['cg22'])"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['Does', 'our', 'society', 'have', 'a', 'runaway', ',', ...]"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "brown.sents(categories=['news', 'editorial', 'reviews'])"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', 'Friday', 'an', 'investigation', 'of', \"Atlanta's\", 'recent', 'primary', 'election', 'produced', '``', 'no', 'evidence', \"''\", 'that', 'any', 'irregularities', 'took', 'place', '.'], ['The', 'jury', 'further', 'said', 'in', 'term-end', 'presentments', 'that', 'the', 'City', 'Executive', 'Committee', ',', 'which', 'had', 'over-all', 'charge', 'of', 'the', 'election', ',', '``', 'deserves', 'the', 'praise', 'and', 'thanks', 'of', 'the', 'City', 'of', 'Atlanta', \"''\", 'for', 'the', 'manner', 'in', 'which', 'the', 'election', 'was', 'conducted', '.'], ...]"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#We'll use NLTK's support for conditional frequency distributions."
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "cfd = nltk.ConditionalFreqDist(\r\n",
    "               (genre, word)\r\n",
    "               for genre in brown.categories()\r\n",
    "               for word in brown.words(categories=genre))\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "genres = ['news', 'religion', 'hobbies', 'science_fiction', 'romance', 'humor']\r\n",
    "modals = ['can', 'could', 'may', 'might', 'must', 'will']\r\n",
    "cfd.tabulate(conditions=genres, samples=modals)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "                  can could   may might  must  will \n",
      "           news    93    86    66    38    50   389 \n",
      "       religion    82    59    78    12    54    71 \n",
      "        hobbies   268    58   131    22    83   264 \n",
      "science_fiction    16    49     4    12     8    16 \n",
      "        romance    74   193    11    51    45    43 \n",
      "          humor    16    30     8     8     9    13 \n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#other corpora can also tried"
   ],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.6.10 64-bit ('tf-gpu': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "interpreter": {
   "hash": "135e78ef6267b613ce7b86630936d470174b66187aad9f784a45e5cc3235687c"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}