05 - NER - Sentiment Analysis - Topic Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9e73fe20",
   "metadata": {},
   "source": [
    "# NER, Sentiment Analysis, and Topic Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bbe80aa",
   "metadata": {},
   "source": [
    "### NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "523bf2a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing the spaCy library into Python\n",
    "import spacy\n",
    "\n",
    "# loading a small language model for English; assign the result under 'nlp'\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90cb531a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's again load our text\n",
    "with open('data/am_i_the_asshole_example.txt', 'r', encoding='UTF-8') as f:\n",
    "    text = f.read()\n",
    "\n",
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19f5ed4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for ent in doc.ents:\n",
    "    print(ent.text,ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e924ba4",
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy.explain('NORP')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c50c0a0a",
   "metadata": {},
   "source": [
    "If you need to adapt your NER model, check it out [here](https://spacy.io/api/entityrecognizer) for more info"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58df428a",
   "metadata": {},
   "source": [
    "### Sentiment Analysis\n",
    "\n",
    "With spaCy and TextBlob. TextBlob is a Lexicon-based sentiment analyzer that contains a dictionary with scores to calculate a sentence's polarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85f3ac71",
   "metadata": {},
   "outputs": [],
   "source": [
    "# installing spacytextblob\n",
    "!pip install spacytextblob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "206be147",
   "metadata": {},
   "outputs": [],
   "source": [
    "# downloading corpora\n",
    "!python -m textblob.download_corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2be354dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90bb87e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# checking the components if the spacy NLP pipeline\n",
    "nlp.components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0ea5823",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacytextblob.spacytextblob import SpacyTextBlob \n",
    "from textblob import TextBlob\n",
    "from textblob.sentiments import NaiveBayesAnalyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c28438b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding to NLP pipeline\n",
    "nlp.add_pipe('spacytextblob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "243d2db6",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "035940a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(text.replace('\\n',' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdbf57c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc._.polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34310d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc._.subjectivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00081175",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc._.assessments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3c3b241",
   "metadata": {},
   "outputs": [],
   "source": [
    "for span in doc.sents:\n",
    "    print(span.text, span._.polarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90dbe5ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "for span in doc.sents:\n",
    "    blob = TextBlob(span.text, analyzer=NaiveBayesAnalyzer())\n",
    "    print(span, blob.sentiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28f5228a",
   "metadata": {},
   "source": [
    "### Topic Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09ee203b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.models import CoherenceModel\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8b2aa4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading the English model\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "# we don't want to split words with apostrophe\n",
    "nlp.tokenizer.rules = {key: value for key, value in nlp.tokenizer.rules.items() if \"'\" not in key and \"’\" not in key and \"‘\" not in key}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "643b48b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_folder = 'data/topic_models/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f01e8b0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_files=sorted([os.path.join(path_to_folder, f) for f in os.listdir(path_to_folder)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de11a5c6",
   "metadata": {},
   "source": [
    "#### Preprocessing the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bef82eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add as many stopwords necessary\n",
    "extra_stop = ['mr','ms','mrs','hon']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8872a25a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# turning the texts into tokens:\n",
    "tokenized_corpus = []\n",
    "\n",
    "for my_file in path_to_files:\n",
    "    \n",
    "    with open(my_file, encoding='utf-8') as f:\n",
    "        #text = f.readlines()\n",
    "        #text = ''.join(text).replace('\\n',' ')\n",
    "        \n",
    "        # turn all the lines into a single string\n",
    "        #text = ''.join(text)\n",
    "        text = f.read().replace('\\n','')\n",
    "        \n",
    "        # create the spacy doc object with the text all in lowercase\n",
    "        doc = nlp(text)\n",
    "\n",
    "        # filtering tokens and lemmatizing\n",
    "        text = []\n",
    "        for word in doc:\n",
    "            if not word.is_stop and not word.is_punct and not word.like_num and word.lemma_ not in extra_stop:\n",
    "                #print(word.lemma_)\n",
    "                text.append(word.lemma_.lower())\n",
    "            \n",
    "        tokenized_corpus.append(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "408c3691",
   "metadata": {},
   "source": [
    "#### Turning the corpus into bags of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69361986",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mapping words to ids\n",
    "words_id = corpora.Dictionary(tokenized_corpus)\n",
    "\n",
    "# corpus becomes a bag of words\n",
    "corpus = [words_id.doc2bow(txt) for txt in tokenized_corpus]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c40d7c00",
   "metadata": {},
   "source": [
    "##### Checking coherence score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "674ebdee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# checking \"optimal\" number of topics\n",
    "k_init = 5\n",
    "k_final = 15\n",
    "for k in range(k_init,k_final+1):\n",
    "    lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
    "                                           id2word=words_id,\n",
    "                                           num_topics=k, \n",
    "                                           random_state=50,\n",
    "                                           passes=20,\n",
    "                                           per_word_topics=True)\n",
    "    \n",
    "    # let's compute perplexity (lower) and coherence score (higher)\n",
    "    per_lda = lda_model.log_perplexity(corpus)\n",
    "    coherence_model_lda = CoherenceModel(model=lda_model, texts=tokenized_corpus, dictionary=words_id, coherence='c_v')\n",
    "    coherence_lda = coherence_model_lda.get_coherence()\n",
    "    print(k,per_lda,coherence_lda)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d72ac02e",
   "metadata": {},
   "source": [
    "##### Running lda model for number of topics with highest cohrence score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61dadbc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
    "                                           id2word=words_id,\n",
    "                                           num_topics=9, \n",
    "                                           random_state=50,\n",
    "                                           passes=20,\n",
    "                                           per_word_topics=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33987485",
   "metadata": {},
   "source": [
    "##### Topic composition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7b4696f",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model.show_topics(num_words=20,num_topics=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08ea1f65",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}