word2vec_text_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## word2vec classification\n",
    "\n",
    "#### Dataset: [Kaggle Bag of Words meets Bag of Popcorns](https://www.kaggle.com/c/word2vec-nlp-tutorial/data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.models import word2vec\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import nltk.data\n",
    "from nltk.corpus import stopwords\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dropout, Dense, Activation\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/labeledTrainData.tsv\", header=0, delimiter=\"\\t\", quoting=3, encoding='utf-8')\n",
    "test = pd.read_csv(\"data/testData.tsv\", header=0, delimiter=\"\\t\", quoting=3, encoding='utf-8')\n",
    "unlabeled_train = pd.read_csv(\"data/unlabeledTrainData.tsv\", header=0, delimiter=\"\\t\", quoting=3, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The function below takes in a review, removes HTML tags off of it and splits it to return a list of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def review2list(review):\n",
    "    review_text = BeautifulSoup(review).get_text()\n",
    "    review_text = re.sub(\"[^a-zA-Z]\", \" \", review_text)\n",
    "    words = review_text.lower().split()\n",
    "    return(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The function below splits the reviews to sentences and rturns a list of sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def review2sentences(review, tokenizer):\n",
    "    raw_sentences = tokenizer.tokenize(review.strip())\n",
    "    sentences = []\n",
    "    for raw_sentence in raw_sentences:\n",
    "        if len(raw_sentence)>0:\n",
    "            sentences.append(review2list(raw_sentence))\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making a list of all the sentences we wan't to feed in to our word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n"
     ]
    }
   ],
   "source": [
    "sentences = []\n",
    "i=0\n",
    "for review in train[\"review\"]:\n",
    "    if (i+1)%1000 == 0:\n",
    "        print i+1\n",
    "    sentences += review2sentences(review, tokenizer)\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n"
     ]
    }
   ],
   "source": [
    "i=0\n",
    "for review in unlabeled_train[\"review\"]:\n",
    "    if (i+1)%1000 == 0:\n",
    "        print i+1\n",
    "    sentences += review2sentences(review, tokenizer)\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "795538"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building the word2vec model\n",
    "\n",
    "You can find more about the hyperparameters over [here](https://radimrehurek.com/gensim/models/word2vec.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_features = 300\n",
    "min_word_count = 40\n",
    "num_workers = 4\n",
    "context = 10\n",
    "\n",
    "model = word2vec.Word2Vec(sentences, \n",
    "                          workers=num_workers, \n",
    "                          size = num_features,\n",
    "                          min_count = min_word_count,\n",
    "                          window = context)\n",
    "\n",
    "model.init_sims(replace=True)\n",
    "\n",
    "model.save(\"300features_40minwords_10context\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'kitchen'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.doesnt_match(\"man woman child kitchen\".split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_feature_vector(words, model, num_features):\n",
    "    featurevec = np.zeros((num_features,), dtype=\"float32\")\n",
    "    nwords=0\n",
    "    index2word_set = set(model.wv.index2word)\n",
    "    for word in words:\n",
    "        if word in index2word_set:\n",
    "            nwords+=1\n",
    "            featurevec = np.add(featurevec, model[word])\n",
    "    \n",
    "    featurevec = np.divide(featurevec, nwords)\n",
    "    return featurevec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_avg_vector(reviews, model, num_features):\n",
    "    counter=0\n",
    "    reviewfeaturevec = np.zeros((len(reviews), num_features), dtype=\"float32\")\n",
    "    for review in reviews:\n",
    "        if counter % 1000 == 0:\n",
    "            print \"Review %d of %d\" % (counter, len(reviews))\n",
    "        reviewfeaturevec[counter] = make_feature_vector(review, model, num_features)\n",
    "        counter = counter + 1\n",
    "    return reviewfeaturevec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def review2list(review, remove_stopwords=False):\n",
    "    review_text = BeautifulSoup(review).get_text()\n",
    "    review_text = re.sub(\"[^a-zA-Z]\", \" \", review_text)\n",
    "    words = review_text.lower().split()\n",
    "    if remove_stopwords:\n",
    "        stop = set(stopwords.words(\"english\"))\n",
    "        words = [w for w in words if not w in stop]\n",
    "    return(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review 0 of 25000\n",
      "Review 1000 of 25000\n",
      "Review 2000 of 25000\n",
      "Review 3000 of 25000\n",
      "Review 4000 of 25000\n",
      "Review 5000 of 25000\n",
      "Review 6000 of 25000\n",
      "Review 7000 of 25000\n",
      "Review 8000 of 25000\n",
      "Review 9000 of 25000\n",
      "Review 10000 of 25000\n",
      "Review 11000 of 25000\n",
      "Review 12000 of 25000\n",
      "Review 13000 of 25000\n",
      "Review 14000 of 25000\n",
      "Review 15000 of 25000\n",
      "Review 16000 of 25000\n",
      "Review 17000 of 25000\n",
      "Review 18000 of 25000\n",
      "Review 19000 of 25000\n",
      "Review 20000 of 25000\n",
      "Review 21000 of 25000\n",
      "Review 22000 of 25000\n",
      "Review 23000 of 25000\n",
      "Review 24000 of 25000\n",
      "Review 0 of 25000\n",
      "Review 1000 of 25000\n",
      "Review 2000 of 25000\n",
      "Review 3000 of 25000\n",
      "Review 4000 of 25000\n",
      "Review 5000 of 25000\n",
      "Review 6000 of 25000\n",
      "Review 7000 of 25000\n",
      "Review 8000 of 25000\n",
      "Review 9000 of 25000\n",
      "Review 10000 of 25000\n",
      "Review 11000 of 25000\n",
      "Review 12000 of 25000\n",
      "Review 13000 of 25000\n",
      "Review 14000 of 25000\n",
      "Review 15000 of 25000\n",
      "Review 16000 of 25000\n",
      "Review 17000 of 25000\n",
      "Review 18000 of 25000\n",
      "Review 19000 of 25000\n",
      "Review 20000 of 25000\n",
      "Review 21000 of 25000\n",
      "Review 22000 of 25000\n",
      "Review 23000 of 25000\n",
      "Review 24000 of 25000\n"
     ]
    }
   ],
   "source": [
    "clean_train_reviews = []\n",
    "for review in train[\"review\"]:\n",
    "    clean_train_reviews.append(review2list(review, remove_stopwords=True))\n",
    "    \n",
    "traindatavecs = get_avg_vector(clean_train_reviews, model, num_features)\n",
    "\n",
    "clean_test_reviews = []\n",
    "\n",
    "for review in test[\"review\"]:\n",
    "    clean_test_reviews.append(review2list(review, remove_stopwords=True))\n",
    "    \n",
    "testdatavecs = get_avg_vector(clean_test_reviews, model, num_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 300)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traindatavecs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_labels = np_utils.to_categorical(train['sentiment'], 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline model (Softmax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/25\n",
      "0s - loss: 0.2142 - acc: 0.7436 - val_loss: 0.1925 - val_acc: 0.7744\n",
      "Epoch 2/25\n",
      "0s - loss: 0.1771 - acc: 0.7913 - val_loss: 0.1687 - val_acc: 0.8010\n",
      "Epoch 3/25\n",
      "0s - loss: 0.1579 - acc: 0.8144 - val_loss: 0.1544 - val_acc: 0.8142\n",
      "Epoch 4/25\n",
      "0s - loss: 0.1455 - acc: 0.8282 - val_loss: 0.1450 - val_acc: 0.8224\n",
      "Epoch 5/25\n",
      "0s - loss: 0.1368 - acc: 0.8369 - val_loss: 0.1375 - val_acc: 0.8330\n",
      "Epoch 6/25\n",
      "0s - loss: 0.1304 - acc: 0.8429 - val_loss: 0.1324 - val_acc: 0.8362\n",
      "Epoch 7/25\n",
      "0s - loss: 0.1256 - acc: 0.8470 - val_loss: 0.1280 - val_acc: 0.8402\n",
      "Epoch 8/25\n",
      "0s - loss: 0.1218 - acc: 0.8497 - val_loss: 0.1250 - val_acc: 0.8432\n",
      "Epoch 9/25\n",
      "0s - loss: 0.1188 - acc: 0.8522 - val_loss: 0.1220 - val_acc: 0.8464\n",
      "Epoch 10/25\n",
      "0s - loss: 0.1162 - acc: 0.8547 - val_loss: 0.1198 - val_acc: 0.8488\n",
      "Epoch 11/25\n",
      "0s - loss: 0.1141 - acc: 0.8549 - val_loss: 0.1178 - val_acc: 0.8504\n",
      "Epoch 12/25\n",
      "0s - loss: 0.1124 - acc: 0.8557 - val_loss: 0.1162 - val_acc: 0.8534\n",
      "Epoch 13/25\n",
      "0s - loss: 0.1109 - acc: 0.8573 - val_loss: 0.1148 - val_acc: 0.8542\n",
      "Epoch 14/25\n",
      "0s - loss: 0.1096 - acc: 0.8579 - val_loss: 0.1136 - val_acc: 0.8552\n",
      "Epoch 15/25\n",
      "0s - loss: 0.1085 - acc: 0.8596 - val_loss: 0.1126 - val_acc: 0.8572\n",
      "Epoch 16/25\n",
      "0s - loss: 0.1075 - acc: 0.8602 - val_loss: 0.1116 - val_acc: 0.8572\n",
      "Epoch 17/25\n",
      "0s - loss: 0.1066 - acc: 0.8616 - val_loss: 0.1109 - val_acc: 0.8578\n",
      "Epoch 18/25\n",
      "0s - loss: 0.1058 - acc: 0.8617 - val_loss: 0.1100 - val_acc: 0.8586\n",
      "Epoch 19/25\n",
      "0s - loss: 0.1051 - acc: 0.8629 - val_loss: 0.1094 - val_acc: 0.8594\n",
      "Epoch 20/25\n",
      "0s - loss: 0.1045 - acc: 0.8636 - val_loss: 0.1087 - val_acc: 0.8586\n",
      "Epoch 21/25\n",
      "0s - loss: 0.1039 - acc: 0.8629 - val_loss: 0.1083 - val_acc: 0.8594\n",
      "Epoch 22/25\n",
      "0s - loss: 0.1033 - acc: 0.8650 - val_loss: 0.1079 - val_acc: 0.8592\n",
      "Epoch 23/25\n",
      "0s - loss: 0.1029 - acc: 0.8645 - val_loss: 0.1074 - val_acc: 0.8598\n",
      "Epoch 24/25\n",
      "0s - loss: 0.1024 - acc: 0.8649 - val_loss: 0.1069 - val_acc: 0.8604\n",
      "Epoch 25/25\n",
      "0s - loss: 0.1020 - acc: 0.8653 - val_loss: 0.1066 - val_acc: 0.8604\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(input_shape=(300,), units=2))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "\n",
    "model.compile(loss='mse', \n",
    "              optimizer='adam', \n",
    "              metrics=['accuracy'])\n",
    "history = model.fit(traindatavecs, train_labels, validation_split=0.20, verbose=2, epochs = 25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi Layer Perceptron\n",
    "\n",
    "P.S. This has huge scope of improvement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/25\n",
      "1s - loss: 0.1233 - acc: 0.8377 - val_loss: 0.1035 - val_acc: 0.8598\n",
      "Epoch 2/25\n",
      "1s - loss: 0.0977 - acc: 0.8650 - val_loss: 0.1000 - val_acc: 0.8654\n",
      "Epoch 3/25\n",
      "1s - loss: 0.0953 - acc: 0.8700 - val_loss: 0.0981 - val_acc: 0.8652\n",
      "Epoch 4/25\n",
      "1s - loss: 0.0938 - acc: 0.8715 - val_loss: 0.0972 - val_acc: 0.8644\n",
      "Epoch 5/25\n",
      "1s - loss: 0.0928 - acc: 0.8729 - val_loss: 0.1020 - val_acc: 0.8588\n",
      "Epoch 6/25\n",
      "1s - loss: 0.0916 - acc: 0.8746 - val_loss: 0.0966 - val_acc: 0.8696\n",
      "Epoch 7/25\n",
      "1s - loss: 0.0910 - acc: 0.8758 - val_loss: 0.0957 - val_acc: 0.8666\n",
      "Epoch 8/25\n",
      "1s - loss: 0.0902 - acc: 0.8765 - val_loss: 0.0978 - val_acc: 0.8692\n",
      "Epoch 9/25\n",
      "1s - loss: 0.0896 - acc: 0.8781 - val_loss: 0.0957 - val_acc: 0.8680\n",
      "Epoch 10/25\n",
      "1s - loss: 0.0886 - acc: 0.8805 - val_loss: 0.0970 - val_acc: 0.8664\n",
      "Epoch 11/25\n",
      "1s - loss: 0.0880 - acc: 0.8814 - val_loss: 0.0946 - val_acc: 0.8694\n",
      "Epoch 12/25\n",
      "1s - loss: 0.0877 - acc: 0.8819 - val_loss: 0.0960 - val_acc: 0.8654\n",
      "Epoch 13/25\n",
      "1s - loss: 0.0871 - acc: 0.8840 - val_loss: 0.0949 - val_acc: 0.8702\n",
      "Epoch 14/25\n",
      "1s - loss: 0.0859 - acc: 0.8851 - val_loss: 0.0966 - val_acc: 0.8680\n",
      "Epoch 15/25\n",
      "1s - loss: 0.0856 - acc: 0.8857 - val_loss: 0.0947 - val_acc: 0.8684\n",
      "Epoch 16/25\n",
      "1s - loss: 0.0846 - acc: 0.8878 - val_loss: 0.0954 - val_acc: 0.8702\n",
      "Epoch 17/25\n",
      "1s - loss: 0.0842 - acc: 0.8863 - val_loss: 0.0966 - val_acc: 0.8652\n",
      "Epoch 18/25\n",
      "1s - loss: 0.0835 - acc: 0.8886 - val_loss: 0.0962 - val_acc: 0.8656\n",
      "Epoch 19/25\n",
      "1s - loss: 0.0828 - acc: 0.8896 - val_loss: 0.0950 - val_acc: 0.8672\n",
      "Epoch 20/25\n",
      "1s - loss: 0.0823 - acc: 0.8905 - val_loss: 0.0941 - val_acc: 0.8700\n",
      "Epoch 21/25\n",
      "1s - loss: 0.0813 - acc: 0.8923 - val_loss: 0.0992 - val_acc: 0.8626\n",
      "Epoch 22/25\n",
      "1s - loss: 0.0805 - acc: 0.8931 - val_loss: 0.0935 - val_acc: 0.8716\n",
      "Epoch 23/25\n",
      "1s - loss: 0.0803 - acc: 0.8936 - val_loss: 0.0970 - val_acc: 0.8660\n",
      "Epoch 24/25\n",
      "1s - loss: 0.0794 - acc: 0.8949 - val_loss: 0.0953 - val_acc: 0.8658\n",
      "Epoch 25/25\n",
      "1s - loss: 0.0787 - acc: 0.8970 - val_loss: 0.0951 - val_acc: 0.8704\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(input_shape=(300,), units=150))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(2))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "\n",
    "model.compile(loss='mse', \n",
    "              optimizer='adam', \n",
    "              metrics=['accuracy'])\n",
    "history = model.fit(traindatavecs, train_labels, validation_split=0.20, verbose=2, epochs = 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}