Doc_retrieval_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for Python 2: use print only as a function\n",
    "# from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "import gensim\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import collections "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = 'C:\\Users\\Pushpendra\\Desktop\\qa\\datascience_datasets_for_doc_similarity.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d_set = pd.read_csv(path,header=None,names=['tag','title','body'],dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'adding', u'more', u'data', u'does', u'not', u'always', u'help', u'however', u'you', u'can', u'get', u'an', u'estimate', u'if', u'more', u'data', u'will', u'help', u'you', u'by', u'the', u'following', u'procedure', u'make', u'plot', u'on', u'the', u'axis', u'is', u'the', u'amount', u'of', u'training', u'examples', u'starting', u'at', u'one', u'example', u'per', u'class', u'going', u'to', u'wherever', u'you', u'are', u'currently', u'the', u'axis', u'shows', u'the', u'error', u'now', u'you', u'should', u'add', u'two', u'curves', u'training', u'and', u'test', u'error', u'for', u'low', u'the', u'training', u'error', u'should', u'be', u'very', u'low', u'almost', u'and', u'the', u'test', u'error', u'very', u'high', u'with', u'enough', u'data', u'they', u'should', u'be', u'about', u'the', u'same', u'by', u'plotting', u'those', u'curves', u'you', u'can', u'make', u'an', u'educated', u'guess', u'how', u'much', u'more', u'data', u'will', u'give', u'you', u'how', u'much', u'improvement', u'blockquote', u'when', u'doing', u'this', u'should', u'add', u'data', u'to', u'both', u'the', u'training', u'set', u'and', u'the', u'test', u'set', u'blockquote', u'depends', u'on', u'what', u'you', u'want', u'to', u'achieve', u'if', u'only', u'getting', u'better', u'classifier', u'then', u'you', u'can', u'only', u'add', u'it', u'to', u'the', u'training', u'set', u'however', u'if', u'you', u're', u'doing', u'this', u'in', u'scientific', u'setting', u'this', u'might', u'be', u'more', u'difficult', u'assume', u'that', u'your', u'test', u'set', u'is', u'of', u'reasonable', u'size', u'you', u'might', u'want', u'to', u'have', u'look', u'at', u'href', u'https', u'en', u'wikipedia', u'org', u'wiki', u'cross', u'validation_', u'statistics', u'rel', u'nofollow', u'cross', u'validation']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>['machine-learning', 'classification', 'evalua...</td>\n",
       "      <td>How to improve an existing (trained) classifier?</td>\n",
       "      <td>&lt;p&gt;Adding more data does not always help. Howe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>['machine-learning', 'r', 'logistic-regression...</td>\n",
       "      <td>Random Forest, Type - Regression, Calculation ...</td>\n",
       "      <td>&lt;p&gt;MSE is measure of error of the overall regr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>['visualization']</td>\n",
       "      <td>How to analyze which site has most numbers</td>\n",
       "      <td>&lt;p&gt;The most obvious way of visualizing this is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>['bigdata']</td>\n",
       "      <td>Privacy through fake data?</td>\n",
       "      <td>&lt;p&gt;There's an inherent problem that anybody wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>['r', 'data-wrangling']</td>\n",
       "      <td>When to choose character instead of factor in R?</td>\n",
       "      <td>&lt;p&gt;Factors are stored as numbers and a table o...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 tag  \\\n",
       "0  ['machine-learning', 'classification', 'evalua...   \n",
       "1  ['machine-learning', 'r', 'logistic-regression...   \n",
       "2                                  ['visualization']   \n",
       "3                                        ['bigdata']   \n",
       "4                            ['r', 'data-wrangling']   \n",
       "\n",
       "                                               title  \\\n",
       "0   How to improve an existing (trained) classifier?   \n",
       "1  Random Forest, Type - Regression, Calculation ...   \n",
       "2         How to analyze which site has most numbers   \n",
       "3                         Privacy through fake data?   \n",
       "4   When to choose character instead of factor in R?   \n",
       "\n",
       "                                                body  \n",
       "0  <p>Adding more data does not always help. Howe...  \n",
       "1  <p>MSE is measure of error of the overall regr...  \n",
       "2  <p>The most obvious way of visualizing this is...  \n",
       "3  <p>There's an inherent problem that anybody wh...  \n",
       "4  <p>Factors are stored as numbers and a table o...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print gensim.utils.simple_preprocess(d_set['body'][0])   \n",
    "d_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building corpus for Doc2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TaggedDocument(words=[u'adding', u'more', u'data', u'does', u'not', u'always', u'help', u'however', u'you', u'can', u'get', u'an', u'estimate', u'if', u'more', u'data', u'will', u'help', u'you', u'by', u'the', u'following', u'procedure', u'make', u'plot', u'on', u'the', u'axis', u'is', u'the', u'amount', u'of', u'training', u'examples', u'starting', u'at', u'one', u'example', u'per', u'class', u'going', u'to', u'wherever', u'you', u'are', u'currently', u'the', u'axis', u'shows', u'the', u'error', u'now', u'you', u'should', u'add', u'two', u'curves', u'training', u'and', u'test', u'error', u'for', u'low', u'the', u'training', u'error', u'should', u'be', u'very', u'low', u'almost', u'and', u'the', u'test', u'error', u'very', u'high', u'with', u'enough', u'data', u'they', u'should', u'be', u'about', u'the', u'same', u'by', u'plotting', u'those', u'curves', u'you', u'can', u'make', u'an', u'educated', u'guess', u'how', u'much', u'more', u'data', u'will', u'give', u'you', u'how', u'much', u'improvement', u'blockquote', u'when', u'doing', u'this', u'should', u'add', u'data', u'to', u'both', u'the', u'training', u'set', u'and', u'the', u'test', u'set', u'blockquote', u'depends', u'on', u'what', u'you', u'want', u'to', u'achieve', u'if', u'only', u'getting', u'better', u'classifier', u'then', u'you', u'can', u'only', u'add', u'it', u'to', u'the', u'training', u'set', u'however', u'if', u'you', u're', u'doing', u'this', u'in', u'scientific', u'setting', u'this', u'might', u'be', u'more', u'difficult', u'assume', u'that', u'your', u'test', u'set', u'is', u'of', u'reasonable', u'size', u'you', u'might', u'want', u'to', u'have', u'look', u'at', u'href', u'https', u'en', u'wikipedia', u'org', u'wiki', u'cross', u'validation_', u'statistics', u'rel', u'nofollow', u'cross', u'validation'], tags=[0]), TaggedDocument(words=[u'mse', u'is', u'measure', u'of', u'error', u'of', u'the', u'overall', u'regression', u'model', u'frac', u'sum', u'y_i', u'hat', u'y_i', u'for', u'an', u'important', u'variable', u'if', u'it', u'is', u'replaced', u'with', u'random', u'noise', u'you', u'would', u'imagine', u'mse', u'with', u'the', u'faulty', u'data', u'to', u'increase', u'strong', u'incmse', u'strong', u'incremental', u'mse', u'for', u'particular', u'variable', u'is', u'how', u'much', u'the', u'mse', u'will', u'increase', u'if', u'the', u'variable', u'is', u'completely', u'randomized', u'this', u'is', u'usually', u'computed', u'on', u'the', u'out', u'of', u'bag', u'data', u'hr', u'node', u'purity', u'is', u'measure', u'of', u'how', u'homogeneous', u'node', u'is', u'an', u'example', u'of', u'node', u'purity', u'is', u'information', u'entropy', u'p_', u'log', u'p_', u'p_', u'log', u'p_', u'if', u'there', u'are', u'two', u'classes', u'for', u'regression', u'models', u'node', u'em', u'im', u'em', u'purity', u'is', u'usually', u'taken', u'as', u'the', u'variance', u'in', u'node', u'everytime', u'you', u'split', u'node', u'you', u'do', u'it', u'to', u'make', u'the', u'new', u'nodes', u'homogeneous', u'hence', u'the', u'purity', u'increases', u'strong', u'incpurity', u'strong', u'of', u'variable', u'is', u'weighted', u'average', u'of', u'incremental', u'purity', u'because', u'of', u'each', u'split', u'by', u'this', u'variable', u'was', u'used', u'to', u'split', u'with', u'the', u'node', u'population', u'as', u'the', u'weight'], tags=[1])] \n",
      "\n",
      "[TaggedDocument(words=[u'how', u'to', u'improve', u'an', u'existing', u'trained', u'classifier'], tags=[0]), TaggedDocument(words=[u'random', u'forest', u'type', u'regression', u'calculation', u'of', u'importance', u'example'], tags=[1])]\n"
     ]
    }
   ],
   "source": [
    "body_corpus = []\n",
    "for i, line in enumerate(d_set['body']): \n",
    "    body_corpus.append(gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line), [i]))   \n",
    "print (body_corpus)[:2], '\\n'    \n",
    "\n",
    "title_corpus = []\n",
    "for i, line in enumerate(d_set['title']):\n",
    "    title_corpus.append(gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line), [i]))\n",
    "print (title_corpus)[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_for_body = gensim.models.doc2vec.Doc2Vec(size=30, min_count=2, iter=50)\n",
    "model_for_title = gensim.models.doc2vec.Doc2Vec(size=15, min_count=2, iter=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_for_body.build_vocab(body_corpus)\n",
    "model_for_title.build_vocab(title_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7330\n",
      "1014\n"
     ]
    }
   ],
   "source": [
    "print len(model_for_body.vocab)\n",
    "print len(model_for_title.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "231\n",
      "62\n"
     ]
    }
   ],
   "source": [
    "print model_for_body.vocab['regression'].count\n",
    "print model_for_title.vocab['regression'].count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## training Doc2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 36.2 s\n",
      "Wall time: 4.35 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "984400"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time model_for_body.train(body_corpus, total_examples=model_for_body.corpus_count) \n",
    "%time model_for_title.train(title_corpus, total_examples=model_for_title.corpus_count) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Taking user's query and processing them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.03127864  0.02093349  0.03002832  0.00866496 -0.00605498  0.01654723\n",
      "  0.00773027 -0.0001666  -0.00406452  0.06625747  0.03237529  0.02781813\n",
      "  0.04272798 -0.01236193 -0.0083826   0.00299702 -0.07634215 -0.0488191\n",
      " -0.03117685 -0.03597934 -0.00298677  0.00257462  0.01823049 -0.02434584\n",
      "  0.01248353 -0.03326915 -0.01310654  0.04946647 -0.01679867  0.00603149] \n",
      "\n",
      "[ 0.09981605  0.12393562 -0.06997055 -0.07479089  0.01536176 -0.04371031\n",
      " -0.00609764 -0.06227994  0.00910525  0.0320807   0.17170812  0.10681643\n",
      "  0.27942887  0.15855436  0.03077954]\n"
     ]
    }
   ],
   "source": [
    "user_query = \"why is overfitting bad in machine learning ?\"  \n",
    "\n",
    "user_body_vec = model_for_body.infer_vector(gensim.utils.simple_preprocess(user_query))\n",
    "user_title_vec = model_for_title.infer_vector(gensim.utils.simple_preprocess(user_query))\n",
    "\n",
    "print user_body_vec, '\\n'\n",
    "print user_title_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieving top 5, most relevant documents corresponding to any user' s query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "body_sims:  [(618, 0.5026001930236816), (1193, 0.4658726155757904), (411, 0.43993499875068665), (883, 0.43816816806793213), (78, 0.41403472423553467)] \n",
      "\n",
      "title_sims:  [(1011, 0.8406146168708801), (356, 0.7910738587379456), (836, 0.6973468661308289), (534, 0.6693466305732727), (229, 0.6513209342956543)] \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "overfitting is em empirically em bad suppose you have data set which you split in two test and training an overfitted model is one that performs much worse on the test dataset than on training dataset it is often observed that models like that also in general perform worse on additional new test datasets than models which are not overfitted one way to understand that intuitively is that model may use some relevant parts of the data signal and some irrelevant parts noise an overfitted model uses more of the noise which increases its performance in the case of known noise training data and decreases its performance in the case of novel noise test data the difference in performance between training and test data indicates how much noise the model picks up and picking up noise directly translates into worse performance on test data including future data summary overfitting is bad by definition this has not much to do with either complexity or ability to generalize but rather has to do with mistaking noise for signal on the ability to generalize part of the question it is very possible to have model which has inherently limited ability to generalize due to the structure of the model for example linear svm but is still prone to overfitting in sense overfitting is just one way that generalization may fail \n",
      "\n",
      "for specific model you feed it data choose the features choose hyperparameters etcetera compared to the reality it makes three types of mistakes ul li bias due to too low model complexity sampling bias in your data li li variance due to noise in your data overfitting of your data li li randomness of the reality you are trying to predict or lack of predictive features in your dataset li ul ensembles average out number of these models the bias due to sampling bias will not be fixed for obvious reasons it can fix some of the model complexity bias however the variance mistakes that are made are very different over your different models especially low correlated models make very different mistakes in this areas certain models perform well in certain parts of your feature space by averaging out these models you reduce this variance quite bit this is why ensembles shine \n",
      "\n",
      "it is so called because it classifies the linear sequence and not because the structure of the graph \n",
      "\n",
      "it little hasty to make too many conclusions about your data based on what you presented here at the end of the day all the information you have right now is that gbt did not work well for this prediction problem and this metric summed up by single rmse comparison this isn very much information it could be that this is bad dataset for gbt and some other model would work it could be that the label can be predicted from these features with any model or there could be some error in model setup validation recommend checking the following hypotheses maybe with your dataset size and the features you have gbt isn very high performance model try something completely different maybe just simple linear regression or random forest or gbdt with very different parameter settings or something else this will help you diagnose whether it an issue with choice of models or with something else if few very different approaches give you roughly similar results you ll know that it not the model choice that is causing these results and if one of those models behaves differently then that gives you additional information to help diagnose the issue maybe there some issue with model setup and validation would recommend doing some exploration to get some intuition as to whether the rmse you re getting is reasonable or whether you should expect better your post contained very little detail about what the data actually represents what you know about the features and labels etc perhaps you know those things but didn include them here but if not you should go back and try to get additional understanding of the data before continuing look at some random data points plot the columns against the target look at the histograms of your features and labels that sort of thing there no substitute for looking at the data maybe there just aren enough data points to justify complex models when you have low numbers of data points lt simpler parametric model built with domain expertise and knowledge of what the features are may very well outperform nonparametric model \n",
      "\n",
      "have been doing similar project in my college have classroom and supposed to collect data like temp humidity light occupancy etc assuming that you have worked with sensors and motes to use going to explain rest of the structure you need sensor network setup and like you said you have done it these sensor networks generally do not send data directly over internet so you need gateway that can collect data from sensors and send it over internet to local server on server side you need rest api and you could use any language to develop it and use php find it very easy to use and develop using php this rest api shall receive data from gateway and store it into database use mysql database because amount of data is not so big for us but if your data is big enough you can use big data nosql tool like mongodb or so whatever type of database you use structure remains same for sending data from gateway to server you can use protocols like http or mqtt whichever you feel comfortable what do is have wsn controller that sends data over usb to gateway then gateway sends data to server over ethernet so had to develop usb to ethernet gateway if you can just take two uart terminals out of your controller you can build uart to ethernet gateway using any microcontroller or even arduino ethernet shield would work in that case in my case data is sensed periodically but as you said you are sensing data when event of interest occurs then you can use poisson distribution method over periodically collected data to predict what is average number of events per day and then you can decide if your data is big or not \n",
      "\n"
     ]
    }
   ],
   "source": [
    "body_sims = model_for_body.docvecs.most_similar([user_body_vec],topn=5) \n",
    "print \"body_sims: \", body_sims, '\\n'  \n",
    "# for i in body_sims:   \n",
    "#     print (' '.join(body_corpus[i[0]].words)), '\\n'    \n",
    "    \n",
    "    \n",
    "title_sims = model_for_title.docvecs.most_similar([user_title_vec],topn=5) \n",
    "print \"title_sims: \", title_sims, '\\n'  \n",
    "# for i in title_sims:   \n",
    "#     print (' '.join(title_corpus[i[0]].words)), '\\n'\n",
    "print '\\n\\n'\n",
    "    \n",
    "for i in title_sims:\n",
    "    print ' '.join(body_corpus[i[0]].words), '\\n'      # infering body from most similar title\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving Document retrieval model on Disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# store the model to mmap-able files\n",
    "model_for_title.save('my_model_for_title.doc2vec')\n",
    "\n",
    "# load the model back\n",
    "model_for_title_loaded = gensim.models.doc2vec.Doc2Vec.load('my_model_for_title.doc2vec') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1011, 0.8406146168708801),\n",
       " (356, 0.791073739528656),\n",
       " (836, 0.6973468661308289),\n",
       " (534, 0.6693466305732727),\n",
       " (229, 0.6513208746910095)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_for_title_loaded.docvecs.most_similar([user_title_vec],topn=5) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}