assignment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PFA dataset Classifier\n",
    "\n",
    "Objective: Build classifiers for severity, category and classification given that we have utterances."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mukesh/miniconda3/envs/statoil/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import enchant\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import re\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "from string import punctuation\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, CuDNNLSTM,GlobalMaxPool1D, Embedding, Dropout, Activation, SpatialDropout1D, Bidirectional\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import Model\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint,ReduceLROnPlateau\n",
    "from keras.optimizers import RMSprop, nadam, adam\n",
    "from keras.utils import np_utils\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='data/'\n",
    "train=path+'train_CD.csv'\n",
    "\n",
    "embedding_file=path+'wiki.en.vec' #fasttext word embeddings\n",
    "max_words=100000\n",
    "max_seq_len=60\n",
    "embedding_dim=300\n",
    "\n",
    "num_gru = 128\n",
    "num_dense = 64\n",
    "rate_drop_dense = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mukesh/miniconda3/envs/statoil/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2728: DtypeWarning: Columns (3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Description</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Category</th>\n",
       "      <th>Severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>** PROBLEM Service Alert: USSGWIFIMGMT001/USDA...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1,225</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Please Collect ID Card/Access Card for - Rachn...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1,451</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Install 32-bit TortoiseSVN on my machine</td>\n",
       "      <td>1.0</td>\n",
       "      <td>970</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tubelight above the cubical is not working. Pl...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>174</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Require Toner</td>\n",
       "      <td>1.0</td>\n",
       "      <td>920</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Description  Classification Category  \\\n",
       "0  ** PROBLEM Service Alert: USSGWIFIMGMT001/USDA...             8.0    1,225   \n",
       "1  Please Collect ID Card/Access Card for - Rachn...             1.0    1,451   \n",
       "2           Install 32-bit TortoiseSVN on my machine             1.0      970   \n",
       "3  Tubelight above the cubical is not working. Pl...            10.0      174   \n",
       "4                                      Require Toner             1.0      920   \n",
       "\n",
       "  Severity  \n",
       "0        3  \n",
       "1      NaN  \n",
       "2        4  \n",
       "3       12  \n",
       "4        4  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(train)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "263394"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Null values\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Description         85\n",
       "Classification     672\n",
       "Category           655\n",
       "Severity          2320\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Null values\")\n",
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "260910"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=data.dropna()\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    260910.000000\n",
       "mean         36.381315\n",
       "std          47.741645\n",
       "min           1.000000\n",
       "25%           1.000000\n",
       "50%           8.000000\n",
       "75%         109.000000\n",
       "max         173.000000\n",
       "Name: Classification, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Classification\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     260910\n",
       "unique       719\n",
       "top        1,148\n",
       "freq       18104\n",
       "Name: Category, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Category\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     260910\n",
       "unique        45\n",
       "top            4\n",
       "freq      117673\n",
       "Name: Severity, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Severity\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are few values in classification column having commas e.g 1,234\n",
    "# we need to convert it to int or float\n",
    "def remove_comma(arr):\n",
    "    new_arr=[]\n",
    "    for p in arr:\n",
    "        try:\n",
    "            q=p.split(',')\n",
    "            #print(q)\n",
    "            q=''.join(q)\n",
    "            new_arr.append(q)\n",
    "        except:\n",
    "            new_arr.append(p)\n",
    "    return new_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"Category\"]=remove_comma(data[\"Category\"].values)\n",
    "data[\"Severity\"]=remove_comma(data[\"Severity\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"Severity\"]=data[\"Severity\"].astype(np.float64)\n",
    "data[\"Category\"]=data[\"Category\"].astype(np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Classification</th>\n",
       "      <th>Category</th>\n",
       "      <th>Severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>260910.000000</td>\n",
       "      <td>260910.000000</td>\n",
       "      <td>260910.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>36.381315</td>\n",
       "      <td>768.815948</td>\n",
       "      <td>5.141681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>47.741645</td>\n",
       "      <td>416.174000</td>\n",
       "      <td>5.639528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>323.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>926.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>109.000000</td>\n",
       "      <td>1083.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>173.000000</td>\n",
       "      <td>2466.000000</td>\n",
       "      <td>1095.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Classification       Category       Severity\n",
       "count   260910.000000  260910.000000  260910.000000\n",
       "mean        36.381315     768.815948       5.141681\n",
       "std         47.741645     416.174000       5.639528\n",
       "min          1.000000       1.000000       1.000000\n",
       "25%          1.000000     323.000000       3.000000\n",
       "50%          8.000000     926.000000       4.000000\n",
       "75%        109.000000    1083.000000       4.000000\n",
       "max        173.000000    2466.000000    1095.000000"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(46.98753976466981, 28.361221781943883, 270)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lens=data.Description.str.len()\n",
    "lens.mean(), lens.std(), lens.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f5acc3c4ba8>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFcZJREFUeJzt3X+MXfV55/H3U7umbtrEEKqRZVtr\nd2N15eBtS0bgVapoFHfBQFWzEomMUDFZN1YVaNNqqtY0f7hKgkR216VBIkje2hsTRXFYmgprMet6\nCVdR/rADJBRjKGEKThnL4DY20CFK2KHP/nG/3lyGGfs79177zhzeL+nK5z7ne875PhzEx+fHDJGZ\nSJJU42cGPQFJ0vxhaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqrZw0BPot0sv\nvTRXrlw56+3eeOMN3vOe9/R/QnNI03tsen/Q/B6b3h/M3R6feOKJf87MXzrXuMaFxsqVK3n88cdn\nvV2r1WJkZKT/E5pDmt5j0/uD5vfY9P5g7vYYET+oGeftKUlSNUNDklTtnKEREbsj4mREPN1R+68R\n8fcR8VRE/E1ELOlYd3tEjEXEcxFxdUd9Q6mNRcS2jvqqiDhc6l+PiEWlflH5PlbWr+xX05Kk7tRc\naXwZ2DCldhC4LDP/PfB94HaAiFgDbAI+WLb5UkQsiIgFwD3ANcAa4MYyFuALwF2Z+QHgNLCl1LcA\np0v9rjJOkjRA5wyNzPwWcGpK7W8zc7J8PQQsL8sbgb2Z+ZPMfBEYA64on7HMfCEz3wT2AhsjIoCP\nAg+U7fcA13fsa09ZfgBYX8ZLkgakH880/jPwcFleBrzUsW681Gaqvx94tSOAztTftq+y/rUyXpI0\nID29chsRnwEmga/2Zzpdz2MrsBVgaGiIVqs1631MTEx0td180vQem94fNL/HpvcH87/HrkMjIm4B\nfgtYnz/9f8YeB1Z0DFteasxQ/yGwJCIWlquJzvFn9jUeEQuB95Xx75CZO4GdAMPDw9nNO9Bz9d3p\nfmp6j03vD5rfY9P7g/nfY1e3pyJiA/AnwG9n5o86Vu0DNpU3n1YBq4HvAI8Bq8ubUotoPyzfV8Lm\nUeCGsv1m4MGOfW0uyzcA30z/h+aSNFDnvNKIiK8BI8ClETEObKf9ttRFwMHybPpQZv5eZh6NiPuB\nZ2jftro1M98q+7kNOAAsAHZn5tFyiD8F9kbE54HvAbtKfRfwlYgYo/0gflMf+p2zVm576LwfY3Tt\nJLdMOc6xO68778eV1BznDI3MvHGa8q5pamfG3wHcMU19P7B/mvoLtN+umlr/MfCxc81PknTh+BPh\nkqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhka\nkqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhka\nkqRqhoYkqdo5QyMidkfEyYh4uqN2SUQcjIjny58Xl3pExN0RMRYRT0XE5R3bbC7jn4+IzR31D0XE\nkbLN3RERZzuGJGlwaq40vgxsmFLbBjySmauBR8p3gGuA1eWzFbgX2gEAbAeuBK4AtneEwL3AJzu2\n23COY0iSBuScoZGZ3wJOTSlvBPaU5T3A9R31+7LtELAkIpYCVwMHM/NUZp4GDgIbyrr3ZuahzEzg\nvin7mu4YkqQB6faZxlBmnijLLwNDZXkZ8FLHuPFSO1t9fJr62Y4hSRqQhb3uIDMzIrIfk+n2GBGx\nlfbtMIaGhmi1WrM+xsTERFfb9cvo2snzfoyhxe88ziB77rdBn8MLoek9Nr0/mP89dhsar0TE0sw8\nUW4xnSz148CKjnHLS+04MDKl3ir15dOMP9sx3iEzdwI7AYaHh3NkZGSmoTNqtVp0s12/3LLtofN+\njNG1k+w48vZTfuymkfN+3Atl0OfwQmh6j03vD+Z/j93entoHnHkDajPwYEf95vIW1TrgtXKL6QBw\nVURcXB6AXwUcKOtej4h15a2pm6fsa7pjSJIG5JxXGhHxNdpXCZdGxDjtt6DuBO6PiC3AD4CPl+H7\ngWuBMeBHwCcAMvNURHwOeKyM+2xmnnm4/inab2gtBh4uH85yDEnSgJwzNDLzxhlWrZ9mbAK3zrCf\n3cDuaeqPA5dNU//hdMeQJA2OPxEuSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKma\noSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKma\noSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqvUUGhHxRxFxNCKejoivRcTPRcSqiDgcEWMR8fWI\nWFTGXlS+j5X1Kzv2c3upPxcRV3fUN5TaWERs62WukqTedR0aEbEM+ANgODMvAxYAm4AvAHdl5geA\n08CWsskW4HSp31XGERFrynYfBDYAX4qIBRGxALgHuAZYA9xYxkqSBqTX21MLgcURsRD4eeAE8FHg\ngbJ+D3B9Wd5YvlPWr4+IKPW9mfmTzHwRGAOuKJ+xzHwhM98E9paxkqQB6To0MvM48N+Af6QdFq8B\nTwCvZuZkGTYOLCvLy4CXyraTZfz7O+tTtpmpLkkakIXdbhgRF9P+m/8q4FXgf9K+vXTBRcRWYCvA\n0NAQrVZr1vuYmJjoart+GV07ee5BPRpa/M7jDLLnfhv0ObwQmt5j0/uD+d9j16EB/CbwYmb+E0BE\nfAP4MLAkIhaWq4nlwPEy/jiwAhgvt7PeB/ywo35G5zYz1d8mM3cCOwGGh4dzZGRk1s20Wi262a5f\nbtn20Hk/xujaSXYcefspP3bTyHk/7oUy6HN4ITS9x6b3B/O/x16eafwjsC4ifr48m1gPPAM8CtxQ\nxmwGHizL+8p3yvpvZmaW+qbydtUqYDXwHeAxYHV5G2sR7Yfl+3qYrySpR11faWTm4Yh4APguMAl8\nj/bf9h8C9kbE50ttV9lkF/CViBgDTtEOATLzaETcTztwJoFbM/MtgIi4DThA+82s3Zl5tNv5SpJ6\n18vtKTJzO7B9SvkF2m8+TR37Y+BjM+znDuCOaer7gf29zFGS1D/+RLgkqZqhIUmqZmhIkqoZGpKk\naoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKk\naoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmq1lNoRMSSiHgg\nIv4+Ip6NiP8QEZdExMGIeL78eXEZGxFxd0SMRcRTEXF5x342l/HPR8TmjvqHIuJI2ebuiIhe5itJ\n6k2vVxpfBP53Zv474FeBZ4FtwCOZuRp4pHwHuAZYXT5bgXsBIuISYDtwJXAFsP1M0JQxn+zYbkOP\n85Uk9aDr0IiI9wEfAXYBZOabmfkqsBHYU4btAa4vyxuB+7LtELAkIpYCVwMHM/NUZp4GDgIbyrr3\nZuahzEzgvo59SZIGYGEP264C/gn4HxHxq8ATwKeBocw8Uca8DAyV5WXASx3bj5fa2erj09TPm5Xb\nHjqfu5ekea+X0FgIXA78fmYejogv8tNbUQBkZkZE9jLBGhGxlfYtL4aGhmi1WrPex8TEBKNr3+rz\nzOaWocUwunbybbVu/lnNVRMTE43qZzpN77Hp/cH877GX0BgHxjPzcPn+AO3QeCUilmbmiXKL6WRZ\nfxxY0bH98lI7DoxMqbdKffk0498hM3cCOwGGh4dzZGRkumFn1Wq12PHtN2a93XwyunaSHUfefsqP\n3TQymMmcB61Wi27O/XzS9B6b3h/M/x67fqaRmS8DL0XEr5TSeuAZYB9w5g2ozcCDZXkfcHN5i2od\n8Fq5jXUAuCoiLi4PwK8CDpR1r0fEuvLW1M0d+5IkDUAvVxoAvw98NSIWAS8An6AdRPdHxBbgB8DH\ny9j9wLXAGPCjMpbMPBURnwMeK+M+m5mnyvKngC8Di4GHy0eSNCA9hUZmPgkMT7Nq/TRjE7h1hv3s\nBnZPU38cuKyXOUqS+sefCJckVTM0JEnVen2moXluUD+bcuzO6wZyXEm98UpDklTN0JAkVTM0JEnV\nDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnV\nDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdV6Do2IWBAR34uI/1W+r4qIwxEx\nFhFfj4hFpX5R+T5W1q/s2Mftpf5cRFzdUd9QamMRsa3XuUqSetOPK41PA892fP8CcFdmfgA4DWwp\n9S3A6VK/q4wjItYAm4APAhuAL5UgWgDcA1wDrAFuLGMlSQPSU2hExHLgOuCvyvcAPgo8UIbsAa4v\nyxvLd8r69WX8RmBvZv4kM18ExoArymcsM1/IzDeBvWWsJGlAer3S+EvgT4B/Ld/fD7yamZPl+ziw\nrCwvA14CKOtfK+P/f33KNjPVJUkDsrDbDSPit4CTmflERIz0b0pdzWUrsBVgaGiIVqs1631MTEww\nuvatPs9sbhlaDKNrJ8898ALo5hydy8TExHnZ71zS9B6b3h/M/x67Dg3gw8BvR8S1wM8B7wW+CCyJ\niIXlamI5cLyMPw6sAMYjYiHwPuCHHfUzOreZqf42mbkT2AkwPDycIyMjs26m1Wqx49tvzHq7+WR0\n7SQ7jvRyyvvn2E0jfd9nq9Wim3M/nzS9x6b3B/O/x65vT2Xm7Zm5PDNX0n6Q/c3MvAl4FLihDNsM\nPFiW95XvlPXfzMws9U3l7apVwGrgO8BjwOryNtaicox93c5XktS78/HXzj8F9kbE54HvAbtKfRfw\nlYgYA07RDgEy82hE3A88A0wCt2bmWwARcRtwAFgA7M7Mo+dhvpKkSn0JjcxsAa2y/ALtN5+mjvkx\n8LEZtr8DuGOa+n5gfz/mKEnqnT8RLkmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSp\nmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSp\nmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqp1HRoRsSIiHo2IZyLiaER8utQviYiDEfF8+fPi\nUo+IuDsixiLiqYi4vGNfm8v45yNic0f9QxFxpGxzd0REL81KknrTy5XGJDCamWuAdcCtEbEG2AY8\nkpmrgUfKd4BrgNXlsxW4F9ohA2wHrgSuALafCZoy5pMd223oYb6SpB51HRqZeSIzv1uW/wV4FlgG\nbAT2lGF7gOvL8kbgvmw7BCyJiKXA1cDBzDyVmaeBg8CGsu69mXkoMxO4r2NfkqQB6MszjYhYCfw6\ncBgYyswTZdXLwFBZXga81LHZeKmdrT4+TV2SNCALe91BRPwC8NfAH2bm652PHTIzIyJ7PUbFHLbS\nvuXF0NAQrVZr1vuYmJhgdO1bfZ7Z3DK0GEbXTg56GgBdnaNzmZiYOC/7nUua3mPT+4P532NPoRER\nP0s7ML6amd8o5VciYmlmnii3mE6W+nFgRcfmy0vtODAypd4q9eXTjH+HzNwJ7AQYHh7OkZGR6Yad\nVavVYse335j1dvPJ6NpJdhzp+e8JfXHsppG+77PVatHNuZ9Pmt5j0/uD+d9jL29PBbALeDYz/6Jj\n1T7gzBtQm4EHO+o3l7eo1gGvldtYB4CrIuLi8gD8KuBAWfd6RKwrx7q5Y1+SpAHo5a+dHwZ+BzgS\nEU+W2p8BdwL3R8QW4AfAx8u6/cC1wBjwI+ATAJl5KiI+BzxWxn02M0+V5U8BXwYWAw+XjyRpQLoO\njcz8NjDTz02sn2Z8ArfOsK/dwO5p6o8Dl3U7R81dK7c91Pd9jq6d5JaK/R6787q+H1t6t/AnwiVJ\n1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ\n1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrWFg56AdKGt3PbQQI57\n7M7rBnJcqZ+80pAkVTM0JEnVDA1JUrU5HxoRsSEinouIsYjYNuj5SNK72Zx+EB4RC4B7gP8IjAOP\nRcS+zHxmsDOTZq8fD+BH105ySxf78SG8+mVOhwZwBTCWmS8ARMReYCNgaEiz4Btj6pe5HhrLgJc6\nvo8DVw5oLpJmabZh1e2V1HxyPnu8ECEdmXneD9KtiLgB2JCZv1u+/w5wZWbeNmXcVmBr+forwHNd\nHO5S4J97mO580PQem94fNL/HpvcHc7fHf5OZv3SuQXP9SuM4sKLj+/JSe5vM3Ans7OVAEfF4Zg73\nso+5ruk9Nr0/aH6PTe8P5n+Pc/3tqceA1RGxKiIWAZuAfQOekyS9a83pK43MnIyI24ADwAJgd2Ye\nHfC0JOlda06HBkBm7gf2X4BD9XR7a55oeo9N7w+a32PT+4N53uOcfhAuSZpb5vozDUnSHGJo0Mxf\nVRIRxyLiSEQ8GRGPl9olEXEwIp4vf1486HnORkTsjoiTEfF0R23anqLt7nJOn4qIywc38zoz9Pfn\nEXG8nMcnI+LajnW3l/6ei4irBzPr2YmIFRHxaEQ8ExFHI+LTpd6I83iW/ppzHjPzXf2h/YD9H4Bf\nBhYBfwesGfS8+tDXMeDSKbX/Amwry9uALwx6nrPs6SPA5cDT5+oJuBZ4GAhgHXB40PPvsr8/B/54\nmrFryr+rFwGryr/DCwbdQ0WPS4HLy/IvAt8vvTTiPJ6lv8acR680On5VSWa+CZz5VSVNtBHYU5b3\nANcPcC6zlpnfAk5NKc/U00bgvmw7BCyJiKUXZqbdmaG/mWwE9mbmTzLzRWCM9r/Lc1pmnsjM75bl\nfwGepf2bHxpxHs/S30zm3Xk0NKb/VSVnO8nzRQJ/GxFPlJ+YBxjKzBNl+WVgaDBT66uZemrSeb2t\n3JrZ3XFLcd73FxErgV8HDtPA8zilP2jIeTQ0mus3MvNy4Brg1oj4SOfKbF8bN+rVuSb2BNwL/Fvg\n14ATwI7BTqc/IuIXgL8G/jAzX+9c14TzOE1/jTmPhkblryqZbzLzePnzJPA3tC95XzlzaV/+PDm4\nGfbNTD014rxm5iuZ+VZm/ivw3/nprYt5219E/Czt/6B+NTO/UcqNOY/T9dek82hoNPBXlUTEeyLi\nF88sA1cBT9Pua3MZthl4cDAz7KuZetoH3FzevlkHvNZx+2PemHL//j/RPo/Q7m9TRFwUEauA1cB3\nLvT8ZisiAtgFPJuZf9GxqhHncab+GnUeB/0kfi58aL+h8X3aby58ZtDz6UM/v0z7jYy/A46e6Ql4\nP/AI8Dzwf4BLBj3XWfb1NdqX9v+X9r3fLTP1RPttm3vKOT0CDA96/l3295Uy/6do/wdmacf4z5T+\nngOuGfT8K3v8Ddq3np4Cniyfa5tyHs/SX2POoz8RLkmq5u0pSVI1Q0OSVM3QkCRVMzQkSdUMDUlS\nNUNDklTN0JAkVTM0JEnV/h9mFzeB2K0kZgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5acc39a908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lens.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    260910.000000\n",
       "mean         46.987540\n",
       "std          28.361222\n",
       "min           1.000000\n",
       "25%          29.000000\n",
       "50%          40.000000\n",
       "75%          62.000000\n",
       "max         270.000000\n",
       "Name: Description, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lens.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Container object of 65 artists>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAER1JREFUeJzt3X+s3XV9x/Hna60wReWHNIS1zFtn\n51JNNrBBFn/8IQYKOss2NRAjnWOSRdh0P+LqTIbxRyL7oZPEYZh0gnECQw3NwCFD3bI/QMoPhYLI\nlR/SpkC1CGZOsfreH+fT7XA/97anP27PKff5SE7u9/v+fr7nvM+n33tf93zP99ymqpAkadgvjLsB\nSdLkMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUWTzuBvbW0UcfXVNTU+NuQ5IO\nGrfeeuv3qmrJKGMP2nCYmppi48aN425Dkg4aSR4adaynlSRJHcNBktQxHCRJHcNBktQxHCRJHcNB\nktQxHCRJHcNBktQxHCRJnQUZDlPrrh13C5I00RZkOEiSds1wkCR1DAdJUsdwkCR1DAdJUsdwkCR1\nDAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJ\nUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUmekcEjyJ0k2JbkryeeS/GKS5Ulu\nTjKd5Mokh7Sxh7b16bZ9auh+3tvq9yY5dai+utWmk6zb309SkrRndhsOSZYCfwysqqqXAYuAM4EL\ngY9V1YuBx4Fz2i7nAI+3+sfaOJKsbPu9FFgN/EOSRUkWAZ8ATgNWAme1sZKkMRn1tNJi4NlJFgPP\nAbYCrwWubtsvA85oy2vaOm37yUnS6ldU1U+q6gFgGjix3aar6v6qegq4oo2VJI3JbsOhqrYAfwt8\nl0EoPAHcCvygqna0YZuBpW15KfBw23dHG/+C4fqMfeaqd5Kcm2Rjko3btm0b5flJkvbCKKeVjmTw\nm/xy4JeAwxicFjrgquqSqlpVVauWLFkyjhYkaUEY5bTS64AHqmpbVf0U+ALwSuCIdpoJYBmwpS1v\nAY4DaNsPB74/XJ+xz1x1SdKYjBIO3wVOSvKc9t7BycDdwFeBN7Uxa4Fr2vKGtk7b/pWqqlY/s13N\ntBxYAXwduAVY0a5+OoTBm9Yb9v2pSZL21uLdDaiqm5NcDdwG7ABuBy4BrgWuSPKhVru07XIp8Jkk\n08B2Bj/sqapNSa5iECw7gPOq6mcASc4HrmdwJdT6qtq0/56iJGlP7TYcAKrqAuCCGeX7GVxpNHPs\nj4E3z3E/HwY+PEv9OuC6UXqRJM0/PyEtSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKk\njuEgSeoYDtICMbXu2nG3oIOI4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ\n6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgO\nkqSO4SBJ6hgOkqTOSOGQ5IgkVyf5VpJ7kvxmkqOS3JDkvvb1yDY2SS5KMp3km0lOGLqftW38fUnW\nDtVfnuTOts9FSbL/n6okaVSjvnL4OPBvVfVrwK8D9wDrgBuragVwY1sHOA1Y0W7nAhcDJDkKuAB4\nBXAicMHOQGlj3jG03+p9e1qSpH2x23BIcjjwGuBSgKp6qqp+AKwBLmvDLgPOaMtrgMtr4CbgiCTH\nAqcCN1TV9qp6HLgBWN22Pb+qbqqqAi4fui9J0hiM8sphObAN+Kcktyf5VJLDgGOqamsb8whwTFte\nCjw8tP/mVttVffMsdUnSmIwSDouBE4CLq+p44L/5/1NIALTf+Gv/t/d0Sc5NsjHJxm3bts33w0nS\ngjVKOGwGNlfVzW39agZh8Wg7JUT7+ljbvgU4bmj/Za22q/qyWeqdqrqkqlZV1aolS5aM0LokaW/s\nNhyq6hHg4SQvaaWTgbuBDcDOK47WAte05Q3A2e2qpZOAJ9rpp+uBU5Ic2d6IPgW4vm17MslJ7Sql\ns4fuS5I0BotHHPdHwGeTHALcD7ydQbBcleQc4CHgLW3sdcDpwDTwozaWqtqe5IPALW3cB6pqe1t+\nJ/Bp4NnAl9pNkjQmI4VDVd0BrJpl08mzjC3gvDnuZz2wfpb6RuBlo/QiSZp/fkJaktQxHCRJHcNB\nktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQx\nHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJ\nHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQZORySLEpye5J/bevLk9ycZDrJlUkOafVD\n2/p02z41dB/vbfV7k5w6VF/datNJ1u2/pydJ2ht78srhXcA9Q+sXAh+rqhcDjwPntPo5wOOt/rE2\njiQrgTOBlwKrgX9ogbMI+ARwGrASOKuNlSSNyUjhkGQZ8HrgU209wGuBq9uQy4Az2vKatk7bfnIb\nvwa4oqp+UlUPANPAie02XVX3V9VTwBVtrCRpTEZ95fD3wHuAn7f1FwA/qKodbX0zsLQtLwUeBmjb\nn2jj/68+Y5+56pKkMdltOCR5A/BYVd16APrZXS/nJtmYZOO2bdvG3Y4kPWON8srhlcAbkzzI4JTP\na4GPA0ckWdzGLAO2tOUtwHEAbfvhwPeH6zP2maveqapLqmpVVa1asmTJCK1LkvbGbsOhqt5bVcuq\naorBG8pfqaq3Al8F3tSGrQWuacsb2jpt+1eqqlr9zHY103JgBfB14BZgRbv66ZD2GBv2y7OTJO2V\nxbsfMqe/AK5I8iHgduDSVr8U+EySaWA7gx/2VNWmJFcBdwM7gPOq6mcASc4HrgcWAeuratM+9CVJ\n2kd7FA5V9TXga235fgZXGs0c82PgzXPs/2Hgw7PUrwOu25NeJEnzx09IS5I6hoMkqWM4SJI6hoMk\nqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4\nSJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6\nhoMkqWM4SJI6hoMkqWM4zGFq3bXjbkGSxsZwkCR1DAdJUme34ZDkuCRfTXJ3kk1J3tXqRyW5Icl9\n7euRrZ4kFyWZTvLNJCcM3dfaNv6+JGuH6i9Pcmfb56IkmY8nK0kazSivHHYAf1ZVK4GTgPOSrATW\nATdW1QrgxrYOcBqwot3OBS6GQZgAFwCvAE4ELtgZKG3MO4b2W73vT02StLd2Gw5VtbWqbmvLPwTu\nAZYCa4DL2rDLgDPa8hrg8hq4CTgiybHAqcANVbW9qh4HbgBWt23Pr6qbqqqAy4fuS5I0Bnv0nkOS\nKeB44GbgmKra2jY9AhzTlpcCDw/ttrnVdlXfPEtdkjQmI4dDkucCnwfeXVVPDm9rv/HXfu5tth7O\nTbIxycZt27bN98NJ0oI1UjgkeRaDYPhsVX2hlR9tp4RoXx9r9S3AcUO7L2u1XdWXzVLvVNUlVbWq\nqlYtWbJklNYlSXthlKuVAlwK3FNVHx3atAHYecXRWuCaofrZ7aqlk4An2umn64FTkhzZ3og+Bbi+\nbXsyyUntsc4eui9J0hgsHmHMK4G3AXcmuaPV/hL4CHBVknOAh4C3tG3XAacD08CPgLcDVNX2JB8E\nbmnjPlBV29vyO4FPA88GvtRuE2Pnp6Uf/Mjrx9yJJB0Yuw2HqvovYK7PHZw8y/gCzpvjvtYD62ep\nbwRetrteJEkHhp+Q3gv+3SVJz3SGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqG\ngySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzjsJ/7vcJKeSQwHSVLHcJAk\ndQwHSVLHcJAkdQwHSVLHcNgDXpEkaaEwHCRJHcNBktQxHA4AT0dJOtgYDpKkjuEgSeoYDvNgat21\nnkqSdFAzHCRJHcNBktQxHCRJHcNBktRZPO4GdkqyGvg4sAj4VFV9ZMwtSc8IXhyhvTERrxySLAI+\nAZwGrATOSrJyvF1J0sI1EeEAnAhMV9X9VfUUcAWwZsw9SfPK3+g1ySYlHJYCDw+tb261eTX8zbmn\n36h7+1mGmfvt6j784XHw2tt/uz09rnaO35P9Zht3oPrVwSNVNe4eSPImYHVV/UFbfxvwiqo6f8a4\nc4Fz2+pLgHv34uGOBr63D+0eaPY7v+x3ftnv/NrTfl9YVUtGGTgpb0hvAY4bWl/Wak9TVZcAl+zL\nAyXZWFWr9uU+DiT7nV/2O7/sd37NZ7+TclrpFmBFkuVJDgHOBDaMuSdJWrAm4pVDVe1Icj5wPYNL\nWddX1aYxtyVJC9ZEhANAVV0HXHcAHmqfTkuNgf3OL/udX/Y7v+at34l4Q1qSNFkm5T0HSdIEWTDh\nkGR1knuTTCdZN+5+ZkpyXJKvJrk7yaYk72r19yfZkuSOdjt93L3ulOTBJHe2vja22lFJbkhyX/t6\n5Lj7BEjykqE5vCPJk0nePWnzm2R9kseS3DVUm3VOM3BRO6a/meSECej1b5J8q/XzxSRHtPpUkv8Z\nmudPHshed9PznMdAkve2+b03yakT0u+VQ70+mOSOVt+/c1xVz/gbgze5vwO8CDgE+Aawctx9zejx\nWOCEtvw84NsM/pTI+4E/H3d/c/T8IHD0jNpfA+va8jrgwnH3Ocfx8AjwwkmbX+A1wAnAXbubU+B0\n4EtAgJOAmyeg11OAxW35wqFep4bHTdj8znoMtO+/bwCHAsvbz5BF4+53xva/A/5qPuZ4obxymPg/\nz1FVW6vqtrb8Q+AeDsCnxOfBGuCytnwZcMYYe5nLycB3quqhcTcyU1X9J7B9RnmuOV0DXF4DNwFH\nJDn2wHQ6e69V9eWq2tFWb2LwmaWJMcf8zmUNcEVV/aSqHgCmGfwsOWB21W+SAG8BPjcfj71QwmEs\nf55jbyWZAo4Hbm6l89vL9PWTcpqmKeDLSW5tn14HOKaqtrblR4BjxtPaLp3J07+hJnV+d5prTif9\nuP59Bq9sdlqe5PYk/5Hk1eNqag6zHQOTPr+vBh6tqvuGavttjhdKOBw0kjwX+Dzw7qp6ErgY+BXg\nN4CtDF5GTopXVdUJDP6a7nlJXjO8sQavdSfqcrj2Ics3Av/SSpM8v51JnNPZJHkfsAP4bCttBX65\nqo4H/hT45yTPH1d/MxxUx8CQs3j6Lzn7dY4XSjiM9Oc5xi3JsxgEw2er6gsAVfVoVf2sqn4O/CMH\n+GXtrlTVlvb1MeCLDHp7dOepjfb1sfF1OKvTgNuq6lGY7PkdMtecTuRxneT3gDcAb21hRjs18/22\nfCuD8/e/OrYmh+ziGJjI+QVIshj4HeDKnbX9PccLJRwm/s9ztPOHlwL3VNVHh+rD55B/G7hr5r7j\nkOSwJM/buczgjci7GMzr2jZsLXDNeDqc09N+25rU+Z1hrjndAJzdrlo6CXhi6PTTWGTwn3a9B3hj\nVf1oqL4kg/+3hSQvAlYA94+ny6fbxTGwATgzyaFJljPo+esHur85vA74VlVt3lnY73N8IN95H+eN\nwZUd32aQpu8bdz+z9PcqBqcLvgnc0W6nA58B7mz1DcCx4+619fsiBldyfAPYtHNOgRcANwL3Af8O\nHDXuXod6Pgz4PnD4UG2i5pdBcG0FfsrgHPc5c80pg6uUPtGO6TuBVRPQ6zSD8/Q7j+FPtrG/246T\nO4DbgN+aoPmd8xgA3tfm917gtEnot9U/DfzhjLH7dY79hLQkqbNQTitJkvaA4SBJ6hgOkqSO4SBJ\n6hgOkqSO4SBJ6hgOkqSO4SBJ6vwvjspzL5kUCNkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5acc3613c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "classi = Counter(data['Classification'])\n",
    "plt.bar(classi.keys(),classi.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Container object of 719 artists>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE7pJREFUeJzt3X+s3fV93/Hna3ZBVZoIU+4sF+PZ\nyZxIME0uHBGkJVH2I2DQVJOpSo2m4qUsTpQgNdKmzSx/gNJVSrumkZAyKrJYMVMGYU0oVktKHBQV\nTRoJ14lrbBLiCwFhy9guzkKlVLSQ9/44n7ud+GP72vdc+/j6Ph/S0fme9/fz/Z7P535978vf7+d7\n7k1VIUnSqL836Q5Iki48hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6yyfdgfm6\n4oorau3atZPuhiQtKrt37/6rqpqaq92iDYe1a9cyPT096W5I0qKS5KUzaedlJUlSx3CQJHUMB0lS\nx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHXmDIck25McTbJvpPaVJHva\n48Uke1p9bZK/GVn3RyPbXJfkmSQzSe5Nkla/PMmuJAfa84pzMVBJ0pk7kzOHLwEbRwtV9RtVtaGq\nNgBfBb42svr52XVV9bGR+n3AR4D17TG7z23AE1W1HniivZYkTdCc4VBVTwLHT7au/e//Q8CDp9tH\nklXA26rqqaoq4AHg1rZ6E7CjLe8YqUuSJmTcOYf3Akeq6sBIbV2S7yX5iyTvbbUrgYMjbQ62GsDK\nqjrcll8BVo7ZJ0nSmMb9Yz+38fNnDYeBNVX1apLrgD9Jcs2Z7qyqKkmdan2SrcBWgDVr1syzy5Kk\nucz7zCHJcuBfAV+ZrVXV61X1alveDTwPvBM4BKwe2Xx1qwEcaZedZi8/HT3Ve1bV/VU1qKrB1NSc\nf+VOkjRP41xW+hfAD6rq/10uSjKVZFlbfjvDiecX2mWj15Lc0OYpbgcebZvtBLa05S0jdUnShJzJ\nrawPAv8beFeSg0nuaKs2009Evw/Y225t/WPgY1U1O5n9ceC/ATMMzyi+3uqfAT6Q5ADDwPnMGOOR\nJC2ADG8eWnwGg0FNT09PuhuStKgk2V1Vg7na+QlpSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwH\nSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLH\ncJAkdeYMhyTbkxxNsm+kdk+SQ0n2tMctI+vuSjKT5LkkN43UN7baTJJtI/V1Sb7d6l9JcslCDlCS\ndPbO5MzhS8DGk9Q/V1Ub2uMxgCRXA5uBa9o2/zXJsiTLgM8DNwNXA7e1tgC/1/b1D4EfA3eMMyBJ\n0vjmDIeqehI4fob72wQ8VFWvV9WPgBng+vaYqaoXqupvgYeATUkC/DPgj9v2O4Bbz3IMkqQFNs6c\nw51J9rbLTita7Urg5ZE2B1vtVPVfBv5PVb1xQv2kkmxNMp1k+tixY2N0XZJ0OvMNh/uAdwAbgMPA\nZxesR6dRVfdX1aCqBlNTU+fjLSVpSVo+n42q6sjscpIvAH/aXh4CrhppurrVOEX9VeCyJMvb2cNo\ne0nShMzrzCHJqpGXHwRm72TaCWxOcmmSdcB64DvA08D6dmfSJQwnrXdWVQHfAn69bb8FeHQ+fZIk\nLZw5zxySPAi8H7giyUHgbuD9STYABbwIfBSgqvYneRh4FngD+ERVvdn2cyfwOLAM2F5V+9tb/Efg\noST/Gfge8MUFG50kaV4y/M/74jMYDGp6enrS3ZCkRSXJ7qoazNXOT0hLkjqGgySpYzhIkjqGgySp\nYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhI\nkjqGgySpYzhIkjpzhkOS7UmOJtk3UvsvSX6QZG+SR5Jc1uprk/xNkj3t8Ucj21yX5JkkM0nuTZJW\nvzzJriQH2vOKczFQSdKZO5Mzhy8BG0+o7QL+UVX9Y+CHwF0j656vqg3t8bGR+n3AR4D17TG7z23A\nE1W1HniivZYkTdCc4VBVTwLHT6h9o6reaC+fAlafbh9JVgFvq6qnqqqAB4Bb2+pNwI62vGOkLkma\nkIWYc/gt4Osjr9cl+V6Sv0jy3la7Ejg40uZgqwGsrKrDbfkVYOUC9EmSNIbl42yc5FPAG8CXW+kw\nsKaqXk1yHfAnSa450/1VVSWp07zfVmArwJo1a+bfcUnSac37zCHJvwH+JfCv26Uiqur1qnq1Le8G\nngfeCRzi5y89rW41gCPtstPs5aejp3rPqrq/qgZVNZiamppv1yVJc5hXOCTZCPwH4Neq6qcj9akk\ny9ry2xlOPL/QLhu9luSGdpfS7cCjbbOdwJa2vGWkLkmakDkvKyV5EHg/cEWSg8DdDO9OuhTY1e5I\nfardmfQ+4NNJ/g74GfCxqpqdzP44wzuffpHhHMXsPMVngIeT3AG8BHxoQUYmSZq3tCtCi85gMKjp\n6elJd0OSFpUku6tqMFc7PyEtSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoY\nDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkzhmFQ5LtSY4m\n2TdSuzzJriQH2vOKVk+Se5PMJNmb5NqRbba09geSbBmpX5fkmbbNvUmykIOUJJ2dMz1z+BKw8YTa\nNuCJqloPPNFeA9wMrG+PrcB9MAwT4G7g3cD1wN2zgdLafGRkuxPfS5J0Hp1ROFTVk8DxE8qbgB1t\neQdw60j9gRp6CrgsySrgJmBXVR2vqh8Du4CNbd3bquqpqirggZF9SZImYJw5h5VVdbgtvwKsbMtX\nAi+PtDvYaqerHzxJvZNka5LpJNPHjh0bo+uSpNNZkAnp9j/+Woh9zfE+91fVoKoGU1NT5/rtJGnJ\nGiccjrRLQrTno61+CLhqpN3qVjtdffVJ6pKkCRknHHYCs3ccbQEeHanf3u5augH4Sbv89DhwY5IV\nbSL6RuDxtu61JDe0u5RuH9mXJGkClp9JoyQPAu8HrkhykOFdR58BHk5yB/AS8KHW/DHgFmAG+Cnw\nYYCqOp7kd4CnW7tPV9XsJPfHGd4R9YvA19tDkjQhGU4XLD6DwaCmp6cn3Q1JWlSS7K6qwVzt/IS0\nJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKlj\nOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKkz73BI8q4ke0YeryX5ZJJ7khwaqd8yss1dSWaSPJfk\nppH6xlabSbJt3EFJksazfL4bVtVzwAaAJMuAQ8AjwIeBz1XVH4y2T3I1sBm4BvgV4JtJ3tlWfx74\nAHAQeDrJzqp6dr59kySNZ97hcIJ/DjxfVS8lOVWbTcBDVfU68KMkM8D1bd1MVb0AkOSh1tZwkKQJ\nWag5h83AgyOv70yyN8n2JCta7Urg5ZE2B1vtVHVJ0oSMHQ5JLgF+DfifrXQf8A6Gl5wOA58d9z1G\n3mtrkukk08eOHVuo3UqSTrAQZw43A9+tqiMAVXWkqt6sqp8BX+D/Xzo6BFw1st3qVjtVvVNV91fV\noKoGU1NTC9B1SdLJLEQ43MbIJaUkq0bWfRDY15Z3ApuTXJpkHbAe+A7wNLA+ybp2FrK5tZUkTchY\nE9JJ3sLwLqOPjpR/P8kGoIAXZ9dV1f4kDzOcaH4D+ERVvdn2cyfwOLAM2F5V+8fplyRpPKmqSfdh\nXgaDQU1PT0+6G5K0qCTZXVWDudr5CWlJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1\nDAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1xg6H\nJC8meSbJniTTrXZ5kl1JDrTnFa2eJPcmmUmyN8m1I/vZ0tofSLJl3H5JkuZvoc4c/mlVbaiqQXu9\nDXiiqtYDT7TXADcD69tjK3AfDMMEuBt4N3A9cPdsoEiSzr9zdVlpE7CjLe8Abh2pP1BDTwGXJVkF\n3ATsqqrjVfVjYBew8Rz1TZI0h4UIhwK+kWR3kq2ttrKqDrflV4CVbflK4OWRbQ+22qnqkqQJWL4A\n+3hPVR1K8veBXUl+MLqyqipJLcD70MJnK8CaNWsWYpeSpJMY+8yhqg6156PAIwznDI60y0W056Ot\n+SHgqpHNV7faqeonvtf9VTWoqsHU1NS4XZckncJY4ZDkLUneOrsM3AjsA3YCs3ccbQEebcs7gdvb\nXUs3AD9pl58eB25MsqJNRN/YapKkCRj3stJK4JEks/v6H1X150meBh5OcgfwEvCh1v4x4BZgBvgp\n8GGAqjqe5HeAp1u7T1fV8TH7Jkmap1QtyHTAeTcYDGp6enrS3ZCkRSXJ7pGPHZySn5CWJHUMB0lS\nx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQ\nJHUMB0lSx3DQBW3ttj+bdBekJclwkCR1DAdJUsdwkCR15h0OSa5K8q0kzybZn+S3W/2eJIeS7GmP\nW0a2uSvJTJLnktw0Ut/YajNJto03JEnSuJaPse0bwL+rqu8meSuwO8mutu5zVfUHo42TXA1sBq4B\nfgX4ZpJ3ttWfBz4AHASeTrKzqp4do2+SpDHMOxyq6jBwuC3/dZLvA1eeZpNNwENV9TrwoyQzwPVt\n3UxVvQCQ5KHW1nCQpAlZkDmHJGuBXwW+3Up3JtmbZHuSFa12JfDyyGYHW+1U9ZO9z9Yk00mmjx07\nthBdlySdxNjhkOSXgK8Cn6yq14D7gHcAGxieWXx23PeYVVX3V9WgqgZTU1MLtVtJ0gnGmXMgyS8w\nDIYvV9XXAKrqyMj6LwB/2l4eAq4a2Xx1q3GauiRpAsa5WynAF4HvV9UfjtRXjTT7ILCvLe8ENie5\nNMk6YD3wHeBpYH2SdUkuYThpvXO+/ZIkjW+cM4d/Avwm8EySPa32n4DbkmwACngR+ChAVe1P8jDD\nieY3gE9U1ZsASe4EHgeWAdurav8Y/ZIkjWmcu5X+F5CTrHrsNNv8LvC7J6k/drrtJEnnl5+QliR1\nDAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdw0KLknw+Vzi3DQZLUMRwkSR3DQZLUMRy0aDnvIJ07hoO0\ngAwsXSwMB0lSx3CQJHUMB0lSx3DQkuF8gHTmDAdJUsdwkCR1DAdpnrxMpYvZBRMOSTYmeS7JTJJt\nk+7PuXYx/WA512O5mL5W0mJxQYRDkmXA54GbgauB25JcPdleaZJOFwgLERZnuw8DSkvNBREOwPXA\nTFW9UFV/CzwEbJpwny4I/lAaWsivw8n2daGEhcdbF4oLJRyuBF4eeX2w1dTM94fG2m1/tqh+4Iz2\n9Wz6Pdc4Z9eN+7WYz9dzsR0DCSBVNek+kOTXgY1V9W/b698E3l1Vd57Qbiuwtb18F/DcPN/yCuCv\n5rntYrYUx70UxwyOe6k5m3H/g6qamqvR8vH6s2AOAVeNvF7daj+nqu4H7h/3zZJMV9Vg3P0sNktx\n3EtxzOC4J92P8+1cjPtCuaz0NLA+yboklwCbgZ0T7pMkLVkXxJlDVb2R5E7gcWAZsL2q9k+4W5K0\nZF0Q4QBQVY8Bj52ntxv70tQitRTHvRTHDI57qVnwcV8QE9KSpAvLhTLnIEm6gCypcLjYf0VHkheT\nPJNkT5LpVrs8ya4kB9rzilZPknvb12Jvkmsn2/szl2R7kqNJ9o3UznqcSba09geSbJnEWM7GKcZ9\nT5JD7ZjvSXLLyLq72rifS3LTSH3RfB8kuSrJt5I8m2R/kt9u9Yv6eJ9m3OfveFfVkngwnOh+Hng7\ncAnwl8DVk+7XAo/xReCKE2q/D2xry9uA32vLtwBfBwLcAHx70v0/i3G+D7gW2DffcQKXAy+05xVt\necWkxzaPcd8D/PuTtL26/Ru/FFjX/u0vW2zfB8Aq4Nq2/Fbgh21sF/XxPs24z9vxXkpnDkv1V3Rs\nAna05R3ArSP1B2roKeCyJKsm0cGzVVVPAsdPKJ/tOG8CdlXV8ar6MbAL2Hjuez9/pxj3qWwCHqqq\n16vqR8AMw++BRfV9UFWHq+q7bfmvge8z/O0JF/XxPs24T2XBj/dSCoel8Cs6CvhGkt3t0+QAK6vq\ncFt+BVjZli+2r8fZjvNiGv+d7RLK9tnLK1yE406yFvhV4NssoeN9wrjhPB3vpRQOS8F7qupahr/d\n9hNJ3je6sobnnxf97WlLZZzNfcA7gA3AYeCzk+3OuZHkl4CvAp+sqtdG113Mx/sk4z5vx3sphcMZ\n/YqOxayqDrXno8AjDE8pj8xeLmrPR1vzi+3rcbbjvCjGX1VHqurNqvoZ8AWGxxwuonEn+QWGPyC/\nXFVfa+WL/nifbNzn83gvpXC4qH9FR5K3JHnr7DJwI7CP4Rhn78zYAjzalncCt7e7O24AfjJymr4Y\nne04HwduTLKinZrf2GqLygnzRB9keMxhOO7NSS5Nsg5YD3yHRfZ9kCTAF4HvV9Ufjqy6qI/3qcZ9\nXo/3pGflz+eD4Z0MP2Q4e/+pSfdngcf2doZ3IvwlsH92fMAvA08AB4BvApe3ehj+gaXngWeAwaTH\ncBZjfZDhKfXfMbyGesd8xgn8FsOJuxngw5Me1zzH/d/buPa2b/pVI+0/1cb9HHDzSH3RfB8A72F4\nyWgvsKc9brnYj/dpxn3ejrefkJYkdZbSZSVJ0hkyHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQx\nHCRJnf8LO8x5/rfzj8cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5acc2f50b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cat = Counter(data['Category'])\n",
    "plt.bar(cat.keys(),cat.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification labels  65\n",
      "category labels  719\n"
     ]
    }
   ],
   "source": [
    "list_classes=['Classification','Category','Severity']\n",
    "nb_classification_label = len(set(data['Classification']))\n",
    "nb_category_label = len(set(data['Category']))\n",
    "\n",
    "print(\"classification labels \", nb_classification_label)\n",
    "print(\"category labels \", nb_category_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Few insights from the dataset\n",
    "The dataset contains 4 columns:\n",
    "\n",
    "Description: contains text/complaints. Length varies from 1 to 270\n",
    "\n",
    "Classification: It has 65 different labels\n",
    "\n",
    "Category: There are 719 categories\n",
    "\n",
    "Severity: This column contains continuous values, denoting the severity of issue mentioned in description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train (208728,) (208728, 3)\n",
      "test (52182,) (52182, 3)\n"
     ]
    }
   ],
   "source": [
    "#Splitting the dataset\n",
    "train_x, test_x, train_y, test_y = train_test_split(data['Description'],data[list_classes], test_size=0.2, random_state=2)\n",
    "print(\"train\", train_x.shape, train_y.shape)\n",
    "print(\"test\", test_x.shape, test_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing word vectors\n",
      "Total 2518927 word vectors.\n"
     ]
    }
   ],
   "source": [
    "print('Indexing word vectors')\n",
    "\n",
    "#Glove Vectors\n",
    "embeddings_index = {}\n",
    "f = open(embedding_file)\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = ' '.join(values[:-300])\n",
    "    coefs = np.asarray(values[-300:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SpellingReplacer(object):\n",
    "    def __init__(self, dict_name='en', max_dist=2):\n",
    "        self.spell_dict = enchant.Dict(dict_name)\n",
    "        self.max_dist = max_dist\n",
    "    def replace(self, word):\n",
    "        if self.spell_dict.check(word):\n",
    "            return word\n",
    "        suggestions = self.spell_dict.suggest(word)\n",
    "        if suggestions and edit_distance(word, suggestions[0]) <=self.max_dist:\n",
    "            return suggestions[0]\n",
    "        else:\n",
    "            return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing text dataset\n"
     ]
    }
   ],
   "source": [
    "print('Processing text dataset')\n",
    "\n",
    "#Regex to remove all Non-Alpha Numeric and space\n",
    "special_character_removal=re.compile(r'[^a-z\\d ]',re.IGNORECASE)\n",
    "\n",
    "#regex to replace all numerics\n",
    "replace_numbers=re.compile(r'\\d+',re.IGNORECASE)\n",
    "\n",
    "#regex to replace ip address\n",
    "replace_ip=re.compile(r'([0-9]+)(?:\\.[0-9]+){3}',)\n",
    "\n",
    "#replace_url\n",
    "url_regex = re.compile(r\"(http|https|ftp)://[a-zA-Z0-9\\./]+\")\n",
    "\n",
    "def text_to_wordlist(text, remove_stopwords=True, stem_words=False, spell_correct=False):\n",
    "    # Clean the text, with the option to remove stopwords and to stem words.\n",
    "    \n",
    "    #remove urls\n",
    "    text=url_regex.sub(\"\",text)\n",
    "    \n",
    "    # Convert words to lower case and split them\n",
    "    text = text.lower().split()\n",
    "    \n",
    "    \n",
    "    # Optionally, remove stop words\n",
    "    if remove_stopwords:\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        text = [w for w in text if not w in stops]\n",
    "    \n",
    "    #Optionally, correct misspelled words, It takes lots of time\n",
    "    replacer = SpellingReplacer()\n",
    "    if spell_correct:\n",
    "        text=[replacer.replace(w) if w not in embeddings_index.keys() else w for w in text]\n",
    "        \n",
    "    text = \" \".join(text)\n",
    "    \n",
    "    #Replace IP address\n",
    "    text=replace_ip.sub('',text)\n",
    "    \n",
    "    #Remove Special Characters\n",
    "    text=special_character_removal.sub(' ',text)\n",
    "    \n",
    "    \n",
    "    #Replace Numbers\n",
    "    text=replace_numbers.sub('',text)\n",
    "    \n",
    "    \n",
    "    # Optionally, shorten words to their stems\n",
    "    if stem_words:\n",
    "        text = text.split()\n",
    "        stemmer = SnowballStemmer('english')\n",
    "        stemmed_words = [stemmer.stem(word) for word in text]\n",
    "        text = \" \".join(stemmed_words)\n",
    "    \n",
    "    # Return a list of words\n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 31295 unique tokens\n"
     ]
    }
   ],
   "source": [
    "description = []\n",
    "for text in train_x:\n",
    "    description.append(text_to_wordlist(text))\n",
    "    \n",
    "test_description=[]\n",
    "for text in test_x:\n",
    "    test_description.append(text_to_wordlist(text))\n",
    "\n",
    "tokenizer = Tokenizer(num_words=max_words)\n",
    "tokenizer.fit_on_texts(description+test_description)\n",
    "\n",
    "sequences = tokenizer.texts_to_sequences(description)\n",
    "test_sequences = tokenizer.texts_to_sequences(test_description)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor: (208728, 60)\n",
      "Shape of label tensor: (208728, 3)\n",
      "Shape of test_data tensor: (52182, 60)\n",
      "Shape of label tensor: (52182, 3)\n"
     ]
    }
   ],
   "source": [
    "train_x = pad_sequences(sequences, maxlen=max_seq_len)\n",
    "test_x = pad_sequences(test_sequences, maxlen=max_seq_len)\n",
    "\n",
    "print('Shape of data tensor:', train_x.shape)\n",
    "print('Shape of label tensor:', train_y.shape)\n",
    "\n",
    "print('Shape of test_data tensor:', test_x.shape)\n",
    "print('Shape of label tensor:', test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we have diffent types of predictions: \n",
    "1. Categorical: a) Classification b) Category\n",
    "2. Continuous: a) Severity\n",
    "\n",
    "We need two type of model for this, one for regression and one for classification\n",
    "\n",
    "Also in categorical both `Classification` and `Category` has different number of labels, so we need to train them separately.\n",
    "\n",
    "First, We need to Encode these labels and then convert it to one hot encodings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_OHE(label,nb_classes):\n",
    "    L_enc = LabelEncoder()\n",
    "    L_enc.fit(label)\n",
    "    return np_utils.to_categorical(L_enc.transform(label), nb_classes)\n",
    "\n",
    "def inverse_OHE(label,encoded_labels,nb_classes):\n",
    "    decoded=encoded_labels.argmax(1)\n",
    "    L_enc = LabelEncoder()\n",
    "    L_enc.fit(label)\n",
    "    return L_enc.inverse_transform(decoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert class vectors to binary class matrices\n",
    "train_classify = to_OHE(train_y['Classification'], nb_classification_label)\n",
    "train_category = to_OHE(train_y['Category'], nb_category_label)\n",
    "train_severity = train_y['Severity']\n",
    "\n",
    "test_classify = to_OHE(test_y['Classification'], nb_classification_label)\n",
    "test_category = to_OHE(test_y['Category'], nb_category_label)\n",
    "test_severity = test_y['Severity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing embedding matrix\n",
      "Null word embeddings: 1\n"
     ]
    }
   ],
   "source": [
    "from numpy.random import normal\n",
    "print('Preparing embedding matrix')\n",
    "nb_words = min(max_words, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words, embedding_dim))\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_words:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        try:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "        except:\n",
    "            continue\n",
    "    else:\n",
    "        try:\n",
    "            embedding_matrix[i] = normal(scale=0.6, size=(embedding_dim,))\n",
    "        except:\n",
    "            continue\n",
    "        \n",
    "print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def regression_model():\n",
    "    embedding_layer = Embedding(nb_words,\n",
    "        embedding_dim,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=max_seq_len,\n",
    "        trainable=False)\n",
    "\n",
    "\n",
    "    descriptions = Input(shape=(max_seq_len,), dtype='int32')\n",
    "    \n",
    "    embedded_sequences= embedding_layer(descriptions)\n",
    "    embedded_sequences = BatchNormalization()(embedded_sequences)\n",
    "    embedded_sequences = SpatialDropout1D(0.2)(embedded_sequences)\n",
    "    \n",
    "    x = Bidirectional(CuDNNLSTM(num_gru, return_sequences=True,kernel_initializer='glorot_uniform'))(embedded_sequences)\n",
    "    x = Bidirectional(CuDNNLSTM(num_gru, return_sequences=True,kernel_initializer='glorot_uniform'))(x)\n",
    "    x = GlobalMaxPool1D()(x)\n",
    "    x = Dropout(rate_drop_dense)(x)\n",
    "    \n",
    "    x = Dense(num_dense, activation='relu')(x)\n",
    "    x = Dropout(rate_drop_dense)(x)\n",
    "    \n",
    "    x = Dense(num_dense, activation='relu')(x)\n",
    "    x = Dropout(rate_drop_dense)(x)\n",
    "    \n",
    "    preds = Dense(1,activation='relu')(x)\n",
    "    \n",
    "    model = Model(inputs=descriptions,outputs=preds)\n",
    "    model.compile(loss='mean_absolute_error',optimizer=nadam(lr=0.001),metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "def classification_model(nb_classes):\n",
    "    embedding_layer = Embedding(nb_words,\n",
    "        embedding_dim,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=max_seq_len,\n",
    "        trainable=False)\n",
    "\n",
    "    descriptions = Input(shape=(max_seq_len,), dtype='int32')\n",
    "    \n",
    "    embedded_sequences= embedding_layer(descriptions)\n",
    "    embedded_sequences = BatchNormalization()(embedded_sequences)\n",
    "    embedded_sequences = SpatialDropout1D(0.3)(embedded_sequences)\n",
    "    \n",
    "    x = Bidirectional(CuDNNLSTM(64, return_sequences=True,kernel_initializer='glorot_uniform'))(embedded_sequences)\n",
    "    x = GlobalMaxPool1D()(x)\n",
    "    x = Dropout(rate_drop_dense)(x)\n",
    "    \n",
    "    x = Dense(32, activation='relu')(x)\n",
    "    x = Dropout(rate_drop_dense)(x)\n",
    "    \n",
    "    preds = Dense(nb_classes,activation='sigmoid')(x)\n",
    "    \n",
    "    model = Model(inputs=descriptions,outputs=preds)\n",
    "    model.compile(loss='binary_crossentropy',optimizer=nadam(lr=0.001),metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 166982 samples, validate on 41746 samples\n",
      "Epoch 1/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 1.4861 - acc: 0.4293 - val_loss: 0.9132 - val_acc: 0.7387\n",
      "Epoch 2/100\n",
      "166982/166982 [==============================] - 74s 446us/step - loss: 1.1934 - acc: 0.5507 - val_loss: 0.9724 - val_acc: 0.7302\n",
      "Epoch 3/100\n",
      "166982/166982 [==============================] - 74s 440us/step - loss: 1.1039 - acc: 0.6285 - val_loss: 0.8590 - val_acc: 0.7379\n",
      "Epoch 4/100\n",
      "166982/166982 [==============================] - 74s 442us/step - loss: 1.0466 - acc: 0.6683 - val_loss: 0.7955 - val_acc: 0.7585\n",
      "Epoch 5/100\n",
      "166982/166982 [==============================] - 74s 444us/step - loss: 1.0066 - acc: 0.6925 - val_loss: 0.8707 - val_acc: 0.7302\n",
      "Epoch 6/100\n",
      "166982/166982 [==============================] - 75s 447us/step - loss: 0.9634 - acc: 0.7152 - val_loss: 0.7782 - val_acc: 0.7808\n",
      "Epoch 7/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.9487 - acc: 0.7280 - val_loss: 0.8001 - val_acc: 0.7591\n",
      "Epoch 8/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.9033 - acc: 0.7347 - val_loss: 0.7701 - val_acc: 0.7853\n",
      "Epoch 9/100\n",
      "166982/166982 [==============================] - 75s 452us/step - loss: 0.8653 - acc: 0.7449 - val_loss: 0.7043 - val_acc: 0.7915\n",
      "Epoch 10/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.8260 - acc: 0.7493 - val_loss: 0.7051 - val_acc: 0.7911\n",
      "Epoch 11/100\n",
      "166982/166982 [==============================] - 75s 449us/step - loss: 0.8302 - acc: 0.7483 - val_loss: 0.7202 - val_acc: 0.7822\n",
      "Epoch 12/100\n",
      "166982/166982 [==============================] - 75s 450us/step - loss: 0.8126 - acc: 0.7506 - val_loss: 0.6879 - val_acc: 0.8000\n",
      "Epoch 13/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7991 - acc: 0.7523 - val_loss: 0.6981 - val_acc: 0.7719\n",
      "Epoch 14/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7861 - acc: 0.7531 - val_loss: 0.7012 - val_acc: 0.7965\n",
      "Epoch 15/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7852 - acc: 0.7541 - val_loss: 0.7166 - val_acc: 0.7731\n",
      "Epoch 16/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7893 - acc: 0.7543 - val_loss: 0.7412 - val_acc: 0.7465\n",
      "Epoch 17/100\n",
      "166982/166982 [==============================] - 75s 452us/step - loss: 0.7782 - acc: 0.7553 - val_loss: 0.6878 - val_acc: 0.7992\n",
      "Epoch 18/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7721 - acc: 0.7565 - val_loss: 0.6952 - val_acc: 0.7979\n",
      "Epoch 19/100\n",
      "166982/166982 [==============================] - 76s 456us/step - loss: 0.7764 - acc: 0.7568 - val_loss: 0.6804 - val_acc: 0.8015\n",
      "Epoch 20/100\n",
      "166982/166982 [==============================] - 76s 455us/step - loss: 0.7732 - acc: 0.7567 - val_loss: 0.7101 - val_acc: 0.7587\n",
      "Epoch 21/100\n",
      "166982/166982 [==============================] - 76s 452us/step - loss: 0.7733 - acc: 0.7577 - val_loss: 0.7162 - val_acc: 0.7851\n",
      "Epoch 22/100\n",
      "166982/166982 [==============================] - 76s 453us/step - loss: 0.7627 - acc: 0.7584 - val_loss: 0.6939 - val_acc: 0.7915\n",
      "Epoch 23/100\n",
      "166982/166982 [==============================] - 76s 455us/step - loss: 0.7599 - acc: 0.7593 - val_loss: 0.6785 - val_acc: 0.7921\n",
      "Epoch 24/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7638 - acc: 0.7579 - val_loss: 0.7013 - val_acc: 0.7713\n",
      "Epoch 25/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7677 - acc: 0.7582 - val_loss: 0.6844 - val_acc: 0.8025\n",
      "Epoch 26/100\n",
      "166982/166982 [==============================] - 75s 449us/step - loss: 0.7588 - acc: 0.7591 - val_loss: 0.6960 - val_acc: 0.7945\n",
      "Epoch 27/100\n",
      "166982/166982 [==============================] - 75s 450us/step - loss: 0.7600 - acc: 0.7597 - val_loss: 0.6728 - val_acc: 0.8080\n",
      "Epoch 28/100\n",
      "166982/166982 [==============================] - 75s 449us/step - loss: 0.7522 - acc: 0.7602 - val_loss: 0.6850 - val_acc: 0.8027\n",
      "Epoch 29/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7519 - acc: 0.7601 - val_loss: 0.6700 - val_acc: 0.8061\n",
      "Epoch 30/100\n",
      "166982/166982 [==============================] - 76s 454us/step - loss: 0.7524 - acc: 0.7609 - val_loss: 0.6760 - val_acc: 0.8056\n",
      "Epoch 31/100\n",
      "166982/166982 [==============================] - 76s 456us/step - loss: 0.7473 - acc: 0.7603 - val_loss: 0.6828 - val_acc: 0.7965\n",
      "Epoch 32/100\n",
      "166982/166982 [==============================] - 76s 456us/step - loss: 0.7502 - acc: 0.7603 - val_loss: 0.6750 - val_acc: 0.8029\n",
      "Epoch 33/100\n",
      "166982/166982 [==============================] - 76s 454us/step - loss: 0.7467 - acc: 0.7617 - val_loss: 0.6756 - val_acc: 0.7990\n",
      "Epoch 34/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7462 - acc: 0.7613 - val_loss: 0.6955 - val_acc: 0.7916\n",
      "Epoch 35/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7463 - acc: 0.7609 - val_loss: 0.7021 - val_acc: 0.7806\n",
      "Epoch 36/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7488 - acc: 0.7605 - val_loss: 0.6710 - val_acc: 0.8028\n",
      "Epoch 37/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7517 - acc: 0.7620 - val_loss: 0.6986 - val_acc: 0.7751\n",
      "Epoch 38/100\n",
      "166982/166982 [==============================] - 75s 451us/step - loss: 0.7386 - acc: 0.7616 - val_loss: 0.6669 - val_acc: 0.8083\n",
      "Epoch 39/100\n",
      "166982/166982 [==============================] - 76s 454us/step - loss: 0.7416 - acc: 0.7616 - val_loss: 0.6550 - val_acc: 0.8069\n",
      "Epoch 40/100\n",
      "166982/166982 [==============================] - 75s 449us/step - loss: 0.7389 - acc: 0.7620 - val_loss: 0.6871 - val_acc: 0.7858\n",
      "Epoch 41/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7339 - acc: 0.7635 - val_loss: 0.6782 - val_acc: 0.7995\n",
      "Epoch 42/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7351 - acc: 0.7632 - val_loss: 0.6583 - val_acc: 0.8081\n",
      "Epoch 43/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7327 - acc: 0.7621 - val_loss: 0.7213 - val_acc: 0.7496\n",
      "Epoch 44/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7312 - acc: 0.7637 - val_loss: 0.6771 - val_acc: 0.8007\n",
      "Epoch 45/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7361 - acc: 0.7628 - val_loss: 0.6627 - val_acc: 0.8074\n",
      "Epoch 46/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7311 - acc: 0.7627 - val_loss: 0.6642 - val_acc: 0.8060\n",
      "Epoch 47/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7257 - acc: 0.7637 - val_loss: 0.6635 - val_acc: 0.8075\n",
      "Epoch 48/100\n",
      "166982/166982 [==============================] - 75s 448us/step - loss: 0.7226 - acc: 0.7637 - val_loss: 0.7035 - val_acc: 0.7735\n",
      "minimum mean absolute error 0.6550372471517204\n"
     ]
    }
   ],
   "source": [
    "regressor = regression_model()\n",
    "early_stopping =EarlyStopping(monitor='val_loss', patience=9)\n",
    "bst_regressor = 'regressor_weights.h5'\n",
    "regressor_checkpoint = ModelCheckpoint(bst_regressor, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "hist_r = regressor.fit(train_x, train_severity,validation_split=0.2,epochs=100, batch_size=256, shuffle=True,callbacks=[regressor_checkpoint,early_stopping])\n",
    "bst_val_score_r = min(hist_r.history['val_loss'])\n",
    "print('minimum mean absolute error',bst_val_score_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52182/52182 [==============================] - 17s 333us/step\n",
      "Mean absolute error on test data  0.6816325635854475\n",
      "Accuracy on Test data  0.8047985895610907\n"
     ]
    }
   ],
   "source": [
    "#testing\n",
    "#severity\n",
    "regressor.load_weights(bst_regressor)\n",
    "severity_scores=regressor.evaluate(x=test_x,y=test_severity,verbose=1)\n",
    "print(\"Mean absolute error on test data \", severity_scores[0])\n",
    "print(\"Accuracy on Test data \",severity_scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 166982 samples, validate on 41746 samples\n",
      "Epoch 1/100\n",
      "166982/166982 [==============================] - 30s 181us/step - loss: 0.0739 - acc: 0.9686 - val_loss: 0.0239 - val_acc: 0.9912\n",
      "Epoch 2/100\n",
      "166982/166982 [==============================] - 29s 172us/step - loss: 0.0259 - acc: 0.9904 - val_loss: 0.0200 - val_acc: 0.9924\n",
      "Epoch 3/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0228 - acc: 0.9915 - val_loss: 0.0189 - val_acc: 0.9926\n",
      "Epoch 4/100\n",
      "166982/166982 [==============================] - 29s 172us/step - loss: 0.0214 - acc: 0.9920 - val_loss: 0.0183 - val_acc: 0.9929\n",
      "Epoch 5/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0205 - acc: 0.9923 - val_loss: 0.0179 - val_acc: 0.9930\n",
      "Epoch 6/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0200 - acc: 0.9924 - val_loss: 0.0176 - val_acc: 0.9931\n",
      "Epoch 7/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0196 - acc: 0.9925 - val_loss: 0.0174 - val_acc: 0.9932\n",
      "Epoch 8/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0193 - acc: 0.9926 - val_loss: 0.0172 - val_acc: 0.9932\n",
      "Epoch 9/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0191 - acc: 0.9927 - val_loss: 0.0172 - val_acc: 0.9932\n",
      "Epoch 10/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0188 - acc: 0.9928 - val_loss: 0.0170 - val_acc: 0.9933\n",
      "Epoch 11/100\n",
      "166982/166982 [==============================] - 29s 172us/step - loss: 0.0186 - acc: 0.9929 - val_loss: 0.0170 - val_acc: 0.9933\n",
      "Epoch 12/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0186 - acc: 0.9929 - val_loss: 0.0169 - val_acc: 0.9933\n",
      "Epoch 13/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0184 - acc: 0.9930 - val_loss: 0.0168 - val_acc: 0.9934\n",
      "Epoch 14/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0182 - acc: 0.9930 - val_loss: 0.0168 - val_acc: 0.9934\n",
      "Epoch 15/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0180 - acc: 0.9931 - val_loss: 0.0168 - val_acc: 0.9934\n",
      "Epoch 16/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0180 - acc: 0.9931 - val_loss: 0.0167 - val_acc: 0.9934\n",
      "Epoch 17/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0179 - acc: 0.9931 - val_loss: 0.0167 - val_acc: 0.9934\n",
      "Epoch 18/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0178 - acc: 0.9931 - val_loss: 0.0166 - val_acc: 0.9934\n",
      "Epoch 19/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0177 - acc: 0.9932 - val_loss: 0.0165 - val_acc: 0.9934\n",
      "Epoch 20/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0176 - acc: 0.9932 - val_loss: 0.0165 - val_acc: 0.9934\n",
      "Epoch 21/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0176 - acc: 0.9932 - val_loss: 0.0164 - val_acc: 0.9934\n",
      "Epoch 22/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0175 - acc: 0.9933 - val_loss: 0.0166 - val_acc: 0.9934\n",
      "Epoch 23/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0174 - acc: 0.9933 - val_loss: 0.0166 - val_acc: 0.9934\n",
      "Epoch 24/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0174 - acc: 0.9933 - val_loss: 0.0165 - val_acc: 0.9934\n",
      "Epoch 25/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0174 - acc: 0.9933 - val_loss: 0.0166 - val_acc: 0.9934\n",
      "Epoch 26/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0173 - acc: 0.9933 - val_loss: 0.0164 - val_acc: 0.9934\n",
      "Epoch 27/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0173 - acc: 0.9933 - val_loss: 0.0165 - val_acc: 0.9935\n",
      "Epoch 28/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0172 - acc: 0.9933 - val_loss: 0.0164 - val_acc: 0.9935\n",
      "Epoch 29/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0172 - acc: 0.9933 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 30/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0171 - acc: 0.9934 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 31/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0172 - acc: 0.9933 - val_loss: 0.0164 - val_acc: 0.9934\n",
      "Epoch 32/100\n",
      "166982/166982 [==============================] - 29s 175us/step - loss: 0.0171 - acc: 0.9934 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 33/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0170 - acc: 0.9934 - val_loss: 0.0162 - val_acc: 0.9935\n",
      "Epoch 34/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0169 - acc: 0.9934 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 35/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0169 - acc: 0.9934 - val_loss: 0.0162 - val_acc: 0.9935\n",
      "Epoch 36/100\n",
      "166982/166982 [==============================] - 29s 175us/step - loss: 0.0168 - acc: 0.9934 - val_loss: 0.0161 - val_acc: 0.9935\n",
      "Epoch 37/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0168 - acc: 0.9935 - val_loss: 0.0162 - val_acc: 0.9935\n",
      "Epoch 38/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0168 - acc: 0.9935 - val_loss: 0.0161 - val_acc: 0.9935\n",
      "Epoch 39/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0168 - acc: 0.9935 - val_loss: 0.0161 - val_acc: 0.9935\n",
      "Epoch 40/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0171 - acc: 0.9933 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 41/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0170 - acc: 0.9934 - val_loss: 0.0165 - val_acc: 0.9933\n",
      "Epoch 42/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0173 - acc: 0.9933 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 43/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0170 - acc: 0.9934 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 44/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0169 - acc: 0.9934 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "Epoch 45/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0168 - acc: 0.9935 - val_loss: 0.0162 - val_acc: 0.9935\n",
      "Epoch 46/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0167 - acc: 0.9935 - val_loss: 0.0162 - val_acc: 0.9935\n",
      "Epoch 47/100\n",
      "166982/166982 [==============================] - 29s 173us/step - loss: 0.0172 - acc: 0.9933 - val_loss: 0.0162 - val_acc: 0.9935\n",
      "Epoch 48/100\n",
      "166982/166982 [==============================] - 29s 174us/step - loss: 0.0171 - acc: 0.9934 - val_loss: 0.0163 - val_acc: 0.9935\n",
      "val_loss 0.01612608349647239\n"
     ]
    }
   ],
   "source": [
    "classifier = classification_model(nb_classification_label)\n",
    "early_stopping =EarlyStopping(monitor='val_loss', patience=9)\n",
    "bst_classifier = 'classifier_weights.h5'\n",
    "classifier_checkpoint = ModelCheckpoint(bst_classifier, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "hist_c = classifier.fit(train_x, train_classify,validation_split=0.2,epochs=100, batch_size=256, shuffle=True,callbacks=[classifier_checkpoint,early_stopping])\n",
    "bst_val_score_c = min(hist_c.history['val_loss'])\n",
    "print('val_loss',bst_val_score_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52182/52182 [==============================] - 9s 166us/step\n",
      "Loss on test data  0.1515411853277415\n",
      "Accuracy on test data  0.9791903235448023\n"
     ]
    }
   ],
   "source": [
    "#testing Classification\n",
    "classifier.load_weights(bst_classifier)\n",
    "classification_scores = classifier.evaluate(test_x, test_classify)\n",
    "print(\"Loss on test data \", classification_scores[0])\n",
    "print(\"Accuracy on test data \",classification_scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 166982 samples, validate on 41746 samples\n",
      "Epoch 1/10\n",
      "166982/166982 [==============================] - 31s 188us/step - loss: 0.0276 - acc: 0.9914 - val_loss: 0.0082 - val_acc: 0.9986\n",
      "Epoch 2/10\n",
      "166982/166982 [==============================] - 30s 177us/step - loss: 0.0084 - acc: 0.9986 - val_loss: 0.0077 - val_acc: 0.9986\n",
      "Epoch 3/10\n",
      "166982/166982 [==============================] - 30s 178us/step - loss: 0.0081 - acc: 0.9986 - val_loss: 0.0074 - val_acc: 0.9986\n",
      "Epoch 4/10\n",
      "166982/166982 [==============================] - 30s 177us/step - loss: 0.0077 - acc: 0.9987 - val_loss: 0.0069 - val_acc: 0.9987\n",
      "Epoch 5/10\n",
      "166982/166982 [==============================] - 30s 177us/step - loss: 0.0073 - acc: 0.9987 - val_loss: 0.0065 - val_acc: 0.9988\n",
      "Epoch 6/10\n",
      "166982/166982 [==============================] - 30s 178us/step - loss: 0.0069 - acc: 0.9987 - val_loss: 0.0061 - val_acc: 0.9988\n",
      "Epoch 7/10\n",
      "166982/166982 [==============================] - 30s 178us/step - loss: 0.0066 - acc: 0.9987 - val_loss: 0.0058 - val_acc: 0.9988\n",
      "Epoch 8/10\n",
      "166982/166982 [==============================] - 30s 177us/step - loss: 0.0063 - acc: 0.9988 - val_loss: 0.0055 - val_acc: 0.9988\n",
      "Epoch 9/10\n",
      "166982/166982 [==============================] - 30s 178us/step - loss: 0.0060 - acc: 0.9988 - val_loss: 0.0053 - val_acc: 0.9988\n",
      "Epoch 10/10\n",
      "166982/166982 [==============================] - 30s 178us/step - loss: 0.0058 - acc: 0.9988 - val_loss: 0.0050 - val_acc: 0.9988\n",
      "val_loss 0.005037460078318927\n"
     ]
    }
   ],
   "source": [
    "category = classification_model(nb_category_label)\n",
    "early_stopping =EarlyStopping(monitor='val_loss', patience=9)\n",
    "bst_category = 'category_weights.h5'\n",
    "category_checkpoint = ModelCheckpoint(bst_category, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "hist_c = category.fit(train_x, train_category,validation_split=0.2,epochs=10, batch_size=256, shuffle=True,callbacks=[category_checkpoint,early_stopping])\n",
    "bst_val_score_c = min(hist_c.history['val_loss'])\n",
    "print('val_loss',bst_val_score_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52182/52182 [==============================] - 9s 167us/step\n",
      "Loss on test data  0.017061238498983996\n",
      "Accuracy on test data  0.9982546845002738\n"
     ]
    }
   ],
   "source": [
    "#testing category\n",
    "category.load_weights(bst_category)\n",
    "category_scores = category.evaluate(test_x, test_category)\n",
    "print(\"Loss on test data \", category_scores[0])\n",
    "print(\"Accuracy on test data \",category_scores[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model can further be improved by using K-Fold Cross Validation and some Feature Engineering."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "statoil",
   "language": "python",
   "name": "statoil"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}