#2 - CNN data preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing for Pointer Network\n",
    "In this notebook we are showing how we do data preprocessing in order to feed the CNN stories dataset to our pointer network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import glob\n",
    "#%%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mmap(fn, elem):\n",
    "    return list(map(fn, elem))\n",
    "\n",
    "def getel(n, lst): \n",
    "    return mmap(lambda x: x[n], lst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Cleaning\n",
    "The cleaning of the CNN dataset which we use is kindly taken from machinelearningmaster.com\n",
    "\n",
    "You can find the whole article here:\n",
    "https://machinelearningmastery.com/prepare-news-articles-text-summarization/?unapproved=487774&moderation-hash=c397686a5fc42785c709030b2bcf4ebc#comment-487774"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "import string\n",
    "\n",
    "# load doc into memory\n",
    "\n",
    "\n",
    "def load_doc(filename):\n",
    "    # open the file as read only\n",
    "    file = open(filename, encoding='utf-8')\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    return text\n",
    "\n",
    "# split a document into news story and highlights\n",
    "\n",
    "\n",
    "def split_story(doc):\n",
    "    # find first highlight\n",
    "    index = doc.find('@highlight')\n",
    "    # split into story and highlights\n",
    "    story, highlights = doc[:index], doc[index:].split('@highlight')\n",
    "    # strip extra white space around each highlight\n",
    "    highlights = [h.strip() for h in highlights if len(h) > 0]\n",
    "    return story, highlights\n",
    "\n",
    "# load all stories in a directory\n",
    "\n",
    "\n",
    "def load_stories(directory):\n",
    "    stories = list()\n",
    "    for name in listdir(directory):\n",
    "        filename = directory + '/' + name\n",
    "        # load document\n",
    "        doc = load_doc(filename)\n",
    "        # split into story and highlights\n",
    "        story, highlights = split_story(doc)\n",
    "        # store\n",
    "        stories.append({'story': story, 'highlights': highlights})\n",
    "    return stories\n",
    "\n",
    "# clean a list of lines\n",
    "\n",
    "\n",
    "def clean_lines(lines):\n",
    "    cleaned = list()\n",
    "    # prepare a translation table to remove punctuation\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    for line in lines:\n",
    "        # strip source cnn office if it exists\n",
    "        index = line.find('(CNN) -- ')\n",
    "        if index > -1:\n",
    "            line = line[index+len('(CNN)'):]\n",
    "        # tokenize on white space\n",
    "        line = line.split()\n",
    "        # convert to lower case\n",
    "        line = [word.lower() for word in line]\n",
    "        # remove punctuation from each token\n",
    "        line = [w.translate(table) for w in line]\n",
    "        # remove tokens with numbers in them\n",
    "        line = [word for word in line if word.isalpha()]\n",
    "        # store as string\n",
    "        cleaned.append(' '.join(line))\n",
    "    # remove empty strings\n",
    "    cleaned = [c for c in cleaned if len(c) > 0]\n",
    "    return cleaned\n",
    "\n",
    "\n",
    "# load stories\n",
    "directory = './cnn_dataset/cnn/stories/'\n",
    "stories = load_stories(directory)\n",
    "print('Loaded Stories %d' % len(stories))\n",
    "\n",
    "# clean stories\n",
    "for example in stories:\n",
    "    example['story'] = clean_lines(example['story'].split('\\n'))\n",
    "    example['highlights'] = clean_lines(example['highlights'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stories[0]['highlights']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to file\n",
    "from pickle import dump, load\n",
    "dump(stories, open('cnn_dataset.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded Stories 92579\n"
     ]
    }
   ],
   "source": [
    "# load from file\n",
    "from pickle import dump, load\n",
    "stories = load(open('cnn_dataset.pkl', 'rb'))\n",
    "print('Loaded Stories %d' % len(stories))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data sampling\n",
    "After the dataset is cleaned and tokenized, we can select a sample of the dataset to work with and we can start doing our analysis of the dataset. Here we choose a 10% which account for 9258 examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After the dataset is cleaned and tokenized, we can select a sample of the dataset to work with and we can start doing our analysis of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = len(stories) // 10\n",
    "# Set Seed\n",
    "np.random.seed(seed=13367)\n",
    "stories = np.array(stories)\n",
    "indexes = np.random.choice(list(range(stories.shape[0])), size=size, replace=False)\n",
    "\n",
    "# Sample 10% of dataset\n",
    "random_sample = stories[indexes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = len(stories) // 10\n",
    "# Set Seed\n",
    "np.random.seed(seed=13367)\n",
    "stories = np.array(stories)\n",
    "indexes = np.random.choice(list(range(92579)), size=size, replace=False)\n",
    "\n",
    "# Sample 10% of dataset\n",
    "random_sample = stories[indexes]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also select the first 10 sentences in order to reduce the training size for saving computational times. This is important because there may be outliers in terms of sentence length that will increase the size of the training set because of the padding inserted in the other sentences. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_first_n_sentence(data, n):\n",
    "    tmp = data.tolist()\n",
    "    X = [\". \".join(obj['story'][:n]) for obj in tmp]\n",
    "    y = [\". \".join(obj['highlights']) for obj in tmp]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = get_first_n_sentence(random_sample, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now print respectively the max sentence length and the mean length, in order to see how much more we can save in memory. It seems that the max length here is 1k of words against the 301 of the mean. So we still have outliers to remove from this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max sentence length is 1206\n",
      "Mean sentence length is 301.8644269201685\n"
     ]
    }
   ],
   "source": [
    "# See max and mean length of sentences\n",
    "print('Max sentence length is', np.amax(list(map(lambda x: len(x.split(' ')), X))))\n",
    "print('Mean sentence length is', np.mean(list(map(lambda x: len(x.split(' ')), X))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now want to select a nice cutting point that retains the majority of the dataset without increasing much computational time. Hence we plot the sizes of all the examples and try to understand at first what would be a nice threshold. Looking at the graph a max length of 500 words seems to be a nice solution, but lets get a more accurate visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1278e6f60>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Distribution of words in the dataset, we cannot understand a lot, but it seems ~500 or 600 a nice cutting point\n",
    "# for selecting a max_word_length\n",
    "plt.plot(range(len(X)), list(map(lambda x: len(x.split(' ')), X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we plot the sentence length against the cumulative sum of the longer sentences. We finally choose 500 as the max length of the sentence we want to allow in our dataset. We choose to discard the sentence instead of pruning it because cutting a sentence midway usually add more noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1284394a8>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD8CAYAAACCRVh7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt4VfWd7/H3d+ceICSBcAuUgCA3RcWoXFraiuOt3lp1xraj2HrqdMZz2plnns600+cc5/QyTzvTU1vPzHhqvdT2WG2PtdVaq0Wk9VqQ+/0SECEkkIRcgEBIsvf3/LEXkNhskmiStS+f1/PsZ6/1W7+9812LRT5Zd3N3REREehIJuwAREUleCgkREUlIISEiIgkpJEREJCGFhIiIJKSQEBGRhBQSIiKSkEJCREQSUkiIiEhC2WEXcDajR4/2ioqKsMsQEUkpa9asaXD3soH4rqQOiYqKClavXh12GSIiKcXM3hmo79LuJhERSUghISIiCSkkREQkIYWEiIgkpJAQEZGEFBIiIpKQQkJERBJK6uskRCR1uDudMScaC96jTkcsRkc0RkdnfDgWc6Ie7xOLcWb4dJsT83h7zB13JxrjT4bj45z+bPwVn3amni61dS+0x/Zu/fvS513z3vMy6dq/99r60r/79/fe5/1SSIikMXfnZGeMYyc7OdbWGX/vMnw0GG4N2o+0dXC0rZO2jigd0RjRmNMR7fKLPxajM+qc6IgGfZz2aIzOaIzYIP2SknApJERSSEc0xsGWNmpb2mhsPUnT8Q6ajrfT1NpO0/EOmo+3d2s70tZJtA+/vSMGw/OyGZ6XTVFBDoW5WWRHImRHIuTnGNkRIysSISfLiESMwpws8nOyyM2OkJ1l5GZFyIqc6Rd/N7KzjJysCLlZ8X4Ri7efes+KcGbY4t8dn063vtZtHMyC7zg1LXJm2LrOmHUdPDNi1mMXrMuE7u09f0+3wQH6Tus2A/3rf2ow59s9f8d7oZAQSSIn2qMcaD4RfzWd4EDz8eA9Pn7wSFuPf7HnZUcoHZZLcWEuJYU5zBpXRHFhDiMLchiWl82I/OzTITC8h+GCnKxuv8xETlFIiAyhaMzZXX+MvQ2tXYLgTAgcbm3v1j8rYowfmU95cQHzzxnFxOICyksKmFBcQOmwXEoK46+C3KyQ5kjSnUJCZJA0H29nd/0xdte3srv+GJuqW1i/v5nj7dHTffJzIkwoLqC8uIA5E4ooD0JgYkkh5cUFjC3KJyuiv/AlPAoJkQEQizlvH27llZ31bDrQwqq3G6luOnF6em5WhOljh3PrxROZO7GYaWOGU15SwKhhudrNI0lNISHyHjS2trNuXxPr9jWzpaaF1e80cbStE4DRw3O5eHIJdyyYzLQxw5k6ejgTSwrIztJlSZJ6FBIifXToSBsvbTvEE6v2sfnAESB+zOCcsmFcc944LpxUwuJzRzOxpDDkSkUGjkJCJIFozHmtqoFfb6hh/f5mquqOATBj7Ai+dNUMKieXMHdisQ4aS1pTSIh04e5sqz3Kr9Yf4FfrDlB39CRF+dlcOqWUj19UzpJZY5gxdoSOI0jGUEhIxnN3ttQc4YlV+1ixvY6aljayI8ZHZ47h5nnlfHTmGPKytbUgmUkhIRnL3Xlj92G+u2wna95pIj8nwuLpZXxhyXSunDOO0mG5YZcoEjqFhGSkmuYT/POzW/jd1kOMK8rn3utn87HzxzOmKD/s0kSSikJCMkYs5vx6Yw3Prq/h1aoGAP7x6pl8ZlEF+TnanSTSE4WEpL2TnVF+ufYAP3x1D7vrW5lUWsAnL5nE5xZP1emqIr1QSEjaau+M8cNX9/Dwa2/T2NrO3Ikj+f5tF3L93AlEdKsLkT5RSEjaicacl7fX8W8vbmfnoWNcPnMMdy6s4EPTR+vUVZF+UkhIWlm3r4n/8cwWNh1ooby4gEfurOTymWPDLkskZSkkJC2caI/ytee28MSq/YwZkcd9f3EB18+doPslibxPCglJeW/sbuCfn93CrrpjfO5DU/jiFecyPE+rtshA0P8kSVnRmPMvz2/j4dfeZvTwPB68vZI/m61dSyIDSSEhKenYyU6+8MQ6Xt5ex50LK/jyNTN1rYPIIFBISMppbG3nUz/8I7vqjvHNj5/Hpy+bHHZJImmrT0f1zOzvzGyLmW02syfMLN/MppjZSjPbZWY/M7PcoG9eMF4VTK/o8j1fCdp3mNlVgzNLks721B9j6SOr2NPQyqN3XqKAEBlkvYaEmZUDXwAq3f08IAu4Dfg2cJ+7TweagLuCj9wFNLn7NOC+oB9mNjv43BzgauA/zUz7B6TP3tjdwA3//jr7Go/zwKfnsfjcsrBLEkl7fT0/MBsoMLNsoBCoBS4HngqmPwbcFAzfGIwTTF9i8SuYbgSedPeT7v42UAVc+v5nQTLB4yvf4bM/eovy4gJ++8UPsWSWDlCLDIVeQ8LdDwDfAfYRD4cWYA3Q7O6dQbdqoDwYLgf2B5/tDPqP6trew2dEeuTu/PvLu/jqLzczZ8JIfvq5y5hQXBB2WSIZoy+7m0qIbwVMASYAw4Breujqpz6SYFqi9nf/vLvNbLWZra6vr++tPElz9y+v4ju/28mVs8fys7vnM2p4XtgliWSUvuxuugJ4293r3b0DeBpYCBQHu58AJgI1wXA1MAkgmD4SaOza3sNnTnP3B9290t0ry8q0zzmTPbexhvte2snN8ybyg9sv1tXTIiHoy/+6fcB8MysMji0sAbYCK4Bbgj5LgWeC4WeDcYLpL7u7B+23BWc/TQGmA6sGZjYk3WzY38zf/3wDlZNL+JdPnKcb84mEpNfrJNx9pZk9BawFOoF1wIPAb4AnzewbQdvDwUceBn5iZlXEtyBuC75ni5n9nHjAdAL3uHt0gOdH0sDBljY+9+PVlI3I4we3X6znS4uEyOJ/5CenyspKX716ddhlyBA60R7lz3/wJnvqj/H03yxixrgRYZckknLMbI27Vw7Ed+mKa0ka7s6XntrA5poWHrqjUgEhkgR0JFCSxqOv7+W5jbV86aoZug5CJEkoJCQpPL+plq89t5UrZ4/l84vPCbscEQkoJCR0ja3tfPWXm7hwUjH3f/IiPX9aJIkoJCRUx0528rkfr+ZoWyffvnmubvctkmR04FpC9Z0Xd7B2XxP/+5MX6UC1SBLSloSEZlN1Cz9+cy9/edlkrps7IexyRKQHCgkJRTTmfPVXmxg1PI8vXT0j7HJEJAGFhITiJ2/uZWN1C//9utkU5eeEXY6IJKCQkCF3sKWN7/xuJx+aPprr544PuxwROQuFhAy5f31hO+3RGN+4STfuE0l2CgkZUlV1R/nl+gN8ZmEFk0cNC7scEemFQkKG1H0v7aIwJ4u/+rCuqhZJBQoJGTJba47wm421fPaDUygdlht2OSLSBwoJGTLfXbaTovxs/suHpoZdioj0kUJChsT6/c28tO0Qdy+eysgCnfIqkioUEjIkvrtsJ6XDcrlz0ZSwSxGRflBIyKBb9XYjr+ys5/MfnsrwPN0uTCSVKCRkUHVGY/zrC9spG5HH7fMrwi5HRPpJISGD6qer9rH6nSb+4aoZFOTqNuAiqUYhIYOm+Xg73/rtdhaeM4pbLp4Ydjki8h4oJGTQPL5yH8fbo9x7/RzdfkMkRSkkZFC0dUT50Rt7WXxumR4mJJLCFBIyKH657gD1R0/yV4t14ZxIKlNIyIBr74xx//JdXDCpmIXnjAq7HBF5HxQSMuB+u7mW2pY2/nbJdB2LEElxCgkZcD96Yy9TRg/jw+eWhV2KiLxPCgkZUOv3N7NuXzNLF0wmEtFWhEiqU0jIgHrsjb0Mz8vmZl0XIZIWFBIyYOqOtvHcxhpuuXgiI/J1p1eRdKCQkAHz05X76Ig6SxdWhF2KiAwQhYQMiM5ojCdX7WfxuWVMGa1nV4ukC4WEDIg/7Kzn4JE2PnXpB8IuRUQGkEJCBsQv1lYzalguS2aNCbsUERlAfQoJMys2s6fMbLuZbTOzBWZWambLzGxX8F4S9DUzu9/Mqsxso5nN6/I9S4P+u8xs6WDNlAyt5uPtvLS1jhsunEBOlv7uEEknff0f/X3gBXefCVwAbAO+DCx39+nA8mAc4BpgevC6G3gAwMxKgXuBy4BLgXtPBYuktt9sqqU9GuPmeTrtVSTd9BoSZlYELAYeBnD3dndvBm4EHgu6PQbcFAzfCPzY4/4IFJvZeOAqYJm7N7p7E7AMuHpA50ZC8cy6GqaPGc6cCUVhlyIiA6wvWxJTgXrgUTNbZ2YPmdkwYKy71wIE76d2RpcD+7t8vjpoS9QuKay66Tir9jZy00Xluk+TSBrqS0hkA/OAB9z9IqCVM7uWetLTbwo/S3v3D5vdbWarzWx1fX19H8qTMD2zvgaAGy6YEHIlIjIY+hIS1UC1u68Mxp8iHhqHgt1IBO91XfpP6vL5iUDNWdq7cfcH3b3S3SvLynSDuGT36w01XDy5hEmlhWGXIiKDoNeQcPeDwH4zmxE0LQG2As8Cp85QWgo8Eww/C9wRnOU0H2gJdke9CFxpZiXBAesrgzZJUTsPHWX7waPaihBJY9l97PffgMfNLBfYA3yGeMD83MzuAvYBtwZ9nweuBaqA40Ff3L3RzL4OvBX0+5q7Nw7IXEgonttQQ8TgmvPHhV2KiAySPoWEu68HKnuYtKSHvg7ck+B7HgEe6U+BkpzcnV9vrGXBOaMYMyI/7HJEZJDoyid5T7bUHOHthlaum6tdTSLpTCEh78myrYcwgytnjw27FBEZRAoJeU9+v6OOCycVM2p4XtiliMggUkhIv9UfPcmG6hYun6Gb+YmkO4WE9NsLWw4CcLnu+CqS9hQS0m8/f2s/s8YXMXu87tUkku4UEtIv+xuPs+lACzfP072aRDKBQkL65eXt8buvLJmls5pEMoFCQvpl2dZDTB09TM+xFskQCgnps6bWdt7cc5irztNtOEQyhUJC+mzZtkNEY841CgmRjKGQkD57YfNByosLOL98ZNiliMgQUUhInxw72clruxq4+rxxOqtJJIMoJKRPXt1ZT3s0pns1iWQYhYT0ybJthxhZkMPFk0vCLkVEhpBCQnoVjTkrttdx+cwxZGdplRHJJPofL71au6+JpuMdLNG9mkQyjkJCevXStkPkZBmLzy0LuxQRGWIKCTkrd+eFzQeZP3UURfk5YZcjIkNMISFntbX2CO8cPs61548PuxQRCYFCQs7q+U21ZEWMq+boKmuRTKSQkITcnd9uOsj8qaWUDssNuxwRCYFCQhLa13icPQ2tXDlbWxEimUohIQm9uqsBgA9OHx1yJSISFoWEJPR6VQMTRuYzVc+OEMlYCgnpUTTmvLH7MIumjdYN/UQymEJCerSlpoWWEx3a1SSS4RQS0qPXquLHIxaeo5AQyWQKCenR61UNzBw3grIReWGXIiIhUkjIn2jriPLW3iY+OE1bESKZTiEhf2Ll2420d8ZYpOMRIhlPISF/4lfrDlCUn82CqaPCLkVEQqaQkG6iMWfFjjqumD2W/JyssMsRkZApJKSb9fubaT7ewUdn6AFDItKPkDCzLDNbZ2bPBeNTzGylme0ys5+ZWW7QnheMVwXTK7p8x1eC9h1mdtVAz4y8f7/fUUfEYPF0PWBIRPq3JfFFYFuX8W8D97n7dKAJuCtovwtocvdpwH1BP8xsNnAbMAe4GvhPM9P+jCSzYkcdF08uYWShHjAkIn0MCTObCHwMeCgYN+By4Kmgy2PATcHwjcE4wfQlQf8bgSfd/aS7vw1UAZcOxEzIwKg72sbmA0f4iHY1iUigr1sS3wP+AYgF46OAZnfvDMargfJguBzYDxBMbwn6n27v4TOSBP6wox6Aj8zQriYRies1JMzsOqDO3dd0be6hq/cy7Wyf6frz7jaz1Wa2ur6+vrfyZAD9fkc9Y0bkMXt8UdiliEiS6MuWxCLgBjPbCzxJfDfT94BiM8sO+kwEaoLhamASQDB9JNDYtb2Hz5zm7g+6e6W7V5aV6S/aodIRjfHKrno+MqNMd30VkdN6DQl3/4q7T3T3CuIHnl92908DK4Bbgm5LgWeC4WeDcYLpL7u7B+23BWc/TQGmA6sGbE7kfVn7ThNH2zp16quIdJPde5eE/hF40sy+AawDHg7aHwZ+YmZVxLcgbgNw9y1m9nNgK9AJ3OPu0ffx82UA/X5nPdkR0604RKSbfoWEu/8e+H0wvIcezk5y9zbg1gSf/ybwzf4WKYPvlZ31zJtcQlG+Tn0VkTN0xbXQ1NrO1tojLNKzI0TkXRQSwh/3HMYdFk3TDf1EpDuFhPD67gaG5WZxwaTisEsRkSSjkBDeqDrMpVNKycnS6iAi3em3QoarbTnBnoZWFukpdCLSA4VEhnt1VwMAC3XQWkR6oJDIcMu3HWJcUT6zxo8IuxQRSUIKiQzW1hHllZ0NXDF7jG7FISI9UkhksDd3H+ZER5QrZo0NuxQRSVIKiQy2bNshhuVmseAcXR8hIj1TSGQod2fF9jo+OH00edl6QKCI9EwhkaF2HDpKbUsbl8/UXV9FJDGFRIZasf3UU+gUEiKSmEIiQ63YUcecCUWMLcoPuxQRSWIKiQzUcryDNe806QFDItIrhUQGerWqnmjM+ehMPR5WRM5OIZGBVmyvp7gwhwsnlYRdiogkOYVEhonFnD/srGPx9DKyIrrKWkTOTiGRYTYdaKHhWLtOfRWRPlFIZJgVO+owg8Xn6niEiPROIZFhVuyo58JJxZQOyw27FBFJAQqJDNJw7CQbq5t16quI9JlCIoO8srMedxQSItJnCokM8vL2OspG5DFnQlHYpYhIilBIZIjOaIxXdtbzkXPLiOjUVxHpI4VEhthQ3cKRtk4+PENnNYlI3ykkMsRruxowg0XnjA67FBFJIQqJDLF8+yHOLx9JiU59FZF+UEhkgJ2HjrKxuoUbLpgQdikikmIUEhng+U21mMGNF5aHXYqIpBiFRAb4w8565k4spmxEXtiliEiKUUikucPHTrJhfzMf1r2aROQ9UEikud9uPkjM4ZrzxoVdioikIIVEmvv1hhqmjRnOzHEjwi5FRFJQryFhZpPMbIWZbTOzLWb2xaC91MyWmdmu4L0kaDczu9/Mqsxso5nN6/JdS4P+u8xs6eDNlgAcbGlj1d5Grp87ATNdZS0i/deXLYlO4O/dfRYwH7jHzGYDXwaWu/t0YHkwDnANMD143Q08APFQAe4FLgMuBe49FSwyOH6zqRZ3uO6C8WGXIiIpqteQcPdad18bDB8FtgHlwI3AY0G3x4CbguEbgR973B+BYjMbD1wFLHP3RndvApYBVw/o3Mhp7s7Ta6uZM6GIc8qGh12OiKSofh2TMLMK4CJgJTDW3WshHiTAqftPlwP7u3ysOmhL1P7un3G3ma02s9X19fX9KU+6WPNOE1tqjvDpyyaHXYqIpLA+h4SZDQd+Afytux85W9ce2vws7d0b3B9090p3rywr02mb79Wjb+ylKD+bmy7SVdYi8t71KSTMLId4QDzu7k8HzYeC3UgE73VBezUwqcvHJwI1Z2mXAXawpY0XNh/kLy6ZRGFudtjliEgK68vZTQY8DGxz9+92mfQscOoMpaXAM13a7wjOcpoPtAS7o14ErjSzkuCA9ZVBmwywx1e+Q8yd2+dXhF2KiKS4vvyZuQi4HdhkZuuDtn8CvgX83MzuAvYBtwbTngeuBaqA48BnANy90cy+DrwV9PuauzcOyFzIaW0dUX66ch9LZo7lA6MKwy5HRFJcryHh7q/R8/EEgCU99HfgngTf9QjwSH8KlP75zcZaDre2c+fCirBLEZE0oCuu04i786M39jJtzHAWTRsVdjkikgYUEmlk7b5mNh1oYemCybrCWkQGhEIijTz2xl5G5GXziXkTwy5FRNKEQiJNHDrSxvObarm1chLD8nTaq4gMDIVEmnh85T6i7tyxQFdYi8jAUUikgZOdUX668h0+OmMMFaOHhV2OiKQRhUQaeGZdDQ3H2lmq015FZIApJFLcifYo//a7HVwwcSQfmjY67HJEJM0oJFLc0+uqqT96kn+6dhaRiE57FZGBpZBIYe7Oj17fy5wJRVw6pTTsckQkDSkkUthrVQ3sqjvGnQsrdPGciAwKhUSKau+M8fXntjJhZD7XX6BnRojI4NBVVynqwVd2s/PQMR66o5L8nKywyxGRNKUtiRS0p/4Y979cxcfOH88Vs8eGXY6IpDGFRIqJxZyv/nIzedkR7r1+dtjliEiaU0ikmP/zym7e3HOYf7p2FmOK8sMuR0TSnEIihbxR1cB3XtzBdXPHc9slk3r/gIjI+6SQSBE7Dh7lrx9fy9Sy4Xz75rk65VVEhoRCIgVU1R3lLx9eSV52hEfvvES3AheRIaPfNklud/0xbntwJQA//dxlTCotDLkiEckkCokktr/xOHc+ugpwnrx7PtPGjAi7JBHJMAqJJLVyz2E+/3/XEHN47LOXKiBEJBQKiST0xz2Huf3hlUwqLeThpZcwRQ8SEpGQKCSSzPaDR/ibx9fygdJCnv7rRYwszAm7JBHJYDq7KYm8ufswtz7wJrlZER5aeokCQkRCp5BIEr/eUMPSR1YxbmQ+T//NQu1iEpGkoN1NIXN37ntpF/cv38WlFaX88I5KbUGISNJQSITkRHuU3209yKOv72X9/mY+Ma+cf/n4+brtt4gkFYXEEIvGnF+sqeZ/LdvBoSMnKS8u4FufOJ8/r5ykZ1SLSNJRSAwRd+cPO+v51m+3s/3gUS6cVMx3//xCFkwdpXAQkaSlkBhk7s5rwd1bN1S38IHSQv79UxfxsfPH6yZ9IpL0FBKD5PCxkzy/+SBPrd7PhuoWyosL+ObHz+OWiyeSl63jDiKSGhQSA6j5eDubDrTwzPoafrOxlhMdUaaNGc43bjqPWysVDiKSeoY8JMzsauD7QBbwkLt/a6hrGChNre2s3dfEun3NrN3XxFt7G+mIOsPzsrnhggncuaiCmeNGaLeSiKSsIQ0JM8sC/gP4M6AaeMvMnnX3rUNZR1+5O20dMZqOt1N39CQbq5tZv6+ZPQ2tHG49yf7GEwBkRYxZ40ewdEEFi6aPZv6UURTkaqtBRFLfUG9JXApUufseADN7ErgRSBgSR9s6cMBjEHMn5o4TH3Y/1Rb/hd51POZOZ9Q50RHlRHuUEx2dtHXE6IjGaO+M0RF1OmMxWk9GaT7RTnNrR/z9eEf8daKdpuMdtHfGutVTNiKPGWNH8IHSQm675ANUTi5h7sRihYKIpKWhDolyYH+X8WrgskSdt9Ue4fx//t2gF5WbHaGkMIfiglyKC3OoGF1IcUExxcPOtI0alsuc8pFMGJmv3UcikjGGOiR6+u3q3TqY3Q3cDVA0YSpfumoGedkRImZEDKzb+6nh7uMRM8wgOxKhMDeL/JwsCnKzyM+JkJsVIScrQm52/D0/J0JBTpZ+8YuI9GCoQ6IamNRlfCJQ07WDuz8IPAhQWVnp93x02tBVJyIi3Qz1XWDfAqab2RQzywVuA54d4hpERKSPhnRLwt07zey/Ai8SPwX2EXffMpQ1iIhI3w35dRLu/jzw/FD/XBER6T89dEhERBJSSIiISEIKCRERSUghISIiCSkkREQkIXP33nuFxMyOAjvCriNJjAYawi4iSWhZnKFlcYaWxRkz3H3EQHxRsj9PYoe7V4ZdRDIws9VaFnFaFmdoWZyhZXGGma0eqO/S7iYREUlIISEiIgkle0g8GHYBSUTL4gwtizO0LM7QsjhjwJZFUh+4FhGRcCX7loSIiIQoaUPCzK42sx1mVmVmXw67nsFmZpPMbIWZbTOzLWb2xaC91MyWmdmu4L0kaDczuz9YPhvNbF64czCwzCzLzNaZ2XPB+BQzWxksh58Ft5rHzPKC8apgekWYdQ8GMys2s6fMbHuwfizIxPXCzP4u+L+x2cyeMLP8TFovzOwRM6szs81d2vq9HpjZ0qD/LjNb2tvPTcqQMLMs4D+Aa4DZwCfNbHa4VQ26TuDv3X0WMB+4J5jnLwPL3X06sDwYh/iymR687gYeGPqSB9UXgW1dxr8N3BcshybgrqD9LqDJ3acB9wX90s33gRfcfSZwAfHlklHrhZmVA18AKt39POKPGriNzFovfgRc/a62fq0HZlYK3Ev8sdGXAveeCpaE3D3pXsAC4MUu418BvhJ2XUO8DJ4B/oz4xYTjg7bxxK8dAfgB8Mku/U/3S/UX8ScWLgcuB54j/tjbBiD73esH8WeTLAiGs4N+FvY8DOCyKALefvc8Zdp6AZQD+4HS4N/5OeCqTFsvgApg83tdD4BPAj/o0t6tX0+vpNyS4MwKcUp10JYRgk3ji4CVwFh3rwUI3scE3dJ5GX0P+AcgFoyPAprdvTMY7zqvp5dDML0l6J8upgL1wKPB7reHzGwYGbZeuPsB4DvAPqCW+L/zGjJ3vTilv+tBv9ePZA0J66EtI07DMrPhwC+Av3X3I2fr2kNbyi8jM7sOqHP3NV2be+jqfZiWDrKBecAD7n4R0MqZXQo9ScvlEewSuRGYAkwAhhHfpfJumbJe9CbR/Pd7uSRrSFQDk7qMTwRqQqplyJhZDvGAeNzdnw6aD5nZ+GD6eKAuaE/XZbQIuMHM9gJPEt/l9D2g2MxO3Uam67yeXg7B9JFA41AWPMiqgWp3XxmMP0U8NDJtvbgCeNvd6929A3gaWEjmrhen9Hc96Pf6kawh8RYwPThzIZf4AapnQ65pUJmZAQ8D29z9u10mPQucOgNhKfFjFafa7wjOYpgPtJza7Exl7v4Vd5/o7hXE/91fdvdPAyuAW4Ju714Op5bPLUH/tPmL0d0PAvvNbEbQtATYSoatF8R3M803s8Lg/8qp5ZCR60UX/V0PXgSuNLOSYOvsyqAtsbAPxJzlAM21wE5gN/DVsOsZgvn9IPHNvo3A+uB1LfH9qMuBXcF7adDfiJ8BthvYRPysj9DnY4CXyUeA54LhqcAqoAr4f0Be0J4fjFcF06eGXfcgLIcLgdXBuvEroCQT1wvgfwLbgc3AT4C8TFovgCeIH4/pIL5FcNd7WQ+AzwbLpQr4TG8/V1dci4hIQsm6u0lERJKAQkJERBJSSIiISEIKCRERSUghISLSD4I6AAAAGUlEQVQiCSkkREQkIYWEiIgkpJAQEZGE/j+eaBn3B8aAbAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "XX = list(map(lambda x: x.split(' '), X))\n",
    "def ds_size(n):\n",
    "    return np.sum(list(map(lambda x: 1 if len(x) < n else 0, XX)))\n",
    "\n",
    "# plot the dataset size to get a nice cutting point, we confirm 500 max_len for this dataset\n",
    "plt.xlim(0, 1000)\n",
    "plt.plot(range(len(X)), [ds_size(n) for n in range(len(X))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes = mmap(lambda x:len(x.split(' ')) < 500, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(X)[indexes]\n",
    "y = np.array(y)[indexes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True 8911\n"
     ]
    }
   ],
   "source": [
    "print(len(X) == len(y), len(X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization and Vocabulary Building\n",
    "We now tokenize our words with TreebankWordTokenizer and select the size of the vocabulary to include less words as possible paying attention not generating too many unknowns in the dataset. We plot the most frequent words and discover as excpected a Zipf distribution where the most common words are very few respect all the others. Then we select a threshold to include only the words that maximizes our tradeoff of voc_size vs recall of words found in the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize with TreebankWord tokenizer\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "tokenizer = TreebankWordTokenizer()\n",
    "\n",
    "X_tokens = [tokenizer.tokenize(x) for x in X]\n",
    "y_tokens = [tokenizer.tokenize(x) for x in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatmap(lst):\n",
    "    return [word for sent in lst for word in sent]\n",
    "\n",
    "voc_counts = Counter(flatmap(X_tokens) + flatmap(y_tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we select the firs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1714e6470>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD8CAYAAABpcuN4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHl1JREFUeJzt3Xt0XHW99/H3N0mbNG3Ta1ra9A6xpRQoEEq5HBSr3I5aeZTnlOVRRHzqBY7iQj2oayl4ZC30ETnigxzrAcELlx7QRVUESgEvKJdSS+kNml5o04YmTds0bZr79/lj79QpTCa3mdnZk89rrVkz85u9Z76/zux+sm+/be6OiIhIV/KiLkBERAY2BYWIiKSkoBARkZQUFCIikpKCQkREUlJQiIhISt0GhZkVmdlLZvaqmW0ws1vC9plm9qKZbTGzh81saNheGD6vDF+fkdkuiIhIJvVkjaIZeK+7nw7MBy41s4XAd4E73L0cOABcG05/LXDA3U8C7ginExGRmOo2KDxwOHw6JLw58F7gkbD9fuDD4ePF4XPC1xeZmaWtYhERyaqCnkxkZvnAK8BJwF3AVuCgu7eFk1QBZeHjMmAXgLu3mVk9MA7Y97b3XAosBRg+fPhZc+bM6V9PRGLiSEsb22qPMGNcMSOLhkRdjsTYK6+8ss/dSzP9OT0KCndvB+ab2WjgN8DJySYL75OtPbxjnBB3XwYsA6ioqPDVq1f3qGCRuHt2cw3X3PcyD3zuPM6aPibqciTGzOzNbHxOr456cveDwHPAQmC0mXUGzRRgT/i4CpgKEL4+CtifjmJFckF1fRMARUN00KHEQ0+OeioN1yQws2HA+4BNwLPAR8PJrgYeCx+vCJ8Tvv6Ma+RBkWMK8oKV7tHFQyOuRKRnerLpaRJwf7ifIg9Y7u6/M7ONwENm9h3g78A94fT3AL8ws0qCNYklGahbJLaOtrYDMGxIfsSViPRMt0Hh7uuAM5K0bwMWJGlvAq5MS3UiOagzKLTpSeJCv1SRLNtWGxxtXlSgNQqJBwWFSJYdOhocVZ6Xp9OLJB4UFCJZ1tzWzsjCHh2ZLjIgKChEsqyptYN3nTAy6jJEekxBIZJlW2oaGKE1CokRBYVIlpkZR5rbup9QZIBQUIhkWVNrO/PKRkVdhkiPKShEssjdaWhqY3ihDo2V+FBQiGRRTUMzAO0dERci0gsKCpEsOhzumzh5ko56kvhQUIhkUc2hYI1i+FAd9STxoaAQyaL6oy0AFGsfhcSIgkIki3bUNQIwbWxxxJWI9JyCQiSLdu0PgmJiSVHElYj0nIJCJIv2HQ72UQzJ16In8aFfq0gWHWhspaRIO7IlXhQUIlnU2t7B3MklUZch0isKCpEs2rHvCKOH6VrZEi8KCpEsys/LO3bSnUhcKChEsqi5rZ2TJoyIugyRXlFQiGRJ54CAuhaFxI2CQiRL6o4EZ2W3dmhEQIkXBYVIlhxsbAXg5BN01JPEi4JCJEtqDjUBOtlO4ke/WJEsqaw9DMD0cRrnSeJFQSGSJS1twb6JKWOGRVyJSO8oKESy5PW3GgAoKRoScSUivdNtUJjZVDN71sw2mdkGM/ti2H6zme02s7Xh7fKEeb5mZpVm9rqZXZLJDojExZGW4ES7vDyLuBKR3unJAd1twI3uvsbMRgKvmNnK8LU73P37iROb2VxgCXAKMBl42sze5e7t6SxcJG4OHGll/IjCqMsQ6bVu1yjcvdrd14SPG4BNQFmKWRYDD7l7s7tvByqBBekoViTOjrS0MUM7siWGerWPwsxmAGcAL4ZN15vZOjO718zGhG1lwK6E2apIHSwig8Lm6gaG66xsiaEeB4WZjQAeBW5w90PA3cCJwHygGri9c9Iks3uS91tqZqvNbHVtbW2vCxeJm4J8Y+xwjRwr8dOjoDCzIQQh8St3/zWAu+9193Z37wB+yj82L1UBUxNmnwLseft7uvsyd69w94rS0tL+9EFkwGtsaaOxpZ2pula2xFBPjnoy4B5gk7v/IKF9UsJkVwDrw8crgCVmVmhmM4Fy4KX0lSwSPzv2BdfK1tXtJI568qs9H/g48JqZrQ3bvg5cZWbzCTYr7QA+A+DuG8xsObCR4Iip63TEkwx2ndfKnqY1ComhboPC3f9C8v0Oj6eY51bg1n7UJZJTahvCoNBRTxJDOjNbJAtqwqDQZVAljhQUIlmwY98RAEpH6oQ7iR8FhUgWdO6jyNfwHRJDCgqRLGhu69A5FBJbCgqRLNj81iFOmawr20k8KShEsmDf4RYKtNlJYkpBIZJhjeHw4ho5VuJKQSGSYYebgqA4beroiCsR6RsFhUiG7W9sAWBEYX7ElYj0jYJCJMPqG1sBGJKvxU3iSb9ckQxbV1UPwMzxwyOuRKRvFBQiGdba0QHAiaUjIq5EpG8UFCIZ9uqugwAUFmhxk3jSL1ckw3YfPErx0HyCS7uIxI+CQiTDjra0M7GkKOoyRPpMQSGSYVtrjzBf51BIjCkoRDLocHPnWdkaEFDiS0EhkkFv1gXXoZg8eljElYj0nYJCJIO2hxcsKp8wMuJKRPpOQSGSQS1twTkUZWO0RiHxpaAQyaBN1YcAGD1sSMSViPSdgkIkg3btPwrAGF3dTmJMQSGSQUdb2xmqM7Il5vQLFsmgbfsOc2rZqKjLEOkXBYVIBu0+cJSSooKoyxDpFwWFSIa4Ox0Oo4u1f0LiTUEhkiF76psAmKXrUEjMKShEMmR7bXCy3QwFhcRct0FhZlPN7Fkz22RmG8zsi2H7WDNbaWZbwvsxYbuZ2Z1mVmlm68zszEx3QmQgen7rPgBOmVwScSUi/dOTNYo24EZ3PxlYCFxnZnOBm4BV7l4OrAqfA1wGlIe3pcDdaa9aJAZ21jUCugSqxF+3QeHu1e6+JnzcAGwCyoDFwP3hZPcDHw4fLwZ+7oEXgNFmNintlYsMcNv3HaGkqEAXLJLY69U+CjObAZwBvAhMdPdqCMIEmBBOVgbsSpitKmx7+3stNbPVZra6tra295WLDHA79zcydWxx1GWI9FuPg8LMRgCPAje4+6FUkyZp83c0uC9z9wp3rygtLe1pGSKx4O4cbm5jigYDlBzQo6AwsyEEIfErd/912Ly3c5NSeF8TtlcBUxNmnwLsSU+5IvFwsLEVgDOnjYm4EpH+68lRTwbcA2xy9x8kvLQCuDp8fDXwWEL7J8KjnxYC9Z2bqEQGi85RY4uH5kdciUj/9WRsgfOBjwOvmdnasO3rwG3AcjO7FtgJXBm+9jhwOVAJNALXpLVikRjYsCcIirNnjo24EpH+6zYo3P0vJN/vALAoyfQOXNfPukRibdu+wwDMGKdDYyX+dGa2SAbsPhgM31E0RJueJP4UFCIZ8NfKfSzQZifJEQoKkTQ72tJOW4dTqAsWSY7QL1kkzXYfDC5/umjOhG6mFIkHBYVImtUcCvZPjB9ZGHElIumhoBBJs621wRFPGgxQcoWCQiTNNr/VAOjQWMkdCgqRNKsOr2w3vFDXypbcoKAQSbM1Ow8waVRR1GWIpI2CQiTNDja2Uj5xZNRliKSNgkIkjd6sC66TfVrZqIgrEUkfBYVIGr2wrQ6AihkaXlxyh4JCJI02VQdHPJ02ZXTElYikj4JCJI227ws2PY0dPjTiSkTSR0EhkkaHm9s4VfsnJMcoKETS6JU3DzBGaxOSYxQUImlytKUdgCF5XV3nSySeFBQiabJzfyMAF58yMeJKRNJLQSGSJk9ueAuAE0YNi7gSkfRSUIikyYY99QCcf+K4iCsRSS8FhUiaPF9Zx7jhQynI12IluUW/aJE0cHcON7dxzixdJ1tyj4JCJA3erAt2ZJdP0GCAknsUFCJp8NTGYEf2WdM1xpPkHgWFSBps2Rtc/vSCk8ZHXIlI+ikoRNLgmc015OcZeTrZTnKQgkIkDeqOtGizk+SsboPCzO41sxozW5/QdrOZ7TazteHt8oTXvmZmlWb2upldkqnCRQaKusPNAMyfqqHFJTf1ZI3iPuDSJO13uPv88PY4gJnNBZYAp4Tz/NjM8tNVrMhAtKUm2D9x0oQREVcikhndBoW7/wnY38P3Www85O7N7r4dqAQW9KM+kQHvpe3B4jF3UknElYhkRn/2UVxvZuvCTVOdG2fLgF0J01SFbe9gZkvNbLWZra6tre1HGSLRenrTXgDmnKBzKCQ39TUo7gZOBOYD1cDtYXuyQz482Ru4+zJ3r3D3itLS0j6WIRK9dVX1TB5VpKE7JGf16Zft7nvdvd3dO4Cf8o/NS1XA1IRJpwB7+leiyMC1Kxxa/N2zJ0RciUjm9CkozGxSwtMrgM4jolYAS8ys0MxmAuXAS/0rUWTgeu71GgDed7KCQnJXQXcTmNmDwHuA8WZWBXwLeI+ZzSfYrLQD+AyAu28ws+XARqANuM7d2zNTukj0/rxlHwDnzNLQ4pK7ug0Kd78qSfM9Kaa/Fbi1P0WJxMXLO/YzsrCAEYXdLkoisaW9byJ9dOBICwcaW6mYoTOyJbcpKET66NE1VQBcduqkbqYUiTcFhUgfrdoU7Mj+0OmTI65EJLMUFCJ99MrOA4wfUUjREI1SI7lNQSHSB+5OS1sH88o0bIfkPgWFSB90ju903ok6LFZyn4JCpA+e2BBc+vSCkzT8jOQ+BYVIHzz+WjUAJ0/SQICS+xQUIr10qKmVvYeamVdWgpkufSq5T0Eh0kvrdtUD8C8VU7uZUiQ3KChEeumXL7wJwIKZ2pEtg4OCQqSXOndkz9aFimSQUFCI9MLW2uD62JeeckLElYhkj4JCpBf+Z3UwvtPHz50ecSUi2aOgEOmFP6wPDos9V9efkEFEQSHSQwcbW3izrpHTp44mL0+HxcrgoaAQ6aEVrwaXf7/qbB0WK4OLgkKkhzove/pBDSsug4yCQqQH3J2VG/cyungIw3XZUxlkFBQiPfC7dcFO7Mvm6bBYGXwUFCI9cNsfNgNw48WzI65EJPsUFCLdqDvczO6DRymfMILxIwqjLkck6xQUIt14dE1wkt3SC2dFXIlINBQUIt1YHp6NraOdZLBSUIik0NreQWXNYaaPK6ZoSH7U5YhEQkEhksJDL+0E4KNnTom4EpHoKChEUug82ulTF8yMuBKR6HQbFGZ2r5nVmNn6hLaxZrbSzLaE92PCdjOzO82s0szWmdmZmSxeJJPW767nSEs7J5YO10l2Mqj1ZI3iPuDSt7XdBKxy93JgVfgc4DKgPLwtBe5OT5ki2fefT28B4PtXnh5xJSLR6jYo3P1PwP63NS8G7g8f3w98OKH95x54ARhtZpPSVaxItjS3tfP0pr0U5BlnTBsTdTkikerrPoqJ7l4NEN5PCNvLgF0J01WFbe9gZkvNbLWZra6tre1jGSKZ8cNwbUL7JkTSvzM72SD9nmxCd1/m7hXuXlFaWprmMkT6zt358XNbAbjx4ndFXI1I9PoaFHs7NymF9zVhexWQOFj/FGBP38sTyb7Oy51+4LRJFBbo3AmRvgbFCuDq8PHVwGMJ7Z8Ij35aCNR3bqISiYOm1na++ug6AL69eF7E1YgMDN0e82dmDwLvAcabWRXwLeA2YLmZXQvsBK4MJ38cuByoBBqBazJQs0jG3LxiAxAM1zF2+NCIqxEZGLoNCne/qouXFiWZ1oHr+luUSBTaO5yHXg6Oxbhdh8SKHKMzs0VCD70cDNfx8YXTGVqgRUOkk5YGEeBoSzvf+E0w+ICOdBI5noJCBLj9qdeB4FKno4u1b0IkkYJCBr3q+qP891+2A/Cjq86IuBqRgUdBIYPeNT97GYCvXTaHgnwtEiJvp6VCBrWNew6x+a0GCgvydKlTkS4oKGTQau9wLr/zzwD88tPnYJZsBBoRUVDIoHX3c5UAzJ1UwtkzxkZcjcjApaCQQelIcxvff+oNAJZ/9tyIqxEZ2BQUMih96eG1AHz23ScyQlevE0lJQSGDzjOb9/LUxr0AfOWS2RFXIzLwKShkUGlsaeNT960GYPlnziU/TzuwRbqjoJBBZcmyFwD45HkzWDBTO7BFekJBIYPGb/5exbqqegC++YG5EVcjEh8KChkU9h9p4UsPvwrAn796EXna5CTSYwoKyXnuzj+HJ9Zdd9GJTB1bHHFFIvGioJCcd/2Df6e6vomy0cP4yiVzoi5HJHYUFJLTfvj0Fn6/Lrhs++/+7YKIqxGJJwWF5KyNew5xx9PB2dePf+GfGKNrYIv0iYJCclJDU+uxAf8e+PQ5zJ1cEnFFIvGloJCcc7i5jfNuewaAD54+mfNOGh9xRSLxpkFuJKe0tndw1n+spLmtg1mlw7lzyfyoSxKJPa1RSM5obe9g0e1/pLmtgxnjiln5pXfrGhMiaaCgkJxx/QNr2Lm/kSH5xh++eKHGcRJJEwWF5ITvPbGZJzcEI8Ku/ebFDBuaH3FFIrlDQSGxd8fKN/jxc1uBYHiO4bq+hEhaKSgk1u57fjs/XLUFgEc/d56G5xDJgH796WVmO4AGoB1oc/cKMxsLPAzMAHYA/9vdD/SvTJHjtXc4X31kHY+uqQLg91+4gFMmj4q4KpHclI41iovcfb67V4TPbwJWuXs5sCp8LpI27s7H/vuFYyHxu39TSIhkUiY25i4G3hM+vh94Dvj3DHyODEINTa18+K7n2Vp7BICXvr6ICSVFEVclktv6GxQOPGVmDvzE3ZcBE929GsDdq81sQn+LFAFYv7ueD/zoLwCUFBXwx69cpPGbRLKgv0FxvrvvCcNgpZlt7umMZrYUWAowbdq0fpYhue7366q57oE1ACyYMZZ7rzmbETq6SSQr+rWkufue8L7GzH4DLAD2mtmkcG1iElDTxbzLgGUAFRUV3p86JHc1trTxvSde576/7gDgux85lX85W39YiGRTn4PCzIYDee7eED6+GPg2sAK4GrgtvH8sHYXK4LNlbwMfufuvHGpqA+CRz55LxYyxEVclMvj0Z41iIvCbcCydAuABd3/CzF4GlpvZtcBO4Mr+lymDibvz+GtvHdvUNOeEkdx3zQJOGKWd1iJR6HNQuPs24PQk7XXAov4UJYPXpupDfOnhtWx+qwGAmz84l0+ePzPiqkQGN+0NlAGhoamVW367kUdeCc6NyM8zHvw/C1kwU5uaRKKmoJBIHWxs4Sd/2sbd4VhNw4bkc8P7yvnk+TMoLNDAfiIDgYJCItHR4dz/tx3c8tuNx9pufP+7+PxFJ2l4cJEBRkEhWXW0pZ3/XPUGj6yuou5ICwD/VD6eby+ex8zxwyOuTkSSUVBIxrV3OH/eUstP/riNv22rO9Z+ySkT+d5HTmdU8ZAIqxOR7igoJKMeeHEnP3t+O1tqDh9r+8blJ/PPp01i8uhhEVYmIj2loJC0OtTUSt3hFr752Hq27ztC1YGjAJxQUsRdHzuT8okjKCnSGoRInCgoJC1W79jPhj2H+NaKDcfa8vOMK8+awmfefSInTRgRYXUi0h8KCumXO1a+wYY99Ty96R9Der1/7kQ+ePpk3jO7VGsPIjlAQSE9Vn+0lU3Vh1i76yDff/J1ANo6gvEc55WV8OkLZnHR7AnaOS2SYxQUkpK78/O/vUlNQxN3Pbv1uNc+ds40xhQP5WMLpzFplHZMi+QqBYUcp6m1nabWdu7+41b+trWOfQ3N7KlvAiDPoGL6WG54fzkTRhZy0oSREVcrItmgoBCer9xH3ZEW6g43H3emNMBFs0s5fepobvnQKbrkqMggpaAYhA43t/G9JzbT2NJOdf1Rnq+sO+71j5w5hVPLSnjf3IlMGVMcUZUiMlAoKHLUwcYW9h5qBqC5rZ3P/2oNLW0dmHGs3QwmlRQxeVQRt15xKlPHFlM8NF8nwonIcRQUOeKZzXvZVnvk2PPv/H7TO6aZPq6Yc2eNA2DM8KF8+eLZGoBPRLqloIih3766h+Wrdx173t7h/HVr3Tume//ciVxxRhkARUPyuLC8lIL8vKzVKSK5QUExgGzYU0/d4Zbj2n70zBZe3nEg6fRnTBt97PFZ08fw5YtnM6+sBIA8M4YX6usVkf7T/yQRePjlnVQmDJIHUNPQzGNr93Q5zxcWlR/3/L1zJjB/6uguphYRSR8FRT+0tHVwsLEl6WvNbR188aG/09jSflx7a3sHW8N9CcVD/3EFtw53CgvyuOVDp1A+8fhxkconjtRQGCISGQVFF9o7nCc3vPWO/+gTffWRVwlHsOjS+BFDOWv6mOPa5kwq4cb3v4tZpRooT0QGvkEZFA+8uJMXtr1z52+itbsOsnN/Y7fvdfKkEv514bSkrw0bks8HTpvM0ALtQBaR+BqwQbH3UBM14fH+PbGlpoFbfrsRM+jugM8Dja0AKS+9mZ9nzJ44kjuvOuO4TUSJzKBs9DDMdIipiOSuAREUNQ3N3PVs5bHn7R3OD1a+0af3umzeCZSOLEw5TZ4ZVy2YxuwTNFaRiEh3BkRQ7D3UxP8Nh61O9LFzpnHR7Ak9fp9xI4ZyxrQx3U8oIiI9NiCCYt7kUfz1O5ce15ZnxhCdHCYiErkBERRmUFiQfD+AiIhES3+yi4hIShkLCjO71MxeN7NKM7spU58jIiKZlZGgMLN84C7gMmAucJWZzc3EZ4mISGZlao1iAVDp7tvcvQV4CFicoc8SEZEMytTO7DJgV8LzKuCcxAnMbCmwNHzabGbrM1TLQDAe2Bd1ERmk/sVXLvcNcr9/s7PxIZkKimSnKh83KpK7LwOWAZjZanevyFAtkVP/4i2X+5fLfYPB0b9sfE6mNj1VAVMTnk8Buh5DW0REBqxMBcXLQLmZzTSzocASYEWGPktERDIoI5ue3L3NzK4HngTygXvdfUOKWZZloo4BRP2Lt1zuXy73DdS/tDD3bi6oICIig5rOzBYRkZQUFCIiklLagsLMZpvZ2oTbITO7wczGmtlKM9sS3o8JpzczuzMc4mOdmZ2Z8F5Xh9NvMbOrE9rPMrPXwnnutCxeMShF/242s90J7ZcnzPO1sNbXzeyShPakw5uEO/9fDPv9cHggQNaY2ZfMbIOZrTezB82sqKuazKwwfF4Zvj6jr/2OuH/3mdn2hO9vfjht3H6fXwz7tcHMbgjbcmLZS9G/2C57ZnavmdVYwvlj2fi+uvqMbrl72m8EO7DfAqYD3wNuCttvAr4bPr4c+APBORcLgRfD9rHAtvB+TPh4TPjaS8C54Tx/AC7LRP297N/NwJeTTDMXeBUoBGYCW8P58sPHs4Ch4TRzw3mWA0vCx/8FfC6LfSoDtgPDEmr5ZFc1AZ8H/it8vAR4uK/9jrh/9wEfTTJ9bH6fwDxgPVBMcIDK00B5rix7KfoX22UPuBA4E1if0Jbx76urz+julqlNT4uAre7+JsHQHfeH7fcDHw4fLwZ+7oEXgNFmNgm4BFjp7vvd/QCwErg0fK3E3f/mQS9/nvBe2ZbYv64sBh5y92Z33w5UEgxtknR4kzDx3ws8Es6f+G+VLQXAMDMrIFgoq1PUlPi9PgIsCvvQq35noU+J3t6/VOf2xOn3eTLwgrs3unsb8EfgCnJn2euqf10Z8Mueu/8J2J+k7kx/X119RkqZCoolwIPh44nuXg0Q3ndesi7ZMB9l3bRXJWmPQmL/AK4PVwnvTViV623/xgEHwwUhsT0r3H038H1gJ0FA1AOvpKjpWD/C1+sJ+tDbfmdFsv65+1Phy7eG398dZtZ5Hd04/T7XAxea2TgzKyb4C3QqubPsddU/yIFlL0E2vq+uPiOltAdFuG3vQ8D/dDdpkjbvQ3tWJenf3cCJwHyC/4Bu75w0yewDtn/hQraYYFV9MjCcYPTfrmqKff/M7F+BrwFzgLMJVuH/vXOWJG8zIPvn7puA7xL8RfkEwSaVthSzxKZvkLJ/ObHs9UDk/cnEGsVlwBp33xs+3xuuChHe14TtXQ3zkap9SpL2bDuuf+6+193b3b0D+CnB6i30vn/7CFYpC97Wni3vA7a7e627twK/Bs5LUdOxfoSvjyJYle5tv7Mlaf/cvTpcpW8Gfkbfv79If5/ufo+7n+nuFxJ8D1vIoWUvWf9yaNnrlI3vq6vPSCkTQXEVx2+WWQF07o2/Gngsof0T4R79hQSbAqoJzua+2MzGhH8FXgw8Gb7WYGYLw22Kn0h4r2w6rn+d/+ihKwhWkyHo3xILjg6aSbDz7SW6GN4k3Jb4LPDRcP7Ef6ts2AksNLPi8N93EbAxRU2J3+tHgWfCPvSq31noV6dk/duUsNAYwfbaxO8vNr9PM5sQ3k8D/hfBbzRnlr1k/cuhZa9TNr6vrj4jtZ7s8e7pjWAHYR0wKqFtHLCK4C+cVcDYsN0ILm60FXgNqEiY51MEO6AqgWsS2isIfgxbgf9HeGZ5tm5d9O8XYf3rwi9hUsJr3whrfZ2Eo0QItrG+Eb72jYT2WQQ/6EqCTVuFWe7fLcDm8N/4FwRHjSStCSgKn1eGr8/qa78j7t8z4fe3HvglMCKOv0/gzwTB/iqwKGzLpWUvWf9iu+wRBHk10EqwBnBtNr6vrj6ju5uG8BARkZR0ZraIiKSkoBARkZQUFCIikpKCQkREUlJQiIhISgoKERFJSUEhIiIp/X9owMTK/7K3XAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# get only the most frequent words. It follows a zipf distribution as expected\n",
    "# we need to choose a proper threshold, \n",
    "py = mmap(lambda x: x[1], voc_counts.most_common())[::-1]\n",
    "px = range(0, len(py))\n",
    "plt.xlim(70000, 100000)\n",
    "plt.ylim(0, 300)\n",
    "plt.plot(px, py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The final vocabulary size is 13128 against the unfiltered 88128\n"
     ]
    }
   ],
   "source": [
    "# a vocabulary of 13k words seems fine for this task, we probably can compress more\n",
    "# for saving memory and computation time but let's test this choice first\n",
    "print('The final vocabulary size is', len(py) - 75000, 'against the unfiltered', len(py))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make the cut\n",
    "voc = voc_counts.most_common()[:13128]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now generate the vocabulary and the inverse vocabulary, and we also need to adapt our dataset to be the input of our neural network. As the input follows a seq2seq model we need to remember to define special symbols which are the START and the END tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BUILD THE VOCABULARY\n",
    "\n",
    "words = getel(0, voc)\n",
    "w2id = dict(zip(words, range(len(words))))\n",
    "\n",
    "# add <start>, <end>, <unk> token to vocabulary\n",
    "w2id['<start>'] = len(w2id.keys())\n",
    "w2id['<end>'] = len(w2id.keys())\n",
    "w2id['<unk>'] = len(w2id.keys())\n",
    "w2id['<pad>'] = len(w2id.keys())\n",
    "\n",
    "# BUILD THE REVERSE VOCABULARY\n",
    "\n",
    "id2w = dict(mmap(lambda x: (x[1], x[0]), w2id.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONVERT X TOKENS TO INDEXES\n",
    "\n",
    "X_id = [[w2id.get(word, w2id['<unk>']) for word in sent] for sent in X_tokens]\n",
    "\n",
    "# ADD START AND END TOKEN AND CONVERT Y TO ID\n",
    "y_id = [[w2id['<start>']] + \\\n",
    "        [w2id.get(word, w2id['<unk>']) for word in sent] + \\\n",
    "        [w2id['<end>']] for sent in y_tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# SEQUENCE PADDING\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "X_id=pad_sequences(X_id, value=w2id['<pad>'], padding='post')\n",
    "y_id = pad_sequences(y_id, value=w2id['<pad>'], padding='post')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adapting the dataset to the network\n",
    "We need to generated 3 input for our network:\n",
    "* The X input which holds or the articles, composed by padded sequences of indexes\n",
    "* The Y input which holds indexes for the summaries, format is like the X\n",
    "* The G input, with the same shape of Y, holds the information about if the index in the same position as Y will be generated by the network, or selected from the input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting example n° 0\n",
      "starting example n° 1000\n",
      "starting example n° 2000\n",
      "starting example n° 3000\n",
      "starting example n° 4000\n",
      "starting example n° 5000\n",
      "starting example n° 6000\n",
      "starting example n° 7000\n",
      "starting example n° 8000\n",
      "CPU times: user 1min 35s, sys: 84.1 ms, total: 1min 35s\n",
      "Wall time: 1min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_id_new = []\n",
    "pad_index = w2id['<pad>']\n",
    "def p_gen_index(elem, inp, i, log=False, padding=pad_index):\n",
    "    if i % 1000 == 0 and log:\n",
    "        print('starting example n°', i)\n",
    "    if elem == padding:\n",
    "        y_id_new.append(padding)\n",
    "        return padding\n",
    "    try:\n",
    "        index = inp.tolist().index(elem)\n",
    "        y_id_new.append(index)\n",
    "        return 0\n",
    "    except:\n",
    "        y_id_new.append(elem)\n",
    "        return 1\n",
    "\n",
    "gen = [[p_gen_index(elem, ex, i, log=i2 == 0)  for i2, elem in enumerate(y)] for i, (ex, y) in enumerate(zip(X_id, y_id))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_final = np.array(y_id_new).reshape(y_id.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now inspect how X, Y and Gen looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   82,    90,  5240, 13130,     8, 13130,    43,    34,   366,\n",
       "          42,  6331,  4759,    55,     4,  8777, 13130,  2807,     1,\n",
       "       13130,    34, 13130,    16, 13130,    42,  6813,    77,  1998,\n",
       "           4,     7,    68,    42,    22,   651,     1,  3182, 13130,\n",
       "          61,    21,     3,   333,     2,   417,    43,  1934,  1802,\n",
       "          25,    32,   249,   153,    19,   799,     7,    46,    31,\n",
       "          21, 13130,    42,    51,    30,    77,    14,  8351,     7,\n",
       "          46,    31,    21,   317,     1,  3182, 13130, 13130,    31,\n",
       "         633,     5,  5047,    51,    42,    22,   651,   717,  2808,\n",
       "        9229,   394,    50,   200,    34,    11,   227, 13130,  3101,\n",
       "        8136,     4,   494,  5697,   958,  3402,    82,     1,   931,\n",
       "        1998,    25,    42,    22,   213,   134,  5779,  9496,  3327,\n",
       "          57,     2,  2431,     5,     0,   102,   108,    21, 13130,\n",
       "          74,     1,     1,   744,  3711,    20,     0,   130,   263,\n",
       "           7,   182,  4330,     1,    24,  1027, 13130,  1935,   320,\n",
       "          19,     3,   366,  1413,  5111,     2,   421,    57,     3,\n",
       "         284,    31,     8, 13130,     7,  1453,    59,  6039,  1998,\n",
       "          45, 12709,     7,    66,  7418,  5388,  1802,    63,    24,\n",
       "           3,  8351,   661,     4,   320,    24,  3555,     5,     3,\n",
       "        2021,   144,  1033,   824,   934,    54,  3863,    89,  4242,\n",
       "           1,   254,     9,  1998,  1935,    31,   514,    32,    21,\n",
       "       13130,   169,   767,    37,  1820,   312,  1258,     7,     3,\n",
       "        1323,  1874,  7419,    51,  6040, 13130,     3,   313,  9019,\n",
       "        4938,     4,   636,     2,     0,   156,  1298,     2, 13130,\n",
       "       13130, 13130,  1988,  6332,  5531,  6814,     3,  5698,    63,\n",
       "        1266,    54,   313,   590,     6,    59, 13130,  1802,  3182,\n",
       "          45,   684, 13130,   178, 13130,   725, 13130, 13130, 13130,\n",
       "       13130,  7271,    45, 13130, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131], dtype=int32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_id[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([13128,    52,   381,    40,   110,    73,   112,   113,   114,\n",
       "          37,     3,   157,   158,    44,   160,   130,    38,    61,\n",
       "        4636,     3,   400,    38,   298,   631,    73,   299,   156,\n",
       "          52,    37,     3,    15,   113,   130,    44,   476,    17,\n",
       "        1693,   171,  8114,   108,  2877,     6,    44, 13129, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131, 13131,\n",
       "       13131, 13131, 13131])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_final[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131,\n",
       " 13131]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate2text(indexes):\n",
    "    return ' '.join([id2w[id_] for id_ in indexes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<start> ron paul too much government <unk> in economy caused crisis. paul says the <unk> are another case of excessive intervention in economy. the government isnt letting the market adjust prices to lower levels paul says. paul bailout will only increase financial instability in the long run <end> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translate2text(y_id[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ron paul too much government meddling in economy caused crisis. paul says the bailouts are another case of excessive intervention in economy. the government isnt letting the market adjust prices to lower levels paul says. paul bailout will only increase financial instability in the long run'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split Train, Val, Test\n",
    "We split the dataset to train, validation and test set in order to pick and evaluate the best model. We do a 80-10-10 split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val_test, y_train, \\\n",
    "y_val_test, y_raw_train, y_raw_val_test,\\\n",
    "gen_train, gen_val_test = train_test_split(X_id, y_final, y_id, np.array(gen), test_size=0.20 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val, X_test, y_val, \\\n",
    "y_test, y_raw_val, y_raw_test,\\\n",
    "gen_val, gen_test = train_test_split(X_val_test, y_val_test, y_raw_val_test, gen_val_test, test_size=0.50 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.makedirs('processed', exist_ok=True)\n",
    "\n",
    "np.save('processed/X_train', X_train)\n",
    "np.save('processed/X_val', X_val)\n",
    "np.save('processed/X_test', X_test)\n",
    "\n",
    "np.save('processed/y_train', y_train)\n",
    "np.save('processed/y_val', y_val)\n",
    "np.save('processed/y_test', y_test)\n",
    "\n",
    "np.save('processed/y_raw_train', y_raw_train)\n",
    "np.save('processed/y_raw_val', y_raw_val)\n",
    "np.save('processed/y_raw_test', y_raw_test)\n",
    "\n",
    "np.save('processed/gen_train', gen_train)\n",
    "np.save('processed/gen_val', gen_val)\n",
    "np.save('processed/gen_test', gen_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('processed/w2id.pkl', 'wb') as f:\n",
    "    dump(w2id, f)\n",
    "    \n",
    "with open('processed/id2w.pkl', 'wb') as f:\n",
    "    dump(id2w, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}