recsys.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#It uses the database from https://datasets.imdbws.com/\n",
    "\n",
    "ratings = pd.read_csv(r'datasets/ratings.tsv',sep='\\t', low_memory=False)\n",
    "akas = pd.read_csv(r'datasets/akas.tsv',sep='\\t',low_memory=False)\n",
    "basics = pd.read_csv(r'datasets/basics.tsv',sep='\\t',low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(basics,ratings,on='tconst')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns=['isAdult', 'runtimeMinutes'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['startYear'] = [0 if year == '\\\\N' else int(year) for year in df['startYear']]\n",
    "df['endYear'] = [year if year == '\\\\N' else int(year) for year in df['endYear']]\n",
    "df = df[(df['numVotes']>=2500) & (df['averageRating']>=6.4) & (df['startYear']>=1960)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tconst</th>\n",
       "      <th>titleType</th>\n",
       "      <th>primaryTitle</th>\n",
       "      <th>originalTitle</th>\n",
       "      <th>startYear</th>\n",
       "      <th>endYear</th>\n",
       "      <th>genres</th>\n",
       "      <th>averageRating</th>\n",
       "      <th>numVotes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tt0035423</td>\n",
       "      <td>movie</td>\n",
       "      <td>Kate &amp; Leopold</td>\n",
       "      <td>Kate &amp; Leopold</td>\n",
       "      <td>2001</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Comedy,Fantasy,Romance</td>\n",
       "      <td>6.4</td>\n",
       "      <td>77701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tt0052698</td>\n",
       "      <td>movie</td>\n",
       "      <td>Classe Tous Risques</td>\n",
       "      <td>Classe tous risques</td>\n",
       "      <td>1960</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Crime,Drama,Romance</td>\n",
       "      <td>7.5</td>\n",
       "      <td>3102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tt0052832</td>\n",
       "      <td>movie</td>\n",
       "      <td>The Fugitive Kind</td>\n",
       "      <td>The Fugitive Kind</td>\n",
       "      <td>1960</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Drama,Romance</td>\n",
       "      <td>7.2</td>\n",
       "      <td>5398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tt0052997</td>\n",
       "      <td>movie</td>\n",
       "      <td>The League of Gentlemen</td>\n",
       "      <td>The League of Gentlemen</td>\n",
       "      <td>1960</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Comedy,Crime,Thriller</td>\n",
       "      <td>7.3</td>\n",
       "      <td>3757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tt0053106</td>\n",
       "      <td>movie</td>\n",
       "      <td>Letter Never Sent</td>\n",
       "      <td>Neotpravlennoe pismo</td>\n",
       "      <td>1960</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Adventure,Drama</td>\n",
       "      <td>7.9</td>\n",
       "      <td>2940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13940</th>\n",
       "      <td>tt9827854</td>\n",
       "      <td>series</td>\n",
       "      <td>Hollywood</td>\n",
       "      <td>Hollywood</td>\n",
       "      <td>2020</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Drama</td>\n",
       "      <td>7.6</td>\n",
       "      <td>24608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13941</th>\n",
       "      <td>tt9877170</td>\n",
       "      <td>movie</td>\n",
       "      <td>Malang</td>\n",
       "      <td>Malang - Unleash the Madness</td>\n",
       "      <td>2020</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Action,Crime,Drama</td>\n",
       "      <td>6.5</td>\n",
       "      <td>6853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13942</th>\n",
       "      <td>tt9879074</td>\n",
       "      <td>series</td>\n",
       "      <td>The Disappearance of Madeleine McCann</td>\n",
       "      <td>The Disappearance of Madeleine McCann</td>\n",
       "      <td>2019</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Crime,Documentary,Drama</td>\n",
       "      <td>6.6</td>\n",
       "      <td>7545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13943</th>\n",
       "      <td>tt9900092</td>\n",
       "      <td>series</td>\n",
       "      <td>Motherland: Fort Salem</td>\n",
       "      <td>Motherland: Fort Salem</td>\n",
       "      <td>2020</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Drama,Fantasy,Sci-Fi</td>\n",
       "      <td>7.1</td>\n",
       "      <td>5111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13944</th>\n",
       "      <td>tt9900782</td>\n",
       "      <td>movie</td>\n",
       "      <td>Kaithi</td>\n",
       "      <td>Kaithi</td>\n",
       "      <td>2019</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Action,Thriller</td>\n",
       "      <td>8.5</td>\n",
       "      <td>8161</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13945 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          tconst titleType                           primaryTitle  \\\n",
       "0      tt0035423     movie                         Kate & Leopold   \n",
       "1      tt0052698     movie                    Classe Tous Risques   \n",
       "2      tt0052832     movie                      The Fugitive Kind   \n",
       "3      tt0052997     movie                The League of Gentlemen   \n",
       "4      tt0053106     movie                      Letter Never Sent   \n",
       "...          ...       ...                                    ...   \n",
       "13940  tt9827854    series                              Hollywood   \n",
       "13941  tt9877170     movie                                 Malang   \n",
       "13942  tt9879074    series  The Disappearance of Madeleine McCann   \n",
       "13943  tt9900092    series                 Motherland: Fort Salem   \n",
       "13944  tt9900782     movie                                 Kaithi   \n",
       "\n",
       "                               originalTitle  startYear endYear  \\\n",
       "0                             Kate & Leopold       2001      \\N   \n",
       "1                        Classe tous risques       1960      \\N   \n",
       "2                          The Fugitive Kind       1960      \\N   \n",
       "3                    The League of Gentlemen       1960      \\N   \n",
       "4                       Neotpravlennoe pismo       1960      \\N   \n",
       "...                                      ...        ...     ...   \n",
       "13940                              Hollywood       2020      \\N   \n",
       "13941           Malang - Unleash the Madness       2020      \\N   \n",
       "13942  The Disappearance of Madeleine McCann       2019      \\N   \n",
       "13943                 Motherland: Fort Salem       2020      \\N   \n",
       "13944                                 Kaithi       2019      \\N   \n",
       "\n",
       "                        genres  averageRating  numVotes  \n",
       "0       Comedy,Fantasy,Romance            6.4     77701  \n",
       "1          Crime,Drama,Romance            7.5      3102  \n",
       "2                Drama,Romance            7.2      5398  \n",
       "3        Comedy,Crime,Thriller            7.3      3757  \n",
       "4              Adventure,Drama            7.9      2940  \n",
       "...                        ...            ...       ...  \n",
       "13940                    Drama            7.6     24608  \n",
       "13941       Action,Crime,Drama            6.5      6853  \n",
       "13942  Crime,Documentary,Drama            6.6      7545  \n",
       "13943     Drama,Fantasy,Sci-Fi            7.1      5111  \n",
       "13944          Action,Thriller            8.5      8161  \n",
       "\n",
       "[13945 rows x 9 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('datasets/imdb_large.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Simplify the title_types\n",
    "def select_titles(title_type):\n",
    "    if title_type in ['movie','tvMovie','tvSpecial']:\n",
    "        return 'movie'\n",
    "    elif title_type in ['tvSeries','tvMiniSeries']:\n",
    "        return 'series'\n",
    "    else:\n",
    "        return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['titleType'] =  df['titleType'].apply(select_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "movie     10793\n",
       "series     3152\n",
       "Name: titleType, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb['titleType'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new column title with movie/tv name with date\n",
    "title = []\n",
    "for row in imdb[['titleType','primaryTitle','startYear','endYear']].itertuples():\n",
    "    if row.titleType == 'movie':\n",
    "        title.append('{} ({})'.format(row.primaryTitle,row.startYear))\n",
    "    else:\n",
    "        title.append( '{} ({} -)'.format(row.primaryTitle,row.startYear) if row.endYear == '\\\\N' else '{} ({} - {})'.format(row.primaryTitle,row.startYear, row.endYear))\n",
    "imdb['title']=title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'imdb' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-b9c7f557acfe>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mimdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m8838\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'title'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'imdb' is not defined"
     ]
    }
   ],
   "source": [
    "imdb.iloc[8838]['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb.to_csv('datasets/imdb.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recomm Builder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb = pd.read_csv('datasets/imdb.csv', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "cv = CountVectorizer(dtype=np.uint8)\n",
    "MMS = MinMaxScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb['genres'] = [genre.replace(',', ' ') for genre in imdb['genres']]\n",
    "dtm = cv.fit_transform(imdb['genres']).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reshape(-1,1) is equivalent to transpose\n",
    "new_matrix = np.concatenate((dtm, np.array(imdb['averageRating']).reshape(-1,1)),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "numVotes = np.array(imdb['numVotes']).reshape(-1,1)\n",
    "numVotes = MMS.fit_transform(numVotes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_matrix = np.concatenate((new_matrix,numVotes),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities = cosine_similarity(new_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_recoms(titles_ids, type='movie'):\n",
    "    number_of_titles = len(titles_ids)\n",
    "    sim = similarities[[8838,7600]].sum(axis=0).argsort()[::-1][number_of_titles:500]   # To omit recommending the input movies\n",
    "    tv_similar = [(imdb['title'].iloc[index], imdb['tconst'].iloc[index]) for index in sim if imdb['titleType'].iloc[index]=='series']\n",
    "    movie_similar = [(imdb['title'].iloc[index], imdb['tconst'].iloc[index]) for index in sim if imdb['titleType'].iloc[index]=='movie']\n",
    "    \n",
    "    if type=='movie':\n",
    "        return movie_similar[:10]\n",
    "    else:\n",
    "        return tv_similar[:10]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Jurassic Park (1993)', 'tt0107290'),\n",
       " ('Star Wars: Episode VII - The Force Awakens (2015)', 'tt2488496'),\n",
       " ('Avengers: Infinity War (2018)', 'tt4154756'),\n",
       " ('Captain America: The Winter Soldier (2014)', 'tt1843866'),\n",
       " ('X-Men: Days of Future Past (2014)', 'tt1877832'),\n",
       " ('Aliens (1986)', 'tt0090605'),\n",
       " ('The Hunger Games (2012)', 'tt1392170'),\n",
       " ('Captain America: Civil War (2016)', 'tt3498820'),\n",
       " ('X-Men: First Class (2011)', 'tt1270798'),\n",
       " ('Avengers: Age of Ultron (2015)', 'tt2395427')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_recoms([8838,7600,134344134,23442]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tconst</th>\n",
       "      <th>titleType</th>\n",
       "      <th>startYear</th>\n",
       "      <th>endYear</th>\n",
       "      <th>genres</th>\n",
       "      <th>averageRating</th>\n",
       "      <th>numVotes</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tt0035423</td>\n",
       "      <td>movie</td>\n",
       "      <td>2001</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Comedy Fantasy Romance</td>\n",
       "      <td>6.4</td>\n",
       "      <td>77701</td>\n",
       "      <td>Kate &amp; Leopold (2001)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8711</th>\n",
       "      <td>tt1314652</td>\n",
       "      <td>movie</td>\n",
       "      <td>2010</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Drama Thriller</td>\n",
       "      <td>6.4</td>\n",
       "      <td>9126</td>\n",
       "      <td>The Housemaid (2010)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2809</th>\n",
       "      <td>tt0099819</td>\n",
       "      <td>movie</td>\n",
       "      <td>1990</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Comedy Crime</td>\n",
       "      <td>6.4</td>\n",
       "      <td>13124</td>\n",
       "      <td>I Love You to Death (1990)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2806</th>\n",
       "      <td>tt0099800</td>\n",
       "      <td>movie</td>\n",
       "      <td>1990</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Comedy Music Romance</td>\n",
       "      <td>6.4</td>\n",
       "      <td>12792</td>\n",
       "      <td>House Party (1990)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2805</th>\n",
       "      <td>tt0099797</td>\n",
       "      <td>movie</td>\n",
       "      <td>1990</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Crime Drama Romance</td>\n",
       "      <td>6.4</td>\n",
       "      <td>10212</td>\n",
       "      <td>The Hot Spot (1990)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5041</th>\n",
       "      <td>tt0244911</td>\n",
       "      <td>series</td>\n",
       "      <td>1987</td>\n",
       "      <td>2006</td>\n",
       "      <td>Drama</td>\n",
       "      <td>9.5</td>\n",
       "      <td>3139</td>\n",
       "      <td>Malgudi Days (1987 - 2006)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13378</th>\n",
       "      <td>tt7450814</td>\n",
       "      <td>series</td>\n",
       "      <td>2017</td>\n",
       "      <td>2018</td>\n",
       "      <td>Crime Drama Thriller</td>\n",
       "      <td>9.6</td>\n",
       "      <td>16061</td>\n",
       "      <td>Koombiyo (2017 - 2018)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8089</th>\n",
       "      <td>tt10680606</td>\n",
       "      <td>series</td>\n",
       "      <td>2019</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Action Adventure Comedy</td>\n",
       "      <td>9.7</td>\n",
       "      <td>3574</td>\n",
       "      <td>Minecraft Epic (2019 -)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13893</th>\n",
       "      <td>tt9471404</td>\n",
       "      <td>series</td>\n",
       "      <td>2017</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Drama History</td>\n",
       "      <td>9.8</td>\n",
       "      <td>7737</td>\n",
       "      <td>The Chosen (2017 -)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13723</th>\n",
       "      <td>tt8560994</td>\n",
       "      <td>series</td>\n",
       "      <td>2015</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>9.9</td>\n",
       "      <td>2619</td>\n",
       "      <td>Friday Five Sharp (2015 -)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13945 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           tconst titleType  startYear endYear                   genres  \\\n",
       "0       tt0035423     movie       2001      \\N   Comedy Fantasy Romance   \n",
       "8711    tt1314652     movie       2010      \\N           Drama Thriller   \n",
       "2809    tt0099819     movie       1990      \\N             Comedy Crime   \n",
       "2806    tt0099800     movie       1990      \\N     Comedy Music Romance   \n",
       "2805    tt0099797     movie       1990      \\N      Crime Drama Romance   \n",
       "...           ...       ...        ...     ...                      ...   \n",
       "5041    tt0244911    series       1987    2006                    Drama   \n",
       "13378   tt7450814    series       2017    2018     Crime Drama Thriller   \n",
       "8089   tt10680606    series       2019      \\N  Action Adventure Comedy   \n",
       "13893   tt9471404    series       2017      \\N            Drama History   \n",
       "13723   tt8560994    series       2015      \\N                   Comedy   \n",
       "\n",
       "       averageRating  numVotes                       title  \n",
       "0                6.4     77701       Kate & Leopold (2001)  \n",
       "8711             6.4      9126        The Housemaid (2010)  \n",
       "2809             6.4     13124  I Love You to Death (1990)  \n",
       "2806             6.4     12792          House Party (1990)  \n",
       "2805             6.4     10212         The Hot Spot (1990)  \n",
       "...              ...       ...                         ...  \n",
       "5041             9.5      3139  Malgudi Days (1987 - 2006)  \n",
       "13378            9.6     16061      Koombiyo (2017 - 2018)  \n",
       "8089             9.7      3574     Minecraft Epic (2019 -)  \n",
       "13893            9.8      7737         The Chosen (2017 -)  \n",
       "13723            9.9      2619  Friday Five Sharp (2015 -)  \n",
       "\n",
       "[13945 rows x 8 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the data for future uses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the sample data to be used later:\n",
    "# imdb.drop(columns=['primaryTitle','originalTitle'], inplace=True)\n",
    "# imdb.to_csv('datasets/imdb.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Below this are some test and trial stuff not really needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<PandasArray>\n",
       "[                         'Kate & Leopold (2001)',\n",
       "                     'Classe Tous Risques (1960)',\n",
       "                       'The Fugitive Kind (1960)',\n",
       "                 'The League of Gentlemen (1960)',\n",
       "                       'Letter Never Sent (1960)',\n",
       "                     'Eyes Without a Face (1960)',\n",
       "                              'Breathless (1960)',\n",
       "           'The Andy Griffith Show (1960 - 1968)',\n",
       "              'The Bugs Bunny Show (1960 - 1975)',\n",
       "                  'The Flintstones (1960 - 1966)',\n",
       " ...\n",
       "                              'Isi & Ossi (2020)',\n",
       "                       'Brave New World (2020 -)',\n",
       "                       'Bandish Bandits (2020 -)',\n",
       "                       'Unorthodox (2020 - 2020)',\n",
       "                                'Just 6.5 (2019)',\n",
       "                             'Hollywood (2020 -)',\n",
       "                                  'Malang (2020)',\n",
       " 'The Disappearance of Madeleine McCann (2019 -)',\n",
       "                'Motherland: Fort Salem (2020 -)',\n",
       "                                  'Kaithi (2019)']\n",
       "Length: 13945, dtype: object"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = list(imdb['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "import difflib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "429 ms ± 27.5 ms per loop (mean ± std. dev. of 7 runs, 3 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit -n3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rapidfuzz import process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "from difflib import SequenceMatcher\n",
    "from heapq import nlargest as _nlargest\n",
    "def get_best_matches(word, possibilities, n=3, cutoff=0.6):\n",
    "    \"\"\"Use SequenceMatcher to return a list of the indexes of the best \n",
    "    \"good enough\" matches. word is a sequence for which close matches \n",
    "    are desired (typically a string).\n",
    "    possibilities is a list of sequences against which to match word\n",
    "    (typically a list of strings).\n",
    "    Optional arg n (default 3) is the maximum number of close matches to\n",
    "    return.  n must be > 0.\n",
    "    Optional arg cutoff (default 0.6) is a float in [0, 1].  Possibilities\n",
    "    that don't score at least that similar to word are ignored.\n",
    "    \"\"\"\n",
    "\n",
    "    if not n >  0:\n",
    "        raise ValueError(\"n must be > 0: %r\" % (n,))\n",
    "    if not 0.0 <= cutoff <= 1.0:\n",
    "        raise ValueError(\"cutoff must be in [0.0, 1.0]: %r\" % (cutoff,))\n",
    "    result = []\n",
    "    s = SequenceMatcher()\n",
    "    s.set_seq2(word)\n",
    "    for idx, x in enumerate(possibilities):\n",
    "        s.set_seq1(x)\n",
    "        if s.real_quick_ratio() >= cutoff and \\\n",
    "           s.quick_ratio() >= cutoff and \\\n",
    "           s.ratio() >= cutoff:\n",
    "            result.append((s.ratio(), idx))\n",
    "\n",
    "    # Move the best scorers to head of list\n",
    "    result = _nlargest(n, result)\n",
    "\n",
    "    # Strip scores for the best n matches\n",
    "    return [x for score, x in result]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Salako (1974)', 42.352941176470594, 5173),\n",
       " ('JFK (1991)', 40.0, 2949),\n",
       " ('Wadjda (2012)', 38.86363636363636, 10430),\n",
       " ('Dalida (2016)', 38.86363636363636, 12337),\n",
       " ('Leak (2000)', 38.57142857142857, 4731)]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process.extractBests('jlksdfjldksjfslkaj;fdlkjsdfkljdsflkjdfsalkjdsflkdsjfa;lskdjsldak', imdb['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87.9 ms ± 3.59 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "get_close_matches_indexes('inception', imdb['title'], cutoff=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Inception (2010)'"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb['title'].iloc[8838]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tconst                        tt0253623\n",
       "titleType                         movie\n",
       "startYear                          1974\n",
       "endYear                              \\N\n",
       "genres           Adventure Comedy Crime\n",
       "averageRating                       7.4\n",
       "numVotes                           5484\n",
       "title                     Salako (1974)\n",
       "Name: 5173, dtype: object"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb.iloc[5173]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}