python3_nmf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_sequence</th>\n",
       "      <th>user_id</th>\n",
       "      <th>challenge_sequence</th>\n",
       "      <th>challenge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4576_1</td>\n",
       "      <td>4576</td>\n",
       "      <td>1</td>\n",
       "      <td>CI23714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4576_2</td>\n",
       "      <td>4576</td>\n",
       "      <td>2</td>\n",
       "      <td>CI23855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4576_3</td>\n",
       "      <td>4576</td>\n",
       "      <td>3</td>\n",
       "      <td>CI24917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4576_4</td>\n",
       "      <td>4576</td>\n",
       "      <td>4</td>\n",
       "      <td>CI23663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4576_5</td>\n",
       "      <td>4576</td>\n",
       "      <td>5</td>\n",
       "      <td>CI23933</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_sequence  user_id  challenge_sequence challenge\n",
       "0        4576_1     4576                   1   CI23714\n",
       "1        4576_2     4576                   2   CI23855\n",
       "2        4576_3     4576                   3   CI24917\n",
       "3        4576_4     4576                   4   CI23663\n",
       "4        4576_5     4576                   5   CI23933"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train = pd.read_csv(\"./data/train.csv\")\n",
    "data_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>challenge_ID</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>challenge_series_ID</th>\n",
       "      <th>total_submissions</th>\n",
       "      <th>publish_date</th>\n",
       "      <th>author_ID</th>\n",
       "      <th>author_gender</th>\n",
       "      <th>author_org_ID</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CI23478</td>\n",
       "      <td>2</td>\n",
       "      <td>SI2445</td>\n",
       "      <td>37.0</td>\n",
       "      <td>06-05-2006</td>\n",
       "      <td>AI563576</td>\n",
       "      <td>M</td>\n",
       "      <td>AOI100001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CI23479</td>\n",
       "      <td>2</td>\n",
       "      <td>SI2435</td>\n",
       "      <td>48.0</td>\n",
       "      <td>17-10-2002</td>\n",
       "      <td>AI563577</td>\n",
       "      <td>M</td>\n",
       "      <td>AOI100002</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CI23480</td>\n",
       "      <td>1</td>\n",
       "      <td>SI2435</td>\n",
       "      <td>15.0</td>\n",
       "      <td>16-10-2002</td>\n",
       "      <td>AI563578</td>\n",
       "      <td>M</td>\n",
       "      <td>AOI100003</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CI23481</td>\n",
       "      <td>1</td>\n",
       "      <td>SI2710</td>\n",
       "      <td>236.0</td>\n",
       "      <td>19-09-2003</td>\n",
       "      <td>AI563579</td>\n",
       "      <td>M</td>\n",
       "      <td>AOI100004</td>\n",
       "      <td>70.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CI23482</td>\n",
       "      <td>2</td>\n",
       "      <td>SI2440</td>\n",
       "      <td>137.0</td>\n",
       "      <td>21-03-2002</td>\n",
       "      <td>AI563580</td>\n",
       "      <td>M</td>\n",
       "      <td>AOI100005</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  challenge_ID  programming_language challenge_series_ID  total_submissions  \\\n",
       "0      CI23478                     2              SI2445               37.0   \n",
       "1      CI23479                     2              SI2435               48.0   \n",
       "2      CI23480                     1              SI2435               15.0   \n",
       "3      CI23481                     1              SI2710              236.0   \n",
       "4      CI23482                     2              SI2440              137.0   \n",
       "\n",
       "  publish_date author_ID author_gender author_org_ID  category_id  \n",
       "0   06-05-2006  AI563576             M     AOI100001          NaN  \n",
       "1   17-10-2002  AI563577             M     AOI100002         32.0  \n",
       "2   16-10-2002  AI563578             M     AOI100003          NaN  \n",
       "3   19-09-2003  AI563579             M     AOI100004         70.0  \n",
       "4   21-03-2002  AI563580             M     AOI100005          NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_challenge = pd.read_csv(\"./data/challenge_data.csv\")\n",
    "data_challenge.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_sequence</th>\n",
       "      <th>user_id</th>\n",
       "      <th>challenge_sequence</th>\n",
       "      <th>challenge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4577_1</td>\n",
       "      <td>4577</td>\n",
       "      <td>1</td>\n",
       "      <td>CI23855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4577_2</td>\n",
       "      <td>4577</td>\n",
       "      <td>2</td>\n",
       "      <td>CI23933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4577_3</td>\n",
       "      <td>4577</td>\n",
       "      <td>3</td>\n",
       "      <td>CI24917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4577_4</td>\n",
       "      <td>4577</td>\n",
       "      <td>4</td>\n",
       "      <td>CI24915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4577_5</td>\n",
       "      <td>4577</td>\n",
       "      <td>5</td>\n",
       "      <td>CI23714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_sequence  user_id  challenge_sequence challenge\n",
       "0        4577_1     4577                   1   CI23855\n",
       "1        4577_2     4577                   2   CI23933\n",
       "2        4577_3     4577                   3   CI24917\n",
       "3        4577_4     4577                   4   CI24915\n",
       "4        4577_5     4577                   5   CI23714"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test = pd.read_csv(\"./data/test.csv\")\n",
    "data_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     903916\n",
       "unique     69532\n",
       "top       102167\n",
       "freq          13\n",
       "Name: user_id, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train['user_id'] = data_train['user_id'].apply(lambda x: str(x))\n",
    "data_train['user_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     397320\n",
       "unique     39732\n",
       "top        31599\n",
       "freq          10\n",
       "Name: user_id, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test['user_id'] = data_test['user_id'].apply(lambda x: str(x))\n",
    "data_test['user_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1301236 entries, 44675 to 347408\n",
      "Data columns (total 4 columns):\n",
      "user_sequence         1301236 non-null object\n",
      "user_id               1301236 non-null object\n",
      "challenge_sequence    1301236 non-null int64\n",
      "challenge             1301236 non-null object\n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 49.6+ MB\n"
     ]
    }
   ],
   "source": [
    "data_all = pd.concat([data_train, data_test], axis=0)\n",
    "data_all = data_all.sort_values('user_id')\n",
    "# data_all = data_all.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "data_all.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     1301236\n",
       "unique     109264\n",
       "top        102167\n",
       "freq           13\n",
       "Name: user_id, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all['user_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_sequence         0\n",
       "user_id               0\n",
       "challenge_sequence    0\n",
       "challenge             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_sequence</th>\n",
       "      <th>user_id</th>\n",
       "      <th>challenge_sequence</th>\n",
       "      <th>challenge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44675</th>\n",
       "      <td>10000_8</td>\n",
       "      <td>10000</td>\n",
       "      <td>6</td>\n",
       "      <td>CI24230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44669</th>\n",
       "      <td>10000_2</td>\n",
       "      <td>10000</td>\n",
       "      <td>12</td>\n",
       "      <td>CI24530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44670</th>\n",
       "      <td>10000_3</td>\n",
       "      <td>10000</td>\n",
       "      <td>11</td>\n",
       "      <td>CI26052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44680</th>\n",
       "      <td>10000_13</td>\n",
       "      <td>10000</td>\n",
       "      <td>1</td>\n",
       "      <td>CI25126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44679</th>\n",
       "      <td>10000_12</td>\n",
       "      <td>10000</td>\n",
       "      <td>2</td>\n",
       "      <td>CI26050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44678</th>\n",
       "      <td>10000_11</td>\n",
       "      <td>10000</td>\n",
       "      <td>3</td>\n",
       "      <td>CI23880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44677</th>\n",
       "      <td>10000_10</td>\n",
       "      <td>10000</td>\n",
       "      <td>4</td>\n",
       "      <td>CI26051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44676</th>\n",
       "      <td>10000_9</td>\n",
       "      <td>10000</td>\n",
       "      <td>5</td>\n",
       "      <td>CI25135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44668</th>\n",
       "      <td>10000_1</td>\n",
       "      <td>10000</td>\n",
       "      <td>13</td>\n",
       "      <td>CI23663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44674</th>\n",
       "      <td>10000_7</td>\n",
       "      <td>10000</td>\n",
       "      <td>7</td>\n",
       "      <td>CI23691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44673</th>\n",
       "      <td>10000_6</td>\n",
       "      <td>10000</td>\n",
       "      <td>8</td>\n",
       "      <td>CI23855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44672</th>\n",
       "      <td>10000_5</td>\n",
       "      <td>10000</td>\n",
       "      <td>9</td>\n",
       "      <td>CI24915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44671</th>\n",
       "      <td>10000_4</td>\n",
       "      <td>10000</td>\n",
       "      <td>10</td>\n",
       "      <td>CI24525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788879</th>\n",
       "      <td>100000_1</td>\n",
       "      <td>100000</td>\n",
       "      <td>13</td>\n",
       "      <td>CI26867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788890</th>\n",
       "      <td>100000_12</td>\n",
       "      <td>100000</td>\n",
       "      <td>2</td>\n",
       "      <td>CI27897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788891</th>\n",
       "      <td>100000_13</td>\n",
       "      <td>100000</td>\n",
       "      <td>1</td>\n",
       "      <td>CI28625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788889</th>\n",
       "      <td>100000_11</td>\n",
       "      <td>100000</td>\n",
       "      <td>3</td>\n",
       "      <td>CI27784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788888</th>\n",
       "      <td>100000_10</td>\n",
       "      <td>100000</td>\n",
       "      <td>4</td>\n",
       "      <td>CI28184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788887</th>\n",
       "      <td>100000_9</td>\n",
       "      <td>100000</td>\n",
       "      <td>5</td>\n",
       "      <td>CI26928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788886</th>\n",
       "      <td>100000_8</td>\n",
       "      <td>100000</td>\n",
       "      <td>6</td>\n",
       "      <td>CI26198</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_sequence user_id  challenge_sequence challenge\n",
       "44675        10000_8   10000                   6   CI24230\n",
       "44669        10000_2   10000                  12   CI24530\n",
       "44670        10000_3   10000                  11   CI26052\n",
       "44680       10000_13   10000                   1   CI25126\n",
       "44679       10000_12   10000                   2   CI26050\n",
       "44678       10000_11   10000                   3   CI23880\n",
       "44677       10000_10   10000                   4   CI26051\n",
       "44676        10000_9   10000                   5   CI25135\n",
       "44668        10000_1   10000                  13   CI23663\n",
       "44674        10000_7   10000                   7   CI23691\n",
       "44673        10000_6   10000                   8   CI23855\n",
       "44672        10000_5   10000                   9   CI24915\n",
       "44671        10000_4   10000                  10   CI24525\n",
       "788879      100000_1  100000                  13   CI26867\n",
       "788890     100000_12  100000                   2   CI27897\n",
       "788891     100000_13  100000                   1   CI28625\n",
       "788889     100000_11  100000                   3   CI27784\n",
       "788888     100000_10  100000                   4   CI28184\n",
       "788887      100000_9  100000                   5   CI26928\n",
       "788886      100000_8  100000                   6   CI26198"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all['challenge_sequence'] = data_all['challenge_sequence'].apply(lambda x: 14 - x)\n",
    "data_all.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1301236 entries, 44675 to 347408\n",
      "Data columns (total 3 columns):\n",
      "user_id               1301236 non-null object\n",
      "challenge             1301236 non-null object\n",
      "challenge_sequence    1301236 non-null int64\n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 39.7+ MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>challenge</th>\n",
       "      <th>challenge_sequence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44675</th>\n",
       "      <td>10000</td>\n",
       "      <td>CI24230</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44669</th>\n",
       "      <td>10000</td>\n",
       "      <td>CI24530</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44670</th>\n",
       "      <td>10000</td>\n",
       "      <td>CI26052</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44680</th>\n",
       "      <td>10000</td>\n",
       "      <td>CI25126</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44679</th>\n",
       "      <td>10000</td>\n",
       "      <td>CI26050</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id challenge  challenge_sequence\n",
       "44675   10000   CI24230                   6\n",
       "44669   10000   CI24530                  12\n",
       "44670   10000   CI26052                  11\n",
       "44680   10000   CI25126                   1\n",
       "44679   10000   CI26050                   2"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all = data_all[['user_id', 'challenge', 'challenge_sequence']]\n",
    "data_all.info()\n",
    "data_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from surprise import NMF\n",
    "from surprise import KNNBasic\n",
    "from surprise import Dataset\n",
    "from surprise import Reader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A reader is still needed but only the rating_scale param is requiered.\n",
    "reader = Reader(rating_scale=(1, 13))\n",
    "\n",
    "# The columns must correspond to user id, item id and ratings (in that order).\n",
    "data = Dataset.load_from_df(data_all.iloc[:], reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.matrix_factorization.NMF at 0x7f4bb52ac7f0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Retrieve the trainset.\n",
    "trainset = data.build_full_trainset()\n",
    "\n",
    "# Build an algorithm, and train it.\n",
    "# algo = KNNBasic()\n",
    "algo = NMF()\n",
    "# algo.fit(trainset)\n",
    "\n",
    "# from surprise import NormalPredictor, evaluate\n",
    "# algo = NormalPredictor()\n",
    "algo.fit(trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39732\n",
      "5502\n"
     ]
    }
   ],
   "source": [
    "# uid = '4577'  # raw user id (as in the ratings file).\n",
    "# iid = 'CI23855'  # raw item id (as in the ratings file).\n",
    "\n",
    "# get a prediction for specific users and items, the rating is supposed to be 13.\n",
    "pred_user_id_list = data_test.user_id.unique()\n",
    "print(len(pred_user_id_list))\n",
    "\n",
    "pred_challenge_id_list = data_all.challenge.unique()\n",
    "print(len(pred_challenge_id_list))\n",
    "\n",
    "# pred_challenge_id_list = data_challenge['challenge_ID'].unique()\n",
    "# print(len(pred_challenge_id_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = pd.DataFrame(columns=['user', 'challenge', 'rating'])\n",
    "\n",
    "for pred_user_id in pred_user_id_list[:]:\n",
    "    for pred_challenge_id in pred_challenge_id_list[:]:\n",
    "        repeat_list = data_test['challenge'][data_test['user_id'] == pred_user_id].tolist()\n",
    "        if pred_challenge_id not in repeat_list:\n",
    "            pred = algo.predict(pred_user_id, pred_challenge_id, r_ui=None, verbose=False)\n",
    "            if float(pred.est) > 7.0:\n",
    "                predictions = predictions.append(pd.Series([pred_user_id, pred_challenge_id, pred.est], \\\n",
    "                                                           index=['user', 'challenge', 'rating']), \\\n",
    "                                                           ignore_index=True)\n",
    "# predictions.iloc[0:9:3, 2] = 3.0\n",
    "# predictions.iloc[:15, 1] = 'CI22222'\n",
    "predictions.info()\n",
    "predictions.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions['rank'] = predictions.groupby('user')['rating'].rank(ascending=False)\n",
    "predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = predictions.sort_values(['rank'])\n",
    "predictions = predictions.groupby('user').head(3)\n",
    "predictions = predictions.sort_values(['user', 'rank']).reset_index(drop=True)\n",
    "\n",
    "predictions.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "current = None\n",
    "seq = 11\n",
    "results = pd.DataFrame(columns=['user_sequence', 'challenge'])\n",
    "for index, row in predictions.iterrows():\n",
    "    if row['user'] != current:\n",
    "        seq = 11\n",
    "    \n",
    "    result_user_seq = row['user'] + '_' + str(seq)\n",
    "    result_challenge = row['challenge']\n",
    "    results = results.append(pd.Series([row['user'] + '_' + str(seq), row['challenge']], index=['user_sequence', 'challenge']), ignore_index=True)\n",
    "    \n",
    "    seq += 1\n",
    "    current = row['user']\n",
    "\n",
    "results.head(9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results.to_csv('./submission/submission_nmf.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}