python3_knn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = pd.read_csv(\"./data/train.csv\")\n",
    "data_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_challenge = pd.read_csv(\"./data/challenge_data.csv\")\n",
    "data_challenge.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_test = pd.read_csv(\"./data/test.csv\")\n",
    "data_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train['user_id'] = data_train['user_id'].apply(lambda x: str(x))\n",
    "data_train['user_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_test['user_id'] = data_test['user_id'].apply(lambda x: str(x))\n",
    "data_test['user_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all = pd.concat([data_train, data_test], axis=0)\n",
    "data_all = data_all.sort_values('user_id')\n",
    "# data_all = data_all.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "data_all.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all['user_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all['challenge_sequence'] = data_all['challenge_sequence'].apply(lambda x: 14 - x)\n",
    "data_all.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all = data_all[['user_id', 'challenge', 'challenge_sequence']]\n",
    "data_all.info()\n",
    "data_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from surprise import SVD\n",
    "from surprise import KNNBasic\n",
    "from surprise import Dataset\n",
    "from surprise import Reader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A reader is still needed but only the rating_scale param is requiered.\n",
    "reader = Reader(rating_scale=(1, 13))\n",
    "\n",
    "# The columns must correspond to user id, item id and ratings (in that order).\n",
    "data = Dataset.load_from_df(data_all.iloc[:], reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Retrieve the trainset.\n",
    "trainset = data.build_full_trainset()\n",
    "\n",
    "# Build an algorithm, and train it.\n",
    "algo = KNNBasic()\n",
    "# algo = SVD()\n",
    "# algo.fit(trainset)\n",
    "\n",
    "# from surprise import NormalPredictor, evaluate\n",
    "# algo = NormalPredictor()\n",
    "algo.fit(trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# uid = '4577'  # raw user id (as in the ratings file).\n",
    "# iid = 'CI23855'  # raw item id (as in the ratings file).\n",
    "\n",
    "# get a prediction for specific users and items, the rating is supposed to be 13.\n",
    "pred_user_id_list = data_test.user_id.unique()\n",
    "print(len(pred_user_id_list))\n",
    "\n",
    "pred_challenge_id_list = data_all.challenge.unique()\n",
    "print(len(pred_challenge_id_list))\n",
    "\n",
    "# pred_challenge_id_list = data_challenge['challenge_ID'].unique()\n",
    "# print(len(pred_challenge_id_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = pd.DataFrame(columns=['user', 'challenge', 'rating'])\n",
    "\n",
    "for pred_user_id in pred_user_id_list[:]:\n",
    "    for pred_challenge_id in pred_challenge_id_list[:]:\n",
    "        repeat_list = data_test['challenge'][data_test['user_id'] == pred_user_id].tolist()\n",
    "        if pred_challenge_id not in repeat_list:\n",
    "            pred = algo.predict(pred_user_id, pred_challenge_id, r_ui=None, verbose=False)\n",
    "            if float(pred.est) > 7.0:\n",
    "                predictions = predictions.append(pd.Series([pred_user_id, pred_challenge_id, pred.est], \\\n",
    "                                                           index=['user', 'challenge', 'rating']), \\\n",
    "                                                           ignore_index=True)\n",
    "# predictions.iloc[0:9:3, 2] = 3.0\n",
    "# predictions.iloc[:15, 1] = 'CI22222'\n",
    "predictions.info()\n",
    "predictions.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions['rank'] = predictions.groupby('user')['rating'].rank(ascending=False)\n",
    "predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = predictions.sort_values(['rank'])\n",
    "predictions = predictions.groupby('user').head(3)\n",
    "predictions = predictions.sort_values(['user', 'rank']).reset_index(drop=True)\n",
    "\n",
    "predictions.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "current = None\n",
    "seq = 11\n",
    "results = pd.DataFrame(columns=['user_sequence', 'challenge'])\n",
    "for index, row in predictions.iterrows():\n",
    "    if row['user'] != current:\n",
    "        seq = 11\n",
    "    \n",
    "    result_user_seq = row['user'] + '_' + str(seq)\n",
    "    result_challenge = row['challenge']\n",
    "    results = results.append(pd.Series([row['user'] + '_' + str(seq), row['challenge']], index=['user_sequence', 'challenge']), ignore_index=True)\n",
    "    \n",
    "    seq += 1\n",
    "    current = row['user']\n",
    "\n",
    "results.head(9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results.to_csv('./submission/submission_knn.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}