otto-model-clicks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d2216d17",
   "metadata": {
    "papermill": {
     "duration": 0.006676,
     "end_time": "2023-03-22T19:04:34.401089",
     "exception": false,
     "start_time": "2023-03-22T19:04:34.394413",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Clicks Model and Prediction\n",
    "A model predicting which aid user is going to click next is trained in this notebook, and then the model is used to make predictions for the test dataset. This same notebook was also used for cross-validating the clicks model. This notebook uses a single input - \"W2vec features for clicks\" notebook, all the inputs are already put together in previous notebooks.\n",
    "Here I tried both catboost and LGBM models, and LGBM showed better results. So, I've used the LGBM model to produce final results. The code for catboost model is kept as comments.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1b74e73d",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-22T19:04:34.414661Z",
     "iopub.status.busy": "2023-03-22T19:04:34.413990Z",
     "iopub.status.idle": "2023-03-22T19:04:37.116853Z",
     "shell.execute_reply": "2023-03-22T19:04:37.115862Z"
    },
    "papermill": {
     "duration": 2.712842,
     "end_time": "2023-03-22T19:04:37.119324",
     "exception": false,
     "start_time": "2023-03-22T19:04:34.406482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "#from catboost import CatBoostRanker, Pool\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from lightgbm.sklearn import LGBMRanker\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "024da959",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:04:37.132126Z",
     "iopub.status.busy": "2023-03-22T19:04:37.131658Z",
     "iopub.status.idle": "2023-03-22T19:04:37.137339Z",
     "shell.execute_reply": "2023-03-22T19:04:37.136387Z"
    },
    "papermill": {
     "duration": 0.014423,
     "end_time": "2023-03-22T19:04:37.139436",
     "exception": false,
     "start_time": "2023-03-22T19:04:37.125013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Remove columns and downcast floats to float16.\n",
    "def prepare_df(df):\n",
    "    del df['day_of_week'], df['first_aid'], df['second_aid']\n",
    "    floats = df.select_dtypes(include=['float64','float32'])\n",
    "    for col in floats:\n",
    "        df[col] = df[col].astype(np.float16)\n",
    "        gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1a40cee1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:04:37.151526Z",
     "iopub.status.busy": "2023-03-22T19:04:37.151245Z",
     "iopub.status.idle": "2023-03-22T19:04:37.157522Z",
     "shell.execute_reply": "2023-03-22T19:04:37.156574Z"
    },
    "papermill": {
     "duration": 0.014547,
     "end_time": "2023-03-22T19:04:37.159488",
     "exception": false,
     "start_time": "2023-03-22T19:04:37.144941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function is used instead of GroupKFold to save some memory during cross-validation.\n",
    "def add_fold_column(df, n_splits):\n",
    "    df_session = pd.DataFrame({'session':df['session'].unique()})\n",
    "    kf = KFold(n_splits=4, random_state=13, shuffle=True)\n",
    "    df_session['fold'] = -1\n",
    "    df_session['fold'] = df_session['fold'].astype(np.int8)\n",
    "    for i, (train_index, test_index) in enumerate(kf.split(df_session)):\n",
    "        df_session.iloc[test_index, 1] = i\n",
    "    df = df.merge(df_session, how='left', on='session')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6023ad0e",
   "metadata": {
    "papermill": {
     "duration": 0.005117,
     "end_time": "2023-03-22T19:04:37.170030",
     "exception": false,
     "start_time": "2023-03-22T19:04:37.164913",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Load and prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67a9248f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:04:37.182376Z",
     "iopub.status.busy": "2023-03-22T19:04:37.181558Z",
     "iopub.status.idle": "2023-03-22T19:05:05.399161Z",
     "shell.execute_reply": "2023-03-22T19:05:05.398121Z"
    },
    "papermill": {
     "duration": 28.226311,
     "end_time": "2023-03-22T19:05:05.401640",
     "exception": false,
     "start_time": "2023-03-22T19:04:37.175329",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_parquet('/kaggle/input/otto-time-viewed/cv1_features_with_w2v.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0a88884",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:05.415870Z",
     "iopub.status.busy": "2023-03-22T19:05:05.415552Z",
     "iopub.status.idle": "2023-03-22T19:05:17.594470Z",
     "shell.execute_reply": "2023-03-22T19:05:17.593360Z"
    },
    "papermill": {
     "duration": 12.190132,
     "end_time": "2023-03-22T19:05:17.597900",
     "exception": false,
     "start_time": "2023-03-22T19:05:05.407768",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = prepare_df(df_train)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "af4ab21c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.625068Z",
     "iopub.status.busy": "2023-03-22T19:05:17.624730Z",
     "iopub.status.idle": "2023-03-22T19:05:17.634192Z",
     "shell.execute_reply": "2023-03-22T19:05:17.632669Z"
    },
    "papermill": {
     "duration": 0.025678,
     "end_time": "2023-03-22T19:05:17.636641",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.610963",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.9 GB\n"
     ]
    }
   ],
   "source": [
    "size = df_train.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fdf802d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.649990Z",
     "iopub.status.busy": "2023-03-22T19:05:17.649207Z",
     "iopub.status.idle": "2023-03-22T19:05:17.654092Z",
     "shell.execute_reply": "2023-03-22T19:05:17.653188Z"
    },
    "papermill": {
     "duration": 0.013383,
     "end_time": "2023-03-22T19:05:17.656193",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.642810",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set a few variables, that are used both in cross-validation and while creating submission.\n",
    "CROSS_VALIDATE = False # Should be changed to False to produce submission.\n",
    "x_cols = list(df_train.columns[3:])\n",
    "frac = 0.7 #fracture of records with target==False to be dropped from train to reduce memory usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5b0ae9fc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.668248Z",
     "iopub.status.busy": "2023-03-22T19:05:17.667985Z",
     "iopub.status.idle": "2023-03-22T19:05:17.673779Z",
     "shell.execute_reply": "2023-03-22T19:05:17.672831Z"
    },
    "papermill": {
     "duration": 0.014005,
     "end_time": "2023-03-22T19:05:17.675701",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.661696",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n# Set the catboost model\\nparameters = {\\n    'iterations': 300,\\n    'loss_function': 'QuerySoftMax',    \\n    'learning_rate': 0.15,\\n    #'custom_metric': 'RecallAt:top=20',\\n    'depth' : 7,\\n    'verbose': 5,\\n    'random_seed': 0, \\n    'task_type' : 'GPU'\\n}\\nmodel = CatBoostRanker(**parameters)\\nprint('model_defined')\\n\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# Set the catboost model\n",
    "parameters = {\n",
    "    'iterations': 300,\n",
    "    'loss_function': 'QuerySoftMax',    \n",
    "    'learning_rate': 0.15,\n",
    "    #'custom_metric': 'RecallAt:top=20',\n",
    "    'depth' : 7,\n",
    "    'verbose': 5,\n",
    "    'random_seed': 0, \n",
    "    'task_type' : 'GPU'\n",
    "}\n",
    "model = CatBoostRanker(**parameters)\n",
    "print('model_defined')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cacc90c7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.688169Z",
     "iopub.status.busy": "2023-03-22T19:05:17.687518Z",
     "iopub.status.idle": "2023-03-22T19:05:17.694180Z",
     "shell.execute_reply": "2023-03-22T19:05:17.692741Z"
    },
    "papermill": {
     "duration": 0.015598,
     "end_time": "2023-03-22T19:05:17.696743",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.681145",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_defined\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    \"objective\" : \"lambdarank\",\n",
    "    \"metric\" : \"ndcg\",\n",
    "    \"boosting_type\" : \"gbdt\",\n",
    "    'min_child_samples' : 200,\n",
    "    \"n_estimators\" : 200,\n",
    "    \"num_leaves\" : 64,\n",
    "    \"importance_type\" : 'gain',\n",
    "    'max_depth' : 7,\n",
    "    'learning_rate' : 0.1,\n",
    "    'random_state' : 21,\n",
    "    'device': 'gpu',\n",
    "    'gpu_platform_id': 0,\n",
    "    'gpu_device_id': 0,    \n",
    "}\n",
    "model = LGBMRanker(**parameters)\n",
    "\n",
    "print('model_defined')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f276c25",
   "metadata": {
    "papermill": {
     "duration": 0.005443,
     "end_time": "2023-03-22T19:05:17.708229",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.702786",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b1a1397b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.720984Z",
     "iopub.status.busy": "2023-03-22T19:05:17.720203Z",
     "iopub.status.idle": "2023-03-22T19:05:17.727269Z",
     "shell.execute_reply": "2023-03-22T19:05:17.726299Z"
    },
    "papermill": {
     "duration": 0.015425,
     "end_time": "2023-03-22T19:05:17.729199",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.713774",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n%%time\\n# A cell that performs cross-validation for the catboost model.\\n\\nCROSS_VALIDATE = True\\nx_cols = list(df_train.columns[3:])\\nfrac = 0.7 #fracture of records with target==False to be dropped from train to reduce memory usage\\ndf_train = df_train.reset_index(drop=True)\\n\\nif CROSS_VALIDATE:\\n    # Define the cross-validation splits and a column for prediction\\n    n_splits = 4\\n    df_train = add_fold_column(df_train, n_splits)\\n    df_importances = pd.DataFrame({'columns':x_cols})\\n    df_train['cv_prediction'] = -1\\n    df_train['cv_prediction'] = df_train['cv_prediction'].astype(np.float16)\\n    \\n    # Fit the model and save the predictions.\\n    for i in range(n_splits):\\n        train_index = df_train.loc[df_train['fold'] != i].index\\n        train_index = otto_common.remove_frac(train_index, df_train, frac)\\n        gc.collect()\\n        print('start_fitting, fold = ' + str(i))\\n        train_pool = Pool(\\n            data=df_train[x_cols].iloc[train_index],\\n            label=df_train.iloc[train_index, 2].astype(np.int8),\\n            group_id=df_train.iloc[train_index, 0]\\n        )\\n        gc.collect()\\n        print('start_fitting')\\n        model.fit(train_pool)\\n        cv_pool = Pool(\\n            data=df_train[x_cols].iloc[test_index],\\n            label=df_train.iloc[test_index, 2].astype(np.int8),\\n            group_id=df_train.iloc[test_index, 0]\\n        )\\n        df_train['cv_prediction'].iloc[test_index] = model.predict(cv_pool)\\n        del train_pool, cv_pool\\n        gc.collect()\\n\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "# A cell that performs cross-validation for the catboost model.\n",
    "\n",
    "CROSS_VALIDATE = True\n",
    "x_cols = list(df_train.columns[3:])\n",
    "frac = 0.7 #fracture of records with target==False to be dropped from train to reduce memory usage\n",
    "df_train = df_train.reset_index(drop=True)\n",
    "\n",
    "if CROSS_VALIDATE:\n",
    "    # Define the cross-validation splits and a column for prediction\n",
    "    n_splits = 4\n",
    "    df_train = add_fold_column(df_train, n_splits)\n",
    "    df_importances = pd.DataFrame({'columns':x_cols})\n",
    "    df_train['cv_prediction'] = -1\n",
    "    df_train['cv_prediction'] = df_train['cv_prediction'].astype(np.float16)\n",
    "    \n",
    "    # Fit the model and save the predictions.\n",
    "    for i in range(n_splits):\n",
    "        train_index = df_train.loc[df_train['fold'] != i].index\n",
    "        train_index = otto_common.remove_frac(train_index, df_train, frac)\n",
    "        gc.collect()\n",
    "        print('start_fitting, fold = ' + str(i))\n",
    "        train_pool = Pool(\n",
    "            data=df_train[x_cols].iloc[train_index],\n",
    "            label=df_train.iloc[train_index, 2].astype(np.int8),\n",
    "            group_id=df_train.iloc[train_index, 0]\n",
    "        )\n",
    "        gc.collect()\n",
    "        print('start_fitting')\n",
    "        model.fit(train_pool)\n",
    "        cv_pool = Pool(\n",
    "            data=df_train[x_cols].iloc[test_index],\n",
    "            label=df_train.iloc[test_index, 2].astype(np.int8),\n",
    "            group_id=df_train.iloc[test_index, 0]\n",
    "        )\n",
    "        df_train['cv_prediction'].iloc[test_index] = model.predict(cv_pool)\n",
    "        del train_pool, cv_pool\n",
    "        gc.collect()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ea33b86a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.742225Z",
     "iopub.status.busy": "2023-03-22T19:05:17.741428Z",
     "iopub.status.idle": "2023-03-22T19:05:17.750242Z",
     "shell.execute_reply": "2023-03-22T19:05:17.749092Z"
    },
    "papermill": {
     "duration": 0.017655,
     "end_time": "2023-03-22T19:05:17.752671",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.735016",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 0 ns, total: 4 µs\n",
      "Wall time: 7.15 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# A cell that performs cross-validation for the LGBM model.\n",
    "\n",
    "if CROSS_VALIDATE:\n",
    "    # Define the cross-validation splits and a column for prediction\n",
    "    n_splits = 4\n",
    "    df_train = add_fold_column(df_train, n_splits)\n",
    "    df_importances = pd.DataFrame({'columns':x_cols})\n",
    "    df_train['cv_prediction'] = -1\n",
    "    df_train['cv_prediction'] = df_train['cv_prediction'].astype(np.float16)\n",
    "    \n",
    "    # Fit the model and save the predictions.\n",
    "    for i in range(n_splits):\n",
    "        train_index = df_train.loc[df_train['fold'] != i].index\n",
    "        train_index = otto_common.remove_frac(train_index, df_train, frac)\n",
    "        gc.collect()\n",
    "        print('start_fitting, fold = ' + str(i))\n",
    "        model.fit(\n",
    "            df_train[x_cols].iloc[train_index],\n",
    "            df_train.iloc[train_index, 2].astype(np.int8),\n",
    "            group=df_train.iloc[train_index].groupby('session').size(),\n",
    "        )\n",
    "        column_name = 'imp_' + str(i)\n",
    "        df_importances[column_name] = model.feature_importances_\n",
    "        test_index = df_train.loc[df_train['fold'] == i].index\n",
    "        df_train['cv_prediction'].iloc[test_index] = model.predict(df_train[x_cols].iloc[test_index])\n",
    "        del train_index, test_index\n",
    "        gc.collect()\n",
    "    df_importances['imp_avg'] = df_importances.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c8a9c751",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.766260Z",
     "iopub.status.busy": "2023-03-22T19:05:17.765434Z",
     "iopub.status.idle": "2023-03-22T19:05:17.769608Z",
     "shell.execute_reply": "2023-03-22T19:05:17.768674Z"
    },
    "papermill": {
     "duration": 0.012601,
     "end_time": "2023-03-22T19:05:17.771540",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.758939",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "84632932",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.785125Z",
     "iopub.status.busy": "2023-03-22T19:05:17.784358Z",
     "iopub.status.idle": "2023-03-22T19:05:17.789377Z",
     "shell.execute_reply": "2023-03-22T19:05:17.788555Z"
    },
    "papermill": {
     "duration": 0.013853,
     "end_time": "2023-03-22T19:05:17.791425",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.777572",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e68af042",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.804033Z",
     "iopub.status.busy": "2023-03-22T19:05:17.803753Z",
     "iopub.status.idle": "2023-03-22T19:05:17.807644Z",
     "shell.execute_reply": "2023-03-22T19:05:17.806752Z"
    },
    "papermill": {
     "duration": 0.012384,
     "end_time": "2023-03-22T19:05:17.809545",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.797161",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Print some stats for cross-validation results\n",
    "if CROSS_VALIDATE:\n",
    "    otto_common.calculate_recall(df_train, 'cv_prediction', 1738122)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef39d0a8",
   "metadata": {
    "papermill": {
     "duration": 0.005469,
     "end_time": "2023-03-22T19:05:17.820778",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.815309",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Fit the test model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "288721a7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:05:17.833941Z",
     "iopub.status.busy": "2023-03-22T19:05:17.833090Z",
     "iopub.status.idle": "2023-03-22T19:06:04.956137Z",
     "shell.execute_reply": "2023-03-22T19:06:04.954992Z"
    },
    "papermill": {
     "duration": 47.13336,
     "end_time": "2023-03-22T19:06:04.959784",
     "exception": false,
     "start_time": "2023-03-22T19:05:17.826424",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Drop a fracture of records with target==False from train to reduce memory usage.\n",
    "if frac > 0:\n",
    "    remove_index = df_train.loc[df_train['target'] == False].sample(frac=frac, random_state=25).index\n",
    "    df_train = df_train.drop(remove_index)\n",
    "    del remove_index\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c088cc9b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:06:04.987381Z",
     "iopub.status.busy": "2023-03-22T19:06:04.987051Z",
     "iopub.status.idle": "2023-03-22T19:06:04.992642Z",
     "shell.execute_reply": "2023-03-22T19:06:04.991606Z"
    },
    "papermill": {
     "duration": 0.02229,
     "end_time": "2023-03-22T19:06:04.996163",
     "exception": false,
     "start_time": "2023-03-22T19:06:04.973873",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# Fit the catboost model\\ntrain_pool = Pool(\\n   data=df_train[x_cols],\\n   label=df_train.iloc[:,2].astype(np.int8),\\n   group_id=df_train.iloc[:,0]\\n)\\nmodel.fit(train_pool)\\n\\ndel df_train\\ngc.collect()\\n'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# Fit the catboost model\n",
    "train_pool = Pool(\n",
    "   data=df_train[x_cols],\n",
    "   label=df_train.iloc[:,2].astype(np.int8),\n",
    "   group_id=df_train.iloc[:,0]\n",
    ")\n",
    "model.fit(train_pool)\n",
    "\n",
    "del df_train\n",
    "gc.collect()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d2d39fb5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:06:05.009248Z",
     "iopub.status.busy": "2023-03-22T19:06:05.008870Z",
     "iopub.status.idle": "2023-03-22T19:13:00.374602Z",
     "shell.execute_reply": "2023-03-22T19:13:00.373674Z"
    },
    "papermill": {
     "duration": 415.380755,
     "end_time": "2023-03-22T19:13:00.382865",
     "exception": false,
     "start_time": "2023-03-22T19:06:05.002110",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the LGBM model.\n",
    "model.fit(\n",
    "    df_train[x_cols],\n",
    "    df_train.iloc[:, 2].astype(np.int8),\n",
    "    group=df_train.groupby('session').size(),\n",
    ")\n",
    "\n",
    "del df_train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5499a40e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:13:00.397414Z",
     "iopub.status.busy": "2023-03-22T19:13:00.396488Z",
     "iopub.status.idle": "2023-03-22T19:13:00.404227Z",
     "shell.execute_reply": "2023-03-22T19:13:00.403371Z"
    },
    "papermill": {
     "duration": 0.017388,
     "end_time": "2023-03-22T19:13:00.406288",
     "exception": false,
     "start_time": "2023-03-22T19:13:00.388900",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n%%time\\n# This code was used to view the feature importances for catboost model.\\n\\nimps = model.get_feature_importance(train_pool)\\ndf_imps = pd.DataFrame({'columns':x_cols, 'importances': imps})\\ndf_imps.to_parquet('importances.parquet')\\n\\ndel train_pool\\ngc.collect()\\n\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "# This code was used to view the feature importances for catboost model.\n",
    "\n",
    "imps = model.get_feature_importance(train_pool)\n",
    "df_imps = pd.DataFrame({'columns':x_cols, 'importances': imps})\n",
    "df_imps.to_parquet('importances.parquet')\n",
    "\n",
    "del train_pool\n",
    "gc.collect()\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc2c902f",
   "metadata": {
    "papermill": {
     "duration": 0.005974,
     "end_time": "2023-03-22T19:13:00.418370",
     "exception": false,
     "start_time": "2023-03-22T19:13:00.412396",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict and export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c1468a59",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:13:00.432244Z",
     "iopub.status.busy": "2023-03-22T19:13:00.431601Z",
     "iopub.status.idle": "2023-03-22T19:32:06.568743Z",
     "shell.execute_reply": "2023-03-22T19:32:06.567790Z"
    },
    "papermill": {
     "duration": 1146.154584,
     "end_time": "2023-03-22T19:32:06.579017",
     "exception": false,
     "start_time": "2023-03-22T19:13:00.424433",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start predicting 0\n",
      "Loading finished 0__0\n",
      "Loading finished 0__1\n",
      "Loading finished 0__2\n",
      "Loading finished 0__3\n",
      "Loading finished 0__4\n",
      "Predictions made 0\n",
      "Start predicting 1\n",
      "Loading finished 1__0\n",
      "Loading finished 1__1\n",
      "Loading finished 1__2\n",
      "Loading finished 1__3\n",
      "Loading finished 1__4\n",
      "Predictions made 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make predictions using LGBM model for both chunks of test data. \n",
    "for i in range(2):\n",
    "    file_path = '/kaggle/input/otto-time-viewed/test_features_with_w2v_part_' + str(i) + '.parquet'\n",
    "    print('Start predicting '+ str(i))\n",
    "    j_max = 5\n",
    "    for j in range(j_max):\n",
    "        df_test = pd.read_parquet(file_path)\n",
    "        df_test = otto_common.divide_df_by_column(df_test, j_max, j, 'session')\n",
    "        df_test = prepare_df(df_test)\n",
    "        print('Loading finished '+ str(i) + '__' + str(j))\n",
    "        df_test['gbdt_prediction'] = model.predict(df_test[x_cols])\n",
    "        df_test = df_test[['session','click_predictions','gbdt_prediction']]\n",
    "        gc.collect()\n",
    "        if (i == 0) & (j == 0):\n",
    "            df_test_all = df_test\n",
    "        else:\n",
    "            df_test_all = pd.concat([df_test_all, df_test])\n",
    "    print('Predictions made '+ str(i))\n",
    "del df_test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "39fa0ca3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:32:06.596036Z",
     "iopub.status.busy": "2023-03-22T19:32:06.594489Z",
     "iopub.status.idle": "2023-03-22T19:32:06.603473Z",
     "shell.execute_reply": "2023-03-22T19:32:06.602637Z"
    },
    "papermill": {
     "duration": 0.018897,
     "end_time": "2023-03-22T19:32:06.605432",
     "exception": false,
     "start_time": "2023-03-22T19:32:06.586535",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n# Make predictions using catboost model for both chunks of test data.\\nfor i in range(2):\\n    file_path = '/kaggle/input/otto-time-viewed/test_features_with_w2v_part_' + str(i) + '.parquet'\\n    print('Start predicting '+ str(i))\\n    j_max = 5\\n    for j in range(j_max):\\n        df_test = pd.read_parquet(file_path)\\n        df_test = otto_common.divide_df_by_column(df_test, j_max, j, 'session')\\n        df_test = prepare_df(df_test)\\n        print('Loading finished '+ str(i) + '__' + str(j))\\n        test_pool = Pool(\\n            data=df_test[x_cols],\\n            group_id=df_test['session']\\n        )\\n        df_test['gbdt_prediction'] = model.predict(test_pool)\\n        df_test = df_test[['session','click_predictions','gbdt_prediction']]\\n        del test_pool\\n        gc.collect()\\n        if (i == 0) & (j == 0):\\n            df_test_all = df_test\\n        else:\\n            df_test_all = pd.concat([df_test_all, df_test])\\n    print('Predictions made '+ str(i))\\ndel df_test, test_pool\\ngc.collect()\\n\""
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# Make predictions using catboost model for both chunks of test data.\n",
    "for i in range(2):\n",
    "    file_path = '/kaggle/input/otto-time-viewed/test_features_with_w2v_part_' + str(i) + '.parquet'\n",
    "    print('Start predicting '+ str(i))\n",
    "    j_max = 5\n",
    "    for j in range(j_max):\n",
    "        df_test = pd.read_parquet(file_path)\n",
    "        df_test = otto_common.divide_df_by_column(df_test, j_max, j, 'session')\n",
    "        df_test = prepare_df(df_test)\n",
    "        print('Loading finished '+ str(i) + '__' + str(j))\n",
    "        test_pool = Pool(\n",
    "            data=df_test[x_cols],\n",
    "            group_id=df_test['session']\n",
    "        )\n",
    "        df_test['gbdt_prediction'] = model.predict(test_pool)\n",
    "        df_test = df_test[['session','click_predictions','gbdt_prediction']]\n",
    "        del test_pool\n",
    "        gc.collect()\n",
    "        if (i == 0) & (j == 0):\n",
    "            df_test_all = df_test\n",
    "        else:\n",
    "            df_test_all = pd.concat([df_test_all, df_test])\n",
    "    print('Predictions made '+ str(i))\n",
    "del df_test, test_pool\n",
    "gc.collect()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9aba56f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:32:06.621937Z",
     "iopub.status.busy": "2023-03-22T19:32:06.620489Z",
     "iopub.status.idle": "2023-03-22T19:32:06.627877Z",
     "shell.execute_reply": "2023-03-22T19:32:06.626760Z"
    },
    "papermill": {
     "duration": 0.017509,
     "end_time": "2023-03-22T19:32:06.629942",
     "exception": false,
     "start_time": "2023-03-22T19:32:06.612433",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# Enumereate the results and select top 20 for each session.\\ndf_test_all = df_test_all.sort_values([\\'session\\',\\'gbdt_prediction\\'],ascending=[True,False])\\ndf_test_all[\\'n\\'] = df_test_all.groupby(\\'session\\').cumcount().astype(np.int8)\\ndf_test_all = df_test_all.loc[df_test_all[\\'n\\'] < 20].drop(\\'n\\',axis=1)\\n\\n# Final formatting.\\ndf_test_all[\\'click_predictions\\'] = df_test_all[\\'click_predictions\\'].apply(str)\\ndf_test_all = (df_test_all.groupby(\\'session\\').agg({\\'click_predictions\\': lambda x: \" \".join(x)}))\\n'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# Enumereate the results and select top 20 for each session.\n",
    "df_test_all = df_test_all.sort_values(['session','gbdt_prediction'],ascending=[True,False])\n",
    "df_test_all['n'] = df_test_all.groupby('session').cumcount().astype(np.int8)\n",
    "df_test_all = df_test_all.loc[df_test_all['n'] < 20].drop('n',axis=1)\n",
    "\n",
    "# Final formatting.\n",
    "df_test_all['click_predictions'] = df_test_all['click_predictions'].apply(str)\n",
    "df_test_all = (df_test_all.groupby('session').agg({'click_predictions': lambda x: \" \".join(x)}))\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a20af36c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:32:06.645417Z",
     "iopub.status.busy": "2023-03-22T19:32:06.645149Z",
     "iopub.status.idle": "2023-03-22T19:34:30.816085Z",
     "shell.execute_reply": "2023-03-22T19:34:30.815052Z"
    },
    "papermill": {
     "duration": 144.181938,
     "end_time": "2023-03-22T19:34:30.819003",
     "exception": false,
     "start_time": "2023-03-22T19:32:06.637065",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Select top 20 candidates and format as required by organizers.\n",
    "df_test_all = otto_common.select_top_20_and_format(df_test_all, 'click_predictions','gbdt_prediction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c17fcade",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T19:34:30.837391Z",
     "iopub.status.busy": "2023-03-22T19:34:30.837058Z",
     "iopub.status.idle": "2023-03-22T19:34:32.759772Z",
     "shell.execute_reply": "2023-03-22T19:34:32.758787Z"
    },
    "papermill": {
     "duration": 1.935092,
     "end_time": "2023-03-22T19:34:32.762331",
     "exception": false,
     "start_time": "2023-03-22T19:34:30.827239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save to file.\n",
    "df_test_all.to_parquet('click_predictions.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1808.025761,
   "end_time": "2023-03-22T19:34:34.608287",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-22T19:04:26.582526",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}