otto-click-candidates-generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fb4dfd3f",
   "metadata": {
    "papermill": {
     "duration": 0.004929,
     "end_time": "2023-03-18T13:02:11.620463",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.615534",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Generate candidates for clicks\n",
    "This notebook uses the click2click regular matrix to generate candidates. It produces 4 outputs: candidates for the first cross-validation set (used to train the re-ranking model), candidates for the second cross-validation dataset (planned to also use them for training, but those plans were abandoned, this output is not used), top 50 candidates for the test dataset (used as test dataset for re-ranking model to produce the final results) and top 20 candidates for the test dataset (used to check the results on leaderboard).\n",
    "\n",
    "I have prepared code that would generate 75 best candidates, but because of memory limitations and lack of time decided not to proceed with switching to 75 candidates for clicks model.\n",
    "\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1ef126ad",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.631252Z",
     "iopub.status.busy": "2023-03-18T13:02:11.630228Z",
     "iopub.status.idle": "2023-03-18T13:02:11.678454Z",
     "shell.execute_reply": "2023-03-18T13:02:11.677249Z"
    },
    "papermill": {
     "duration": 0.057139,
     "end_time": "2023-03-18T13:02:11.681472",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.624333",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "        \n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "import itertools\n",
    "from collections import Counter\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9d095464",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.691614Z",
     "iopub.status.busy": "2023-03-18T13:02:11.691224Z",
     "iopub.status.idle": "2023-03-18T13:02:11.705762Z",
     "shell.execute_reply": "2023-03-18T13:02:11.704577Z"
    },
    "papermill": {
     "duration": 0.023104,
     "end_time": "2023-03-18T13:02:11.708707",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.685603",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generate candidates for clicks.\n",
    "\n",
    "def suggest_clicks(df, n_candidates, click_dict, top_dict):\n",
    "    # Set some constants depending on number of candidates.\n",
    "    # n_candidates should be 20, 50 or 75\n",
    "    if n_candidates == 20:\n",
    "        aid_length = 10\n",
    "        top1_dict_limit = 10\n",
    "    if n_candidates == 50:\n",
    "        aid_length = 20\n",
    "        top1_dict_limit = 25\n",
    "    if n_candidates == 75:\n",
    "        aid_length = 26\n",
    "        top1_dict_limit = 35\n",
    "        \n",
    "    # Make a list of history aids\n",
    "    aids=df.aid.tolist()\n",
    "    unique_aids = list(dict.fromkeys(aids[::-1] ))[:aid_length]\n",
    "\n",
    "    # Select top aids suggested for the exact last aid \n",
    "    if unique_aids[0] in click_dict:\n",
    "        top_aids = click_dict[unique_aids[0]][:top1_dict_limit]\n",
    "    else:\n",
    "        top_aids = []\n",
    "\n",
    "    # Select most common aids suggested for the few last \"root\" aids\n",
    "    root_aids=[]\n",
    "    i = 0\n",
    "    if n_candidates == 75:\n",
    "        n_root_aids = 5\n",
    "    else:\n",
    "        n_root_aids = 4\n",
    "    while (len(root_aids) < n_root_aids) & (i < len(unique_aids)):\n",
    "        aid = unique_aids[i]\n",
    "        if aid in click_dict:\n",
    "            root_aids.append(aid)\n",
    "        i+=1\n",
    "    \n",
    "    top_n = n_candidates\n",
    "    if (len(root_aids) > 2) & (n_candidates == 75):\n",
    "        top_n = 60\n",
    "    \n",
    "    aids2 = list(itertools.chain(*[click_dict[aid][:top_n] for aid in root_aids]))\n",
    "    top_aids2 = [aid2 for aid2, cnt in Counter(aids2).most_common(n_candidates) if aid2 not in unique_aids]\n",
    "    \n",
    "    # Make a common list of aids from history and two previous selections    \n",
    "    result = unique_aids\n",
    "    top_aids = top_aids + top_aids2\n",
    "    for aid in top_aids:\n",
    "        if aid not in result:\n",
    "            result.append(aid)\n",
    "    result = result[:n_candidates]\n",
    "    \n",
    "    # Add aids from day top, if free slots\n",
    "    if (n_candidates-len(result)) > 0:\n",
    "        i = 0\n",
    "        day_of_week = max(df.day_of_week.tolist())\n",
    "        days_top = top_dict[day_of_week]\n",
    "        while len(result) < n_candidates:\n",
    "            if days_top[i] not in result:\n",
    "                result.append(days_top[i])\n",
    "            i+=1\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "58a2cf17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.718754Z",
     "iopub.status.busy": "2023-03-18T13:02:11.718342Z",
     "iopub.status.idle": "2023-03-18T13:02:11.728436Z",
     "shell.execute_reply": "2023-03-18T13:02:11.727080Z"
    },
    "papermill": {
     "duration": 0.0185,
     "end_time": "2023-03-18T13:02:11.731158",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.712658",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Print some statistics after candidate generation.\n",
    "def print_stats(df_check):\n",
    "    non_zero_clicks = len(df_check.loc[df_check['clicks'] != -1])\n",
    "    guessed_clicks = len(df_check.loc[df_check['pred_true'] == 1])\n",
    "    print(f\"Total_sessions_with_clicks {non_zero_clicks}\")\n",
    "    print(f\"Sessions_with_correctly_predicted_clicks {guessed_clicks}\")\n",
    "    \n",
    "    non_zero_clicks_long = len(df_check.loc[(df_check['clicks'] != -1) & (df_check['aids'] >= 10)])\n",
    "    guessed_clicks_long = len(df_check.loc[(df_check['pred_true'] == 1) & (df_check['aids'] >= 10)])\n",
    "    print(f\"Total_long_sessions_with_clicks {non_zero_clicks_long}\")\n",
    "    print(f\"Long_sessions_with_correctly_predicted_clicks {guessed_clicks_long}\")\n",
    "    \n",
    "    non_zero_clicks_short = len(df_check.loc[(df_check['clicks'] != -1) & (df_check['aids'] < 10)])\n",
    "    guessed_clicks_short = len(df_check.loc[(df_check['pred_true'] == 1) & (df_check['aids'] < 10)])\n",
    "    print(f\"Total_short_sessions_with_clicks {non_zero_clicks_short}\")\n",
    "    print(f\"Long_short_sessions_with_correctly_predicted_clicks {guessed_clicks_short}\")\n",
    "    \n",
    "    print(f\"Percent= {100*guessed_clicks/non_zero_clicks:.2f}%\")\n",
    "    print(f\"Percent_long= {100*guessed_clicks_long/non_zero_clicks_long:.2f}%\")\n",
    "    print(f\"Percent_short= {100*guessed_clicks_short/non_zero_clicks_short:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "378f8be1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.743096Z",
     "iopub.status.busy": "2023-03-18T13:02:11.742242Z",
     "iopub.status.idle": "2023-03-18T13:02:11.754192Z",
     "shell.execute_reply": "2023-03-18T13:02:11.753133Z"
    },
    "papermill": {
     "duration": 0.020393,
     "end_time": "2023-03-18T13:02:11.756830",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.736437",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prepare all the data for candidate generation. \n",
    "# If candidates are generated for the cross-validation dataset, this function also prints some statistics for percentage of ground truth aids among candidates.\n",
    "# n_candidates should be 20 or 50 or 75\n",
    "\n",
    "def generate_click_candidates(matrix_path, sessions_path, n_candidates, answers_path=None):\n",
    "    # Load co-visitation matrix and transform it to dictionary (to speed up candidate generation)\n",
    "    click_dict = otto_common.matrix_to_dict(matrix_path, n_candidates)\n",
    "    \n",
    "    # Load inputs and prepare some data for statistics.\n",
    "    df = pd.read_parquet(sessions_path)\n",
    "    if answers_path:\n",
    "        unique_session_aids = df.groupby([\"session\"]).aid.nunique()\n",
    "        unique_session_aids = pd.DataFrame({'session':unique_session_aids.index, 'aids':unique_session_aids.values})\n",
    "    \n",
    "    # Add a weekday column and calculate top clicks for each day.\n",
    "    df_test = otto_common.add_datetime(df)\n",
    "    df['day_of_week'] = df['time'].dt.dayofweek.astype(np.int8)\n",
    "    del df['time']\n",
    "    gc.collect()\n",
    "    top_dict = otto_common.build_top_dict(df, n_candidates, 0)\n",
    "    \n",
    "    # The candidate generation itself.\n",
    "    pred_df_clicks = df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n",
    "        lambda x: suggest_clicks(x, n_candidates, click_dict, top_dict)\n",
    "        )\n",
    "         \n",
    "    del df\n",
    "    gc.collect()\n",
    "        \n",
    "    pred_df_clicks = pd.DataFrame({'session':pred_df_clicks.index, 'click_predictions':pred_df_clicks.values})\n",
    "    \n",
    "    # If generating candidates for a cross-validation set, use ground truth to calculate some statistics.\n",
    "    if answers_path:\n",
    "        df_answers = pd.read_parquet(answers_path)\n",
    "        pred_df_clicks = pd.merge(pred_df_clicks, df_answers, on = 'session')\n",
    "        pred_df_clicks['pred_true'] = pred_df_clicks.apply(lambda x: x.clicks in x.click_predictions, axis=1)\n",
    "        pred_df_clicks = pd.merge(pred_df_clicks, unique_session_aids, on = 'session')\n",
    "        print_stats(pred_df_clicks)\n",
    "        \n",
    "    return pred_df_clicks\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0273b67a",
   "metadata": {
    "papermill": {
     "duration": 0.003434,
     "end_time": "2023-03-18T13:02:11.764390",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.760956",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Generate candidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ccf3c978",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.773897Z",
     "iopub.status.busy": "2023-03-18T13:02:11.773488Z",
     "iopub.status.idle": "2023-03-18T13:02:11.778641Z",
     "shell.execute_reply": "2023-03-18T13:02:11.777537Z"
    },
    "papermill": {
     "duration": 0.012941,
     "end_time": "2023-03-18T13:02:11.781094",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.768153",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# All the paths.\n",
    "# Tried to use all the co-visitation matrixes, but final version only uses \"regular\" matrix.\n",
    "matrix_path_regular = '/kaggle/input/otto-prepare-candidates-clicks/regular_click2click_matrix_cv_top_n.parquet'\n",
    "matrix_path_regular_full = '/kaggle/input/otto-prepare-candidates-clicks/regular_click2click_matrix_test_top_n.parquet'\n",
    "\n",
    "sessions_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "sessions_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "sessions_path_test = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "\n",
    "answers_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_labels.parquet'\n",
    "answers_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_labels2.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c91121bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.790415Z",
     "iopub.status.busy": "2023-03-18T13:02:11.790026Z",
     "iopub.status.idle": "2023-03-18T13:02:11.796830Z",
     "shell.execute_reply": "2023-03-18T13:02:11.795822Z"
    },
    "papermill": {
     "duration": 0.015826,
     "end_time": "2023-03-18T13:02:11.800741",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.784915",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 0 ns, total: 4 µs\n",
      "Wall time: 7.87 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# This cell was used for tests and checks.\n",
    "\n",
    "#n_candidates = 20\n",
    "\n",
    "#df_result =  generate_click_candidates(matrix_path_regular, sessions_path_cv1, n_candidates, answers_path_cv1)\n",
    "#df_result =  generate_click_candidates(matrix_path_regular, sessions_path_cv2, n_candidates, answers_path_cv2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "868d4967",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:02:11.812015Z",
     "iopub.status.busy": "2023-03-18T13:02:11.811242Z",
     "iopub.status.idle": "2023-03-18T13:10:15.333826Z",
     "shell.execute_reply": "2023-03-18T13:10:15.332958Z"
    },
    "papermill": {
     "duration": 483.530818,
     "end_time": "2023-03-18T13:10:15.336746",
     "exception": false,
     "start_time": "2023-03-18T13:02:11.805928",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total_sessions_with_clicks 1738122\n",
      "Sessions_with_correctly_predicted_clicks 1050415\n",
      "Total_long_sessions_with_clicks 94739\n",
      "Long_sessions_with_correctly_predicted_clicks 50495\n",
      "Total_short_sessions_with_clicks 1643383\n",
      "Long_short_sessions_with_correctly_predicted_clicks 999920\n",
      "Percent= 60.43%\n",
      "Percent_long= 53.30%\n",
      "Percent_short= 60.85%\n"
     ]
    }
   ],
   "source": [
    "# At some point, I've planned to start using 75 candidates for the clicks model, but at the end of the competition decided to try some other improvements instead.\n",
    "# So, carts and orders models use 75 candidates, while clicks is the only model to use 50 candidates.\n",
    "\n",
    "# Generate candidates for first cross-validation.\n",
    "n_candidates = 50\n",
    "\n",
    "df_result =  generate_click_candidates(matrix_path_regular, sessions_path_cv1, n_candidates, answers_path_cv1)\n",
    "df_result.to_parquet('candidates_click.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89b906de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:10:15.348292Z",
     "iopub.status.busy": "2023-03-18T13:10:15.347404Z",
     "iopub.status.idle": "2023-03-18T13:18:22.099558Z",
     "shell.execute_reply": "2023-03-18T13:18:22.098056Z"
    },
    "papermill": {
     "duration": 486.763525,
     "end_time": "2023-03-18T13:18:22.104356",
     "exception": false,
     "start_time": "2023-03-18T13:10:15.340831",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total_sessions_with_clicks 1738205\n",
      "Sessions_with_correctly_predicted_clicks 1050656\n",
      "Total_long_sessions_with_clicks 95434\n",
      "Long_sessions_with_correctly_predicted_clicks 50637\n",
      "Total_short_sessions_with_clicks 1642771\n",
      "Long_short_sessions_with_correctly_predicted_clicks 1000019\n",
      "Percent= 60.44%\n",
      "Percent_long= 53.06%\n",
      "Percent_short= 60.87%\n"
     ]
    }
   ],
   "source": [
    "# I've planned to use both first and second cross-validations and then take the average prediction. But only did that for orders model.\n",
    "# Other models have all candidates prepared for both cross-validation datasets, but features are engineered only for the first cross-validation dataset.\n",
    "\n",
    "# Generate candidates for second cross-validation.\n",
    "df_result =  generate_click_candidates(matrix_path_regular, sessions_path_cv2, n_candidates, answers_path_cv2)\n",
    "df_result.to_parquet('candidates_click2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4970a7bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:18:22.121425Z",
     "iopub.status.busy": "2023-03-18T13:18:22.121004Z",
     "iopub.status.idle": "2023-03-18T13:25:32.061820Z",
     "shell.execute_reply": "2023-03-18T13:25:32.060480Z"
    },
    "papermill": {
     "duration": 429.953804,
     "end_time": "2023-03-18T13:25:32.068084",
     "exception": false,
     "start_time": "2023-03-18T13:18:22.114280",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generate 50 candidates for the test dataset.\n",
    "\n",
    "df_result =  generate_click_candidates(matrix_path_regular_full, sessions_path_test, n_candidates)\n",
    "df_result.to_parquet('candidates_test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8bafe3f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:25:32.083092Z",
     "iopub.status.busy": "2023-03-18T13:25:32.082274Z",
     "iopub.status.idle": "2023-03-18T13:29:56.969181Z",
     "shell.execute_reply": "2023-03-18T13:29:56.967752Z"
    },
    "papermill": {
     "duration": 264.89675,
     "end_time": "2023-03-18T13:29:56.972595",
     "exception": false,
     "start_time": "2023-03-18T13:25:32.075845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generate 20 candidates for the test dataset (to check on leaderboard).\n",
    "n_candidates = 20\n",
    "\n",
    "df_result =  generate_click_candidates(matrix_path_regular_full, sessions_path_test, n_candidates)\n",
    "df_result.to_parquet('candidates_test_20.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1679.64999,
   "end_time": "2023-03-18T13:29:59.806293",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-18T13:02:00.156303",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}