otto-generate-candidates-carts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "91d16b94",
   "metadata": {
    "papermill": {
     "duration": 0.004489,
     "end_time": "2023-03-18T13:01:07.643220",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.638731",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Generate candidates for carts\n",
    "This notebook uses the click2buy matrix to generate candidates. It produces 4 outputs: candidates for the first cross-validation set (used to train the re-ranking model), candidates for the second cross-validation dataset (planned to also use them for training, but those plans were abandoned, this output is not used), top 75 candidates for the test dataset (used as test dataset for re-ranking model to produce the final results) and top 20 candidates for the test dataset (used to check the results on leaderboard).\n",
    "\n",
    "Until the very end of competition, carts model used 50 candidates, and all the constants for 50 candidates are still there. But the final submission pipeline only uses 75 candidates.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b91b390e",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-18T13:01:07.652272Z",
     "iopub.status.busy": "2023-03-18T13:01:07.651825Z",
     "iopub.status.idle": "2023-03-18T13:01:07.691220Z",
     "shell.execute_reply": "2023-03-18T13:01:07.690005Z"
    },
    "papermill": {
     "duration": 0.047205,
     "end_time": "2023-03-18T13:01:07.694112",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.646907",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "       \n",
    "import gc\n",
    "from datetime import datetime\n",
    "from humanize import naturalsize\n",
    "import itertools\n",
    "from collections import Counter\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5a387ab0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:01:07.703203Z",
     "iopub.status.busy": "2023-03-18T13:01:07.702812Z",
     "iopub.status.idle": "2023-03-18T13:01:07.719627Z",
     "shell.execute_reply": "2023-03-18T13:01:07.718438Z"
    },
    "papermill": {
     "duration": 0.024088,
     "end_time": "2023-03-18T13:01:07.721931",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.697843",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def suggest_buys(df, n_candidates, top_dict, click2buy_dict):\n",
    "    # Set some constants depending on number of candidates.\n",
    "    # n_candidates should be 20, 50 or 75\n",
    "    if n_candidates == 20:\n",
    "        aid_length = 18\n",
    "        max_from_buys = 4\n",
    "    if n_candidates == 50:\n",
    "        aid_length = 30\n",
    "        max_from_buys = 8\n",
    "    if n_candidates == 75:\n",
    "        aid_length = 35\n",
    "        max_from_buys = 10\n",
    "        \n",
    "    # Select all last AIDs and some other info from dataframe.\n",
    "    aids=df.aid.tolist()\n",
    "    unique_aids = list(dict.fromkeys(aids[::-1] ))\n",
    "    day_of_week = max(df.day_of_week.tolist())\n",
    "    df = df.loc[(df['type']==1)|(df['type']==2)]\n",
    "    unique_buys = list(dict.fromkeys( df.aid.tolist()[::-1] ))\n",
    "\n",
    "    # Select last buys and AIDs, suggested from buys.\n",
    "    if len(unique_buys) > 0:\n",
    "\n",
    "        aids2 = list(itertools.chain(*[click2buy_dict[aid][:5] for aid in unique_buys if aid in click2buy_dict]))\n",
    "        result = unique_buys[:aid_length]\n",
    "        top_aids2 = [aid2 for aid2, cnt in Counter(aids2).most_common(n_candidates) if aid2 not in result]       \n",
    "        if n_candidates == 75:\n",
    "            top_from_dict = 45\n",
    "            n_root = 8\n",
    "        elif n_candidates == 50:\n",
    "            top_from_dict = 32\n",
    "            n_root = 6\n",
    "        else:\n",
    "            top_from_dict = 14\n",
    "            n_root = 6\n",
    "    else:\n",
    "        top_aids2 = []\n",
    "        result = []\n",
    "        if n_candidates == 75:\n",
    "            top_from_dict = 65\n",
    "            n_root = 12\n",
    "        elif n_candidates == 50:\n",
    "            top_from_dict = 42\n",
    "            n_root = 10\n",
    "        else:\n",
    "            top_from_dict = 18\n",
    "            n_root = 10\n",
    "    \n",
    "    # Select AIDs, suggested from all the last clicks\n",
    "    root_aids = unique_aids[:n_root]\n",
    "    unique_aids = unique_aids[:aid_length]\n",
    "    aids3 = list(itertools.chain(*[click2buy_dict[aid][:top_from_dict] for aid in root_aids if aid in click2buy_dict]))\n",
    "    top_aids3 = [aid3 for aid3, cnt in Counter(aids3).most_common(n_candidates) if aid3 not in result]\n",
    "    \n",
    "    # Merge all the previously selected AIDs and remove duplicates.\n",
    "    suggested_aids = unique_aids + top_aids2[:max_from_buys] + top_aids3[:n_candidates]\n",
    "    i = 0\n",
    "    while (len(result) < n_candidates) & (i < len(suggested_aids)):\n",
    "        if suggested_aids[i] not in result:\n",
    "            result.append(suggested_aids[i])\n",
    "        i+=1\n",
    "    \n",
    "    # Add items from day top, if free slots.\n",
    "    if (n_candidates-len(result)) > 0:\n",
    "        i = 0\n",
    "        days_top = top_dict[day_of_week]\n",
    "        while len(result) < n_candidates:\n",
    "            if days_top[i] not in result:\n",
    "                result.append(days_top[i])\n",
    "            i+=1\n",
    "    return result   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84b65ef0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:01:07.730647Z",
     "iopub.status.busy": "2023-03-18T13:01:07.730217Z",
     "iopub.status.idle": "2023-03-18T13:01:07.742449Z",
     "shell.execute_reply": "2023-03-18T13:01:07.741336Z"
    },
    "papermill": {
     "duration": 0.019472,
     "end_time": "2023-03-18T13:01:07.745005",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.725533",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prepare all the data for candidate generation.\n",
    "# n_candidates should be 20 or 50 or 75\n",
    "\n",
    "def generate_cart_candidates(click2buy_matrix_path, sessions_path, n_candidates, answers_path=None): \n",
    "    # Load co-visitation matrix and transform it to dictionary (to speed up candidate generation)\n",
    "    click2buy_dict = otto_common.matrix_to_dict(click2buy_matrix_path, n_candidates)\n",
    "    \n",
    "    # Load sessions, add a weekday column and calculate top carts for each day\n",
    "    df = pd.read_parquet(sessions_path)\n",
    "    df = otto_common.add_datetime(df)\n",
    "    df['day_of_week'] = df['time'].dt.dayofweek.astype(np.int8)\n",
    "    del df['time']\n",
    "    gc.collect()\n",
    "    top_dict = otto_common.build_top_dict(df, n_candidates, 1)\n",
    "    \n",
    "    # In case we generate candidates for cross-validation, reduce inputs to those that have positive results as ground truth.\n",
    "    # Alse prepare some data for statistics.\n",
    "    if answers_path:\n",
    "        df, df_answers = otto_common.reduce_df_prepare_answers(df, answers_path, 'carts')\n",
    "        \n",
    "        unique_session_aids = df.groupby([\"session\"]).aid.nunique()\n",
    "        unique_session_aids = pd.DataFrame({'session':unique_session_aids.index,\n",
    "                                            'all_aids':unique_session_aids.values})\n",
    "        unique_session_buys = df.loc[df['type'] > 0].groupby([\"session\"]).aid.nunique()\n",
    "        unique_session_buys = pd.DataFrame({'session':unique_session_buys.index,\n",
    "                                            'buys':unique_session_buys.values})\n",
    "    \n",
    "    # The candidate generation itself.\n",
    "    print('Start suggesting')\n",
    "    pred_df_clicks = df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n",
    "        lambda x: suggest_buys(x, n_candidates, top_dict, click2buy_dict)\n",
    "        )\n",
    "    \n",
    "    del df\n",
    "    gc.collect()\n",
    "    \n",
    "    pred_df_clicks = pd.DataFrame({'session':pred_df_clicks.index, 'cart_predictions':pred_df_clicks.values})\n",
    "    \n",
    "    # If generating candidates for a cross-validation set, use ground truth to calculate some statistics.\n",
    "    # Else just remove columns we do not need anymore.\n",
    "    if answers_path:\n",
    "        pred_df_clicks = otto_common.calculate_stats(pred_df_clicks, df_answers, unique_session_aids,\n",
    "                                         unique_session_buys, 'carts', n_candidates)\n",
    "    else:\n",
    "        pred_df_clicks = pred_df_clicks[['session', 'cart_predictions']]\n",
    "    \n",
    "    return pred_df_clicks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93c358e5",
   "metadata": {
    "papermill": {
     "duration": 0.003222,
     "end_time": "2023-03-18T13:01:07.751818",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.748596",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Generate candidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "121f7dff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:01:07.760785Z",
     "iopub.status.busy": "2023-03-18T13:01:07.760084Z",
     "iopub.status.idle": "2023-03-18T13:01:07.766200Z",
     "shell.execute_reply": "2023-03-18T13:01:07.765041Z"
    },
    "papermill": {
     "duration": 0.013082,
     "end_time": "2023-03-18T13:01:07.768374",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.755292",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# All the paths.\n",
    "# Tried to use all the co-visitation matrixes, but final version only uses \"regular\" matrix.\n",
    "click2buy_matrix_path = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_cv_top_n.parquet'\n",
    "click2buy_full_matrix_path = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_test_top_n.parquet'\n",
    "\n",
    "sessions_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "sessions_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "sessions_path_test = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "\n",
    "answers_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_labels.parquet'\n",
    "answers_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_labels2.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f2ccabd5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:01:07.777269Z",
     "iopub.status.busy": "2023-03-18T13:01:07.776880Z",
     "iopub.status.idle": "2023-03-18T13:01:07.783855Z",
     "shell.execute_reply": "2023-03-18T13:01:07.782659Z"
    },
    "papermill": {
     "duration": 0.015289,
     "end_time": "2023-03-18T13:01:07.787183",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.771894",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3 µs, sys: 1e+03 ns, total: 4 µs\n",
      "Wall time: 7.39 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# This cell was used for tests and checks.\n",
    "\n",
    "#n_candidates = 75\n",
    "\n",
    "#df_result =  generate_cart_candidates(click2buy_matrix_path, sessions_path_cv1, n_candidates, answers_path_cv1)\n",
    "#df_result =  generate_cart_candidates(click2buy_matrix_path, sessions_path_cv2, n_candidates, answers_path_cv2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "db8c8a10",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:01:07.797020Z",
     "iopub.status.busy": "2023-03-18T13:01:07.796637Z",
     "iopub.status.idle": "2023-03-18T13:07:50.686958Z",
     "shell.execute_reply": "2023-03-18T13:07:50.685426Z"
    },
    "papermill": {
     "duration": 402.898138,
     "end_time": "2023-03-18T13:07:50.689816",
     "exception": false,
     "start_time": "2023-03-18T13:01:07.791678",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start suggesting\n",
      "Total carts:  570897\n",
      "Total carts clipped:  570889\n",
      "Total carts guessed:  268978\n",
      "Total carts with buys in history:  309196\n",
      "Carts with buys in history guessed:  125359\n",
      "Total carts with no buys in history:  261693\n",
      "Carts with no buys in history guessed:  143619\n",
      "Total:  47.12%\n",
      "Buys:  40.54%\n",
      "No buys:  54.88%\n",
      "CPU times: user 6min 32s, sys: 9.36 s, total: 6min 42s\n",
      "Wall time: 6min 42s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate candidates for the first cross-validation.\n",
    "\n",
    "n_candidates = 75\n",
    "\n",
    "df_result =  generate_cart_candidates(click2buy_matrix_path, sessions_path_cv1, n_candidates, answers_path_cv1)\n",
    "\n",
    "df_result.to_parquet('candidates_cart_cv1.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d32bb21a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:07:50.699389Z",
     "iopub.status.busy": "2023-03-18T13:07:50.698985Z",
     "iopub.status.idle": "2023-03-18T13:14:32.524702Z",
     "shell.execute_reply": "2023-03-18T13:14:32.523572Z"
    },
    "papermill": {
     "duration": 401.834001,
     "end_time": "2023-03-18T13:14:32.527801",
     "exception": false,
     "start_time": "2023-03-18T13:07:50.693800",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start suggesting\n",
      "Total carts:  570678\n",
      "Total carts clipped:  570610\n",
      "Total carts guessed:  268745\n",
      "Total carts with buys in history:  310000\n",
      "Carts with buys in history guessed:  125727\n",
      "Total carts with no buys in history:  260610\n",
      "Carts with no buys in history guessed:  143018\n",
      "Total:  47.10%\n",
      "Buys:  40.56%\n",
      "No buys:  54.88%\n",
      "CPU times: user 6min 36s, sys: 7.51 s, total: 6min 43s\n",
      "Wall time: 6min 41s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate candidates for the second cross-validation.\n",
    "\n",
    "df_result =  generate_cart_candidates(click2buy_matrix_path, sessions_path_cv2, n_candidates, answers_path_cv2)\n",
    "df_result.to_parquet('candidates_cart_cv2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "60ea4c38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:14:32.538311Z",
     "iopub.status.busy": "2023-03-18T13:14:32.537233Z",
     "iopub.status.idle": "2023-03-18T13:45:23.025300Z",
     "shell.execute_reply": "2023-03-18T13:45:23.023875Z"
    },
    "papermill": {
     "duration": 1850.505721,
     "end_time": "2023-03-18T13:45:23.037558",
     "exception": false,
     "start_time": "2023-03-18T13:14:32.531837",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start suggesting\n",
      "CPU times: user 30min 23s, sys: 33.5 s, total: 30min 57s\n",
      "Wall time: 30min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate candidates for the test dataset.\n",
    "\n",
    "df_result =  generate_cart_candidates(click2buy_full_matrix_path, sessions_path_test, n_candidates)\n",
    "df_result.to_parquet('candidates_cart_test.parquet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cf88fac3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:45:23.049747Z",
     "iopub.status.busy": "2023-03-18T13:45:23.049108Z",
     "iopub.status.idle": "2023-03-18T14:12:15.061529Z",
     "shell.execute_reply": "2023-03-18T14:12:15.060649Z"
    },
    "papermill": {
     "duration": 1612.028436,
     "end_time": "2023-03-18T14:12:15.070938",
     "exception": false,
     "start_time": "2023-03-18T13:45:23.042502",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start suggesting\n",
      "CPU times: user 26min 38s, sys: 26.1 s, total: 27min 5s\n",
      "Wall time: 26min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate 20 candidates for the test dataset (to check on leaderboard).\n",
    "\n",
    "n_candidates = 20\n",
    "df_result =  generate_cart_candidates(click2buy_full_matrix_path, sessions_path_test, n_candidates)\n",
    "df_result.to_parquet('candidates_cart_test20.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 4280.456307,
   "end_time": "2023-03-18T14:12:18.008690",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-18T13:00:57.552383",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}