otto-feature-engineering-carts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bb7609b3",
   "metadata": {
    "papermill": {
     "duration": 0.006694,
     "end_time": "2023-03-24T15:28:27.630880",
     "exception": false,
     "start_time": "2023-03-24T15:28:27.624186",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature engineering for carts model\n",
    "This notebook builds most features for the carts model (except for w2vec features that are calculated in a separate notebook).\n",
    "It takes as input already generated candidates, co-visitation matrixes and some pre-calculations made aside in create_counts_for_buys notebook. Most of the features are built using functions common for all the three models and defined in otto_common_fe notebook, but a few features unique to carts model are built using functions defined in this notebook.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32c64acb",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:27.643882Z",
     "iopub.status.busy": "2023-03-24T15:28:27.643348Z",
     "iopub.status.idle": "2023-03-24T15:28:43.939984Z",
     "shell.execute_reply": "2023-03-24T15:28:43.938418Z"
    },
    "papermill": {
     "duration": 16.306617,
     "end_time": "2023-03-24T15:28:43.943117",
     "exception": false,
     "start_time": "2023-03-24T15:28:27.636500",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting polars\r\n",
      "  Downloading polars-0.16.15-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.6 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.6/16.6 MB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.1.1)\r\n",
      "Installing collected packages: polars\r\n",
      "Successfully installed polars-0.16.15\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "        \n",
    "import gc\n",
    "from datetime import datetime\n",
    "from humanize import naturalsize\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common, otto_common_fe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4ee5ca34",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:43.959953Z",
     "iopub.status.busy": "2023-03-24T15:28:43.959471Z",
     "iopub.status.idle": "2023-03-24T15:28:55.426119Z",
     "shell.execute_reply": "2023-03-24T15:28:55.424525Z"
    },
    "papermill": {
     "duration": 11.478586,
     "end_time": "2023-03-24T15:28:55.429198",
     "exception": false,
     "start_time": "2023-03-24T15:28:43.950612",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: polars in /opt/conda/lib/python3.7/site-packages (0.16.15)\r\n",
      "Requirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.1.1)\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Polars library is used to generate features from co-visitation matrix.\n",
    "!pip install polars\n",
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e5f8d18a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.446410Z",
     "iopub.status.busy": "2023-03-24T15:28:55.445950Z",
     "iopub.status.idle": "2023-03-24T15:28:55.454883Z",
     "shell.execute_reply": "2023-03-24T15:28:55.453498Z"
    },
    "papermill": {
     "duration": 0.020622,
     "end_time": "2023-03-24T15:28:55.457315",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.436693",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function was used for data exploration only.\n",
    "# It selects ground truth aids that were missed during candidate generation.\n",
    "def cand_read_and_reduce_test_not_cands(cand_path, is_cv):\n",
    "    df = pd.read_parquet(cand_path)\n",
    "    df = df.loc[df['pred_true'] == 0]\n",
    "    df = df[['session', 'carts']]\n",
    "    df = df.explode('carts').reset_index(drop=True)\n",
    "    df['session'] = df['session'].astype(np.int32)\n",
    "    df['cart_predictions'] = df['carts'].astype(np.int32)\n",
    "    del df['carts']\n",
    "    gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "10a69ba0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.474316Z",
     "iopub.status.busy": "2023-03-24T15:28:55.473902Z",
     "iopub.status.idle": "2023-03-24T15:28:55.480350Z",
     "shell.execute_reply": "2023-03-24T15:28:55.479467Z"
    },
    "papermill": {
     "duration": 0.017682,
     "end_time": "2023-03-24T15:28:55.482539",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.464857",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Number of carts per aid in history (full sessions).\n",
    "def add_total_carts(train_path, df_candidate):\n",
    "    total_counts = pd.read_parquet(train_path)\n",
    "    total_counts = total_counts.loc[total_counts['type'] == 1]\n",
    "    total_counts = total_counts.groupby('aid')['session'].nunique()\n",
    "    gc.collect()\n",
    "    total_counts.name = \"aid_counts_carts\"\n",
    "    df_candidate = pd.merge(df_candidate, total_counts, how='left',\n",
    "                            left_on='cart_predictions', right_index=True)\n",
    "    df_candidate['aid_counts_carts'] = df_candidate['aid_counts_carts'].fillna(0).astype(np.int16)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "15de38e9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.499330Z",
     "iopub.status.busy": "2023-03-24T15:28:55.498890Z",
     "iopub.status.idle": "2023-03-24T15:28:55.504148Z",
     "shell.execute_reply": "2023-03-24T15:28:55.503246Z"
    },
    "papermill": {
     "duration": 0.016366,
     "end_time": "2023-03-24T15:28:55.506304",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.489938",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Merges into features simple conversion rate calculated in create_counts_for_click notebook. It counts how often aid was either\n",
    "# carted or ordered given it was clicked.\n",
    "def add_conversions(conversions_path, df_candidate):\n",
    "    df_conversions = pd.read_parquet(conversions_path)\n",
    "    df_candidate = pd.merge(df_candidate, df_conversions, how='left',\n",
    "                            left_on='cart_predictions', right_on='aid')\n",
    "    df_candidate = df_candidate.drop('aid', axis=1)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9a7ab201",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.523745Z",
     "iopub.status.busy": "2023-03-24T15:28:55.523310Z",
     "iopub.status.idle": "2023-03-24T15:28:55.531070Z",
     "shell.execute_reply": "2023-03-24T15:28:55.529872Z"
    },
    "papermill": {
     "duration": 0.019912,
     "end_time": "2023-03-24T15:28:55.533711",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.513799",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Adds a feature showing how often aid is carted in case if it was clicked, carted before or ordered.\n",
    "def join_the_2cart_conv(all_2cart_conv_path, df_candidate):\n",
    "    df_conv = pd.read_parquet(all_2cart_conv_path)\n",
    "    df_candidate = pd.merge(df_candidate, df_conv, how='left', left_on='cart_predictions', right_on='aid')\n",
    "    df_candidate['total_2cart_conv'] = 0\n",
    "    df_candidate.loc[df_candidate['type_last'] == 0, 'total_2cart_conv'] = df_candidate['click2cart_conv']\n",
    "    df_candidate.loc[df_candidate['type_last'] == 1, 'total_2cart_conv'] = df_candidate['cart2cart_conv']\n",
    "    df_candidate.loc[df_candidate['type_last'] == 2, 'total_2cart_conv'] = df_candidate['order2cart_conv']\n",
    "    df_candidate['total_2cart_conv'] = df_candidate['total_2cart_conv'].astype(np.float32)\n",
    "    del df_candidate['aid'], df_candidate['order2cart_conv'], df_candidate['cart2cart_conv']\n",
    "    del df_candidate['click2cart_conv']\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0af3fa2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.552104Z",
     "iopub.status.busy": "2023-03-24T15:28:55.551660Z",
     "iopub.status.idle": "2023-03-24T15:28:55.563003Z",
     "shell.execute_reply": "2023-03-24T15:28:55.561650Z"
    },
    "papermill": {
     "duration": 0.024576,
     "end_time": "2023-03-24T15:28:55.565631",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.541055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using buy2buy co-visitation matrix.\n",
    "# Uses all the orders in history and only recent carts. For sessions with no orders and no recent carts, the feature is always 0.\n",
    "def add_buy2buy_weight(input_path, buy2buy_matrice, df_candidate):\n",
    "    n_max = 40\n",
    "    col_name = 'wgt_buy2buy'\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test['time_delta'] = df_test.groupby('session')['ts'].transform(np.max).astype(np.int32)\n",
    "    df_test['time_delta'] = df_test['time_delta'] - df_test['ts']\n",
    "    df_test = df_test.loc[((df_test['time_delta'] < 24 * 60 * 60) & (df_test['type'] == 1)) | (df_test['type'] == 2)]\n",
    "    gc.collect()\n",
    "    df_test = df_test.sort_values(['session','ts'],ascending=[True,False])\n",
    "    df_test['n'] = df_test.groupby('session').cumcount().astype(np.int16)\n",
    "    df_test = df_test.loc[df_test['n'] < n_max]\n",
    "    df_test = df_test.drop('time_delta', axis=1)\n",
    "    df_test = df_test[['session', 'aid', 'n']]\n",
    "    gc.collect()\n",
    "    \n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)    \n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, buy2buy_matrice, df_candidate, col_name, n_max,\n",
    "                                                          'cart_predictions', normalize=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "70dadde3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.582635Z",
     "iopub.status.busy": "2023-03-24T15:28:55.582196Z",
     "iopub.status.idle": "2023-03-24T15:28:55.589779Z",
     "shell.execute_reply": "2023-03-24T15:28:55.588222Z"
    },
    "papermill": {
     "duration": 0.019333,
     "end_time": "2023-03-24T15:28:55.592444",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.573111",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using click2buy short co-visitation matrix. Short refers to time between click and buy events, only\n",
    "# cases of relatively short period of time are counted.\n",
    "# Uses no more than 10 events from the last hour of the session.\n",
    "def add_click2buy_short_weight(input_path, c2buy_short_matrice, df_candidate):\n",
    "    n_max = 10\n",
    "    col_name = 'wgt_c2buy_short'\n",
    "    time_frame = 60 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, c2buy_short_matrice, df_candidate, col_name, n_max,\n",
    "                                                         'cart_predictions')    \n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "affa0b46",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.609242Z",
     "iopub.status.busy": "2023-03-24T15:28:55.608764Z",
     "iopub.status.idle": "2023-03-24T15:28:55.616310Z",
     "shell.execute_reply": "2023-03-24T15:28:55.615046Z"
    },
    "papermill": {
     "duration": 0.018943,
     "end_time": "2023-03-24T15:28:55.618710",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.599767",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using click2buy \"long\" co-visitation matrix. \n",
    "# Uses no more than 20 events from the last 3 hours of the session.\n",
    "def add_click2buy_weight(input_path, c2buy_matrice, df_candidate):\n",
    "    n_max = 20\n",
    "    col_name = 'wgt_c2buy_full'\n",
    "    time_frame = 3 * 60 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, c2buy_matrice, df_candidate, col_name, n_max,\n",
    "                                                          'cart_predictions', normalize=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "54540876",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.635808Z",
     "iopub.status.busy": "2023-03-24T15:28:55.635391Z",
     "iopub.status.idle": "2023-03-24T15:28:55.642653Z",
     "shell.execute_reply": "2023-03-24T15:28:55.641348Z"
    },
    "papermill": {
     "duration": 0.019065,
     "end_time": "2023-03-24T15:28:55.645137",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.626072",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using click2buy \"long\" co-visitation matrix. \n",
    "# Uses just 6 events from the last 3 hours of the session.\n",
    "def add_click2buy_weight_6(input_path, c2buy_matrice, df_candidate):\n",
    "    n_max = 6\n",
    "    col_name = 'wgt_c2buy_6_from_full'\n",
    "    time_frame = 3 * 60 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, c2buy_matrice, df_candidate, col_name, n_max,\n",
    "                                                         'cart_predictions')    \n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94f00b11",
   "metadata": {
    "papermill": {
     "duration": 0.006878,
     "end_time": "2023-03-24T15:28:55.659401",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.652523",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature engineering for cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba70dc0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.675751Z",
     "iopub.status.busy": "2023-03-24T15:28:55.675355Z",
     "iopub.status.idle": "2023-03-24T15:28:55.682909Z",
     "shell.execute_reply": "2023-03-24T15:28:55.681683Z"
    },
    "papermill": {
     "duration": 0.018833,
     "end_time": "2023-03-24T15:28:55.685431",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.666598",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for the first cross-validation dataset.\n",
    "candidates_cv1 = '/kaggle/input/otto-validate-candidates-carts/candidates_cart_cv1.parquet'\n",
    "trunked_sessions = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "buy2buy_matrice_cv = '/kaggle/input/otto-prepare-candidates-buys/matrix_buy2buy_cv.parquet'\n",
    "click2buy_short_matrice_cv = '/kaggle/input/otto-click2buy-short/matrix_click2buy_cv_short.parquet'\n",
    "click2buy_matrice_cv = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_cv.parquet'\n",
    "\n",
    "conversions_cv = '/kaggle/input/create-counts-buys/conversions_cv.parquet'\n",
    "click_before_buy_cv = '/kaggle/input/create-counts-buys/clicks_before_buy_cv.parquet'\n",
    "median_time_viewed_cv = '/kaggle/input/create-counts-for-clicks/time_viewed_agg_cv.parquet'\n",
    "history_path_cv = '/kaggle/input/create-counts-buys/history_change_cv1.parquet'\n",
    "all_2cart_conv_cv = '/kaggle/input/create-counts-buys/2cart_conversions_cv.parquet'\n",
    "full_sessions = '/kaggle/input/otto-prepare-cv/cv_train.parquet'\n",
    "\n",
    "counts_before_path = '/kaggle/input/create-counts-buys/daily_counts_carts_train.parquet'\n",
    "counts_during_path = '/kaggle/input/create-counts-buys/daily_counts_carts_cv1.parquet'\n",
    "counts_before_path_click = '/kaggle/input/create-counts-for-clicks/daily_counts_train.parquet'\n",
    "counts_during_path_click = '/kaggle/input/create-counts-for-clicks/daily_counts_cv1.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cd899b18",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:28:55.701898Z",
     "iopub.status.busy": "2023-03-24T15:28:55.701498Z",
     "iopub.status.idle": "2023-03-24T16:01:50.536796Z",
     "shell.execute_reply": "2023-03-24T16:01:50.534963Z"
    },
    "papermill": {
     "duration": 1974.847442,
     "end_time": "2023-03-24T16:01:50.540066",
     "exception": false,
     "start_time": "2023-03-24T15:28:55.692624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "CPU times: user 59min 42s, sys: 14min 52s, total: 1h 14min 34s\n",
      "Wall time: 32min 54s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Imports candidates and builds features from co-visitation matrixes.\n",
    "\n",
    "df_cand = pd.read_parquet(candidates_cv1)\n",
    "df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'cart', True)\n",
    "\n",
    "df_cand = add_buy2buy_weight(trunked_sessions, buy2buy_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight_6(trunked_sessions, click2buy_matrice_cv, df_cand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0dcd3fea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:01:50.567317Z",
     "iopub.status.busy": "2023-03-24T16:01:50.566613Z",
     "iopub.status.idle": "2023-03-24T16:08:09.395550Z",
     "shell.execute_reply": "2023-03-24T16:08:09.394267Z"
    },
    "papermill": {
     "duration": 378.858152,
     "end_time": "2023-03-24T16:08:09.410560",
     "exception": false,
     "start_time": "2023-03-24T16:01:50.552408",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 9s, sys: 1min 11s, total: 6min 21s\n",
      "Wall time: 6min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Builds all the other features. Takes significantly less time than building features from co-visitation matrixes.\n",
    "\n",
    "#df_cand = pd.read_parquet(candidates_cv1)\n",
    "#df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'cart', True)\n",
    "\n",
    "df_cand = otto_common_fe.add_history_aid_features(trunked_sessions, df_cand, 'cart_predictions')\n",
    "df_cand = otto_common_fe.add_history_agg_features(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_time_viewed(trunked_sessions, df_cand, 'cart_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages(counts_before_path_click, counts_during_path_click, df_cand, 'cart_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path_click, df_cand, 'cart_predictions')\n",
    "df_cand = otto_common_fe.add_type_last(trunked_sessions, df_cand, 'cart_predictions')\n",
    "df_cand = add_total_carts(full_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_weekly_averages(counts_before_path, counts_during_path, df_cand, 'cart_predictions')\n",
    "df_cand = add_conversions(conversions_cv, df_cand)\n",
    "df_cand = otto_common_fe.add_clicks_before_buy(click_before_buy_cv, df_cand, 'cart_predictions')\n",
    "df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_cv, df_cand, 'cart_predictions')\n",
    "df_cand = otto_common_fe.add_session_time(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.count_events_3hours(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_history_similarity(history_path_cv, df_cand)\n",
    "df_cand = join_the_2cart_conv(all_2cart_conv_cv, df_cand)\n",
    "df_cand = otto_common_fe.add_total_buys_in_session(trunked_sessions, df_cand)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "01f94107",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:08:09.437246Z",
     "iopub.status.busy": "2023-03-24T16:08:09.436553Z",
     "iopub.status.idle": "2023-03-24T16:08:19.471866Z",
     "shell.execute_reply": "2023-03-24T16:08:19.470279Z"
    },
    "papermill": {
     "duration": 10.052853,
     "end_time": "2023-03-24T16:08:19.475523",
     "exception": false,
     "start_time": "2023-03-24T16:08:09.422670",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0 GB\n"
     ]
    }
   ],
   "source": [
    "# Check resulting dataframe size and export to file.\n",
    "size = df_cand.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_cand.to_parquet('cv_features_cart.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f3b253fe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:08:19.502076Z",
     "iopub.status.busy": "2023-03-24T16:08:19.501659Z",
     "iopub.status.idle": "2023-03-24T16:08:19.512250Z",
     "shell.execute_reply": "2023-03-24T16:08:19.511386Z"
    },
    "papermill": {
     "duration": 0.02701,
     "end_time": "2023-03-24T16:08:19.515018",
     "exception": false,
     "start_time": "2023-03-24T16:08:19.488008",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n%%time\\n# Was used to check all the ground truth aids that were not selected during candidate generation.\\n\\ndf_cand = pd.read_parquet(candidates_cv1)\\ndf_cand = cand_read_and_reduce_test_not_cands(candidates_cv1, True)\\ndf_cand = add_history_aid_features(trunked_sessions, df_cand)\\ndf_cand = add_history_agg_features(trunked_sessions, df_cand)\\ndf_cand = add_time_viewed(trunked_sessions, df_cand)\\ndf_cand = add_total_carts(full_sessions, df_cand)\\ndf_cand = add_weekly_averages(counts_before_path, counts_during_path, df_cand)\\ndf_cand = add_b2cart_weight(trunked_sessions, buy2cart_matrice_cv, df_cand)\\ndf_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_cv, df_cand)\\ndf_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_cv, df_cand)\\n'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "# Was used to check all the ground truth aids that were not selected during candidate generation.\n",
    "\n",
    "df_cand = pd.read_parquet(candidates_cv1)\n",
    "df_cand = cand_read_and_reduce_test_not_cands(candidates_cv1, True)\n",
    "df_cand = add_history_aid_features(trunked_sessions, df_cand)\n",
    "df_cand = add_history_agg_features(trunked_sessions, df_cand)\n",
    "df_cand = add_time_viewed(trunked_sessions, df_cand)\n",
    "df_cand = add_total_carts(full_sessions, df_cand)\n",
    "df_cand = add_weekly_averages(counts_before_path, counts_during_path, df_cand)\n",
    "df_cand = add_b2cart_weight(trunked_sessions, buy2cart_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_cv, df_cand)\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f1ddd2b",
   "metadata": {
    "papermill": {
     "duration": 0.01213,
     "end_time": "2023-03-24T16:08:19.540002",
     "exception": false,
     "start_time": "2023-03-24T16:08:19.527872",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature engineering for test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9fb5164c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:08:19.567982Z",
     "iopub.status.busy": "2023-03-24T16:08:19.567339Z",
     "iopub.status.idle": "2023-03-24T16:08:19.574277Z",
     "shell.execute_reply": "2023-03-24T16:08:19.573275Z"
    },
    "papermill": {
     "duration": 0.024654,
     "end_time": "2023-03-24T16:08:19.576968",
     "exception": false,
     "start_time": "2023-03-24T16:08:19.552314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for test dataset.\n",
    "candidates_test = '/kaggle/input/otto-validate-candidates-carts/candidates_cart_test.parquet'\n",
    "trunked_sessions = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "buy2buy_matrice_test = '/kaggle/input/otto-prepare-candidates-buys/matrix_buy2buy_test.parquet'\n",
    "click2buy_short_matrice_test = '/kaggle/input/otto-click2buy-short/matrix_click2buy_test_short.parquet'\n",
    "click2buy_matrice_test = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_test.parquet'\n",
    "\n",
    "conversions_test = '/kaggle/input/create-counts-buys/conversions_test.parquet'\n",
    "click_before_buy_test = '/kaggle/input/create-counts-buys/clicks_before_buy_test.parquet'\n",
    "median_time_viewed_test = '/kaggle/input/create-counts-for-clicks/time_viewed_agg_test.parquet'\n",
    "history_path_test = '/kaggle/input/create-counts-buys/history_change_test.parquet'\n",
    "all_2cart_conv_test = '/kaggle/input/create-counts-buys/2cart_conversions_test.parquet'\n",
    "full_sessions = '/kaggle/input/otto-prepare-cv/train_full.parquet'\n",
    "\n",
    "counts_before_path = '/kaggle/input/create-counts-buys/daily_counts_carts_test_full.parquet'\n",
    "counts_during_path = '/kaggle/input/create-counts-buys/daily_counts_carts_test_trunked.parquet'\n",
    "counts_before_path_click = '/kaggle/input/create-counts-for-clicks/daily_counts_test_full.parquet'\n",
    "counts_during_path_click = '/kaggle/input/create-counts-for-clicks/daily_counts_test_trunked.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d614b5fe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:08:19.604760Z",
     "iopub.status.busy": "2023-03-24T16:08:19.604188Z",
     "iopub.status.idle": "2023-03-24T18:28:56.403498Z",
     "shell.execute_reply": "2023-03-24T18:28:56.399701Z"
    },
    "papermill": {
     "duration": 8436.909893,
     "end_time": "2023-03-24T18:28:56.499409",
     "exception": false,
     "start_time": "2023-03-24T16:08:19.589516",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "CPU times: user 3h 32min 46s, sys: 1h 3min 46s, total: 4h 36min 32s\n",
      "Wall time: 2h 20min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Split test dataset into 2 chunks and build features for each chunk.\n",
    "n_splits=2\n",
    "\n",
    "for i in range(n_splits):\n",
    "    df = pd.read_parquet(candidates_test)\n",
    "    df_cand = otto_common.divide_df_by_column(df, n_splits, i, 'session')\n",
    "    del df\n",
    "    gc.collect()\n",
    "    df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'cart', False)\n",
    "    \n",
    "    df_cand = add_buy2buy_weight(trunked_sessions, buy2buy_matrice_test, df_cand)\n",
    "    df_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_test, df_cand)\n",
    "    df_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_test, df_cand)\n",
    "    df_cand = add_click2buy_weight_6(trunked_sessions, click2buy_matrice_test, df_cand)\n",
    "    \n",
    "    df_cand = otto_common_fe.add_history_aid_features(trunked_sessions, df_cand, 'cart_predictions')\n",
    "    df_cand = otto_common_fe.add_history_agg_features(trunked_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.add_time_viewed(trunked_sessions, df_cand, 'cart_predictions')\n",
    "    df_cand = otto_common_fe.add_daily_averages(counts_before_path_click, counts_during_path_click, df_cand, 'cart_predictions')\n",
    "    df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path_click, df_cand, 'cart_predictions')\n",
    "    df_cand = otto_common_fe.add_type_last(trunked_sessions, df_cand, 'cart_predictions')\n",
    "    df_cand = add_total_carts(full_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.add_weekly_averages(counts_before_path, counts_during_path, df_cand, 'cart_predictions')\n",
    "    df_cand = add_conversions(conversions_test, df_cand)\n",
    "    df_cand = otto_common_fe.add_clicks_before_buy(click_before_buy_test, df_cand, 'cart_predictions')\n",
    "    df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_test, df_cand, 'cart_predictions')\n",
    "    df_cand = otto_common_fe.add_session_time(trunked_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.count_events_3hours(trunked_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.add_history_similarity(history_path_test, df_cand)\n",
    "    df_cand = join_the_2cart_conv(all_2cart_conv_test, df_cand)\n",
    "    df_cand = otto_common_fe.add_total_buys_in_session(trunked_sessions, df_cand)\n",
    "    \n",
    "    string_i = 'test_features_cart_part_' + str(i) + '.parquet'\n",
    "    df_cand.to_parquet(string_i)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 10843.103812,
   "end_time": "2023-03-24T18:29:00.118724",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-24T15:28:17.014912",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}