otto-feature-engineering-orders.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d5dedeb7",
   "metadata": {
    "papermill": {
     "duration": 0.007738,
     "end_time": "2023-03-24T15:40:09.899566",
     "exception": false,
     "start_time": "2023-03-24T15:40:09.891828",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature engineering for orders model\n",
    "This notebook builds most features for the orders model (except for w2vec features that are calculated in a separate notebook).\n",
    "It takes as input already generated candidates, co-visitation matrixes and some pre-calculations made aside in create_counts_for_buys notebook. Most of the features are built using functions common for all the three models and defined in otto_common_fe notebook, but a few features unique to orders model are built using functions defined in this notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7583601",
   "metadata": {
    "papermill": {
     "duration": 0.006066,
     "end_time": "2023-03-24T15:40:09.912323",
     "exception": false,
     "start_time": "2023-03-24T15:40:09.906257",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7371ba77",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:09.928712Z",
     "iopub.status.busy": "2023-03-24T15:40:09.927542Z",
     "iopub.status.idle": "2023-03-24T15:40:24.825982Z",
     "shell.execute_reply": "2023-03-24T15:40:24.824622Z"
    },
    "papermill": {
     "duration": 14.910305,
     "end_time": "2023-03-24T15:40:24.829156",
     "exception": false,
     "start_time": "2023-03-24T15:40:09.918851",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting polars\r\n",
      "  Downloading polars-0.16.15-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.6 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.6/16.6 MB\u001b[0m \u001b[31m47.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.1.1)\r\n",
      "Installing collected packages: polars\r\n",
      "Successfully installed polars-0.16.15\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from datetime import datetime\n",
    "from humanize import naturalsize\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common \n",
    "import otto_common_fe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "781fdfc3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:24.847588Z",
     "iopub.status.busy": "2023-03-24T15:40:24.846533Z",
     "iopub.status.idle": "2023-03-24T15:40:36.398607Z",
     "shell.execute_reply": "2023-03-24T15:40:36.396984Z"
    },
    "papermill": {
     "duration": 11.564378,
     "end_time": "2023-03-24T15:40:36.401824",
     "exception": false,
     "start_time": "2023-03-24T15:40:24.837446",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: polars in /opt/conda/lib/python3.7/site-packages (0.16.15)\r\n",
      "Requirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.1.1)\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Polars library is used to generate features from co-visitation matrix.\n",
    "!pip install polars\n",
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e26aa0a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.419332Z",
     "iopub.status.busy": "2023-03-24T15:40:36.418814Z",
     "iopub.status.idle": "2023-03-24T15:40:36.428183Z",
     "shell.execute_reply": "2023-03-24T15:40:36.427106Z"
    },
    "papermill": {
     "duration": 0.0213,
     "end_time": "2023-03-24T15:40:36.430846",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.409546",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function was used for data exploration only.\n",
    "# It selects ground truth aids that were missed during candidate generation.\n",
    "def cand_read_and_reduce_test_not_cands(cand_path, is_cv):\n",
    "    df = pd.read_parquet(cand_path)\n",
    "    df = df.loc[df['pred_true'] == 0]\n",
    "    df = df[['session', 'orders']]\n",
    "    df = df.explode('orders').reset_index(drop=True)\n",
    "    df['session'] = df['session'].astype(np.int32)\n",
    "    df['order_predictions'] = df['orders'].astype(np.int32)\n",
    "    del df['orders']\n",
    "    gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c4836545",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.449448Z",
     "iopub.status.busy": "2023-03-24T15:40:36.448974Z",
     "iopub.status.idle": "2023-03-24T15:40:36.457875Z",
     "shell.execute_reply": "2023-03-24T15:40:36.456423Z"
    },
    "papermill": {
     "duration": 0.021835,
     "end_time": "2023-03-24T15:40:36.460916",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.439081",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Number of buys per aid in history (full sessions).\n",
    "def add_total_buys(train_path, df_candidate):\n",
    "    total_counts = pd.read_parquet(train_path)\n",
    "    total_counts = total_counts.loc[total_counts['type'] > 0]\n",
    "    total_counts = total_counts.groupby('aid')['session'].nunique()\n",
    "    gc.collect()\n",
    "    total_counts.name = \"aid_counts_buys\"\n",
    "    df_candidate = pd.merge(df_candidate, total_counts, how='left',\n",
    "                            left_on='order_predictions', right_index=True)\n",
    "    df_candidate['aid_counts_buys'] = df_candidate['aid_counts_buys'].fillna(0).astype(np.int16)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce63a5be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.478355Z",
     "iopub.status.busy": "2023-03-24T15:40:36.477824Z",
     "iopub.status.idle": "2023-03-24T15:40:36.486594Z",
     "shell.execute_reply": "2023-03-24T15:40:36.484946Z"
    },
    "papermill": {
     "duration": 0.020899,
     "end_time": "2023-03-24T15:40:36.489339",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.468440",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Number of orders per aid in history (full sessions).\n",
    "def add_total_orders(train_path, df_candidate):\n",
    "    total_counts = pd.read_parquet(train_path)\n",
    "    total_counts = total_counts.loc[total_counts['type'] == 2]\n",
    "    total_counts = total_counts.groupby('aid')['session'].nunique()\n",
    "    gc.collect()\n",
    "    total_counts.name = \"aid_counts_orders\"\n",
    "    df_candidate = pd.merge(df_candidate, total_counts, how='left',\n",
    "                            left_on='order_predictions', right_index=True)\n",
    "    df_candidate['aid_counts_orders'] = df_candidate['aid_counts_orders'].fillna(0).astype(np.int16)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "afaa14c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.508234Z",
     "iopub.status.busy": "2023-03-24T15:40:36.507736Z",
     "iopub.status.idle": "2023-03-24T15:40:36.515474Z",
     "shell.execute_reply": "2023-03-24T15:40:36.514184Z"
    },
    "papermill": {
     "duration": 0.019839,
     "end_time": "2023-03-24T15:40:36.518398",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.498559",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Types of buys in session (2 - if at least one aid ordered, 1 - if no orders in history, but at least one aid was carted, 0 if only clicks in session).\n",
    "def add_buys_in_session(sessions_path, df_candidate):\n",
    "    df_session = pd.read_parquet(sessions_path)\n",
    "    df_session = df_session.groupby('session').agg({'type':'max'})\n",
    "    df_session = df_session.rename(columns={'type':'buys_in_session'})\n",
    "    df_candidate = pd.merge(df_candidate, df_session, how='left', left_on='session', right_index=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a470d848",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.535821Z",
     "iopub.status.busy": "2023-03-24T15:40:36.535316Z",
     "iopub.status.idle": "2023-03-24T15:40:36.544353Z",
     "shell.execute_reply": "2023-03-24T15:40:36.542905Z"
    },
    "papermill": {
     "duration": 0.021024,
     "end_time": "2023-03-24T15:40:36.546913",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.525889",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Adds a feature showing how often aid was ordered in case if it was clicked, carted or ordered before.\n",
    "def join_the_2order_conv(all_2order_conv_path, df_candidate):\n",
    "    df_conv = pd.read_parquet(all_2order_conv_path)\n",
    "    df_candidate = pd.merge(df_candidate, df_conv, how='left', left_on='order_predictions', right_on='aid')\n",
    "    df_candidate['total_2order_conv'] = 0\n",
    "    df_candidate.loc[df_candidate['type_last'] == 0, 'total_2order_conv'] = df_candidate['click2order_conv']\n",
    "    df_candidate.loc[df_candidate['type_last'] == 1, 'total_2order_conv'] = df_candidate['cart2order_conv']\n",
    "    df_candidate.loc[df_candidate['type_last'] == 2, 'total_2order_conv'] = df_candidate['order2order_conv']\n",
    "    df_candidate['total_2order_conv'] = df_candidate['total_2order_conv'].astype(np.float32)\n",
    "    del df_candidate['aid'], df_candidate['click2order_conv'], df_candidate['cart2order_conv']\n",
    "    del df_candidate['order2order_conv']\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7460919f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.563976Z",
     "iopub.status.busy": "2023-03-24T15:40:36.563494Z",
     "iopub.status.idle": "2023-03-24T15:40:36.574535Z",
     "shell.execute_reply": "2023-03-24T15:40:36.573210Z"
    },
    "papermill": {
     "duration": 0.023096,
     "end_time": "2023-03-24T15:40:36.577450",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.554354",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using buy2buy co-visitation matrix.\n",
    "# Uses all the orders in session and only recent carts. For sessions with no orders and no recent carts, the feature is always 0.\n",
    "def add_buy2buy_weight(input_path, buy2buy_matrice, df_candidate):\n",
    "    n_max = 40\n",
    "    col_name = 'wgt_buy2buy'\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test['time_delta'] = df_test.groupby('session')['ts'].transform(np.max).astype(np.int32)\n",
    "    df_test['time_delta'] = df_test['time_delta'] - df_test['ts']\n",
    "    df_test = df_test.loc[((df_test['time_delta'] < 24 * 60 * 60) & (df_test['type'] == 1)) | (df_test['type'] == 2)]\n",
    "    gc.collect()\n",
    "    df_test = df_test.sort_values(['session','ts'],ascending=[True,False])\n",
    "    df_test['n'] = df_test.groupby('session').cumcount().astype(np.int16)\n",
    "    df_test = df_test.loc[df_test['n'] < n_max]\n",
    "    df_test = df_test.drop('time_delta', axis=1)\n",
    "    df_test = df_test[['session', 'aid', 'n']]\n",
    "    gc.collect()\n",
    "    \n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)    \n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, buy2buy_matrice, df_candidate, col_name, n_max,\n",
    "                                                          'order_predictions', normalize=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "96907fdd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.594840Z",
     "iopub.status.busy": "2023-03-24T15:40:36.594363Z",
     "iopub.status.idle": "2023-03-24T15:40:36.601583Z",
     "shell.execute_reply": "2023-03-24T15:40:36.600321Z"
    },
    "papermill": {
     "duration": 0.018939,
     "end_time": "2023-03-24T15:40:36.604013",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.585074",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using click2buy short co-visitation matrix. Short refers to time between click and buy events, only\n",
    "# cases of relatively short period of time are counted.\n",
    "# Uses no more than 15 events from the last hour of the session.\n",
    "def add_click2buy_short_weight(input_path, c2buy_short_matrice, df_candidate):\n",
    "    n_max = 15\n",
    "    col_name = 'wgt_c2buy_short'\n",
    "    time_frame = 60 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, c2buy_short_matrice, df_candidate, col_name, n_max,\n",
    "                                                         'order_predictions')\n",
    "    return df_candidate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "649ed1e5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.621193Z",
     "iopub.status.busy": "2023-03-24T15:40:36.620687Z",
     "iopub.status.idle": "2023-03-24T15:40:36.627963Z",
     "shell.execute_reply": "2023-03-24T15:40:36.626756Z"
    },
    "papermill": {
     "duration": 0.019118,
     "end_time": "2023-03-24T15:40:36.630691",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.611573",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using click2buy \"long\" co-visitation matrix. \n",
    "# Uses no more than 30 events from the last 3 hours of each session.\n",
    "def add_click2buy_weight(input_path, c2buy_matrice, df_candidate):\n",
    "    n_max = 30\n",
    "    col_name = 'wgt_c2buy_full'\n",
    "    time_frame = 3 * 60 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, c2buy_matrice, df_candidate, col_name, n_max,\n",
    "                                                          'order_predictions', normalize=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1d0ab6b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.647941Z",
     "iopub.status.busy": "2023-03-24T15:40:36.647453Z",
     "iopub.status.idle": "2023-03-24T15:40:36.654775Z",
     "shell.execute_reply": "2023-03-24T15:40:36.653472Z"
    },
    "papermill": {
     "duration": 0.019224,
     "end_time": "2023-03-24T15:40:36.657461",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.638237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generates a feature using click2buy \"long-visitation matrix. \n",
    "# Uses just 6 events from the last 3 hours of each session.\n",
    "def add_click2buy_weight_6(input_path, c2buy_matrice, df_candidate):\n",
    "    n_max = 6\n",
    "    col_name = 'wgt_c2buy_6_from_full'\n",
    "    time_frame = 3 * 60 * 60\n",
    "    df_test = pd.read_parquet(input_path)    \n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, c2buy_matrice, df_candidate, col_name, n_max,\n",
    "                                                         'order_predictions')\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "141d6a0d",
   "metadata": {
    "papermill": {
     "duration": 0.007028,
     "end_time": "2023-03-24T15:40:36.671954",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.664926",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature engineering for cross-validation datasets\n",
    "For the orders model, two different cross-validation datasets are used to create the same features. Later two different models will be trained on those data and then their predictions will be averaged to get the final prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3f0845f6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.689851Z",
     "iopub.status.busy": "2023-03-24T15:40:36.689325Z",
     "iopub.status.idle": "2023-03-24T15:40:36.697754Z",
     "shell.execute_reply": "2023-03-24T15:40:36.696025Z"
    },
    "papermill": {
     "duration": 0.020629,
     "end_time": "2023-03-24T15:40:36.700447",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.679818",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for the first cross-validation dataset.\n",
    "# Most inputs are the same for both cross-validation datasets, but not all of them.\n",
    "candidates_cv1 = '/kaggle/input/otto-validate-candidates-orders/candidates_order_cv1.parquet'\n",
    "trunked_sessions = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "buy2buy_matrice_cv = '/kaggle/input/otto-prepare-candidates-buys/matrix_buy2buy_cv.parquet'\n",
    "\n",
    "click2buy_short_matrice_cv = '/kaggle/input/otto-click2buy-short/matrix_click2buy_cv_short.parquet'\n",
    "click2buy_matrice_cv = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_cv.parquet'\n",
    "conversions_cv = '/kaggle/input/create-counts-buys/conversions_cv.parquet'\n",
    "all2order_conv_cv = '/kaggle/input/create-counts-buys/2order_conversions_cv.parquet'\n",
    "click_before_buy_cv = '/kaggle/input/create-counts-buys/clicks_before_buy_cv.parquet'\n",
    "median_time_viewed_cv = '/kaggle/input/create-counts-for-clicks/time_viewed_agg_cv.parquet'\n",
    "history_path_cv = '/kaggle/input/create-counts-buys/history_change_cv1.parquet'\n",
    "full_sessions = '/kaggle/input/otto-prepare-cv/cv_train.parquet'\n",
    "\n",
    "counts_before_path = '/kaggle/input/create-counts-buys/daily_counts_orders_train.parquet'\n",
    "counts_during_path = '/kaggle/input/create-counts-buys/daily_counts_orders_cv1.parquet'\n",
    "counts_before_path_clicks = '/kaggle/input/create-counts-for-clicks/daily_counts_train.parquet'\n",
    "counts_during_path_clicks = '/kaggle/input/create-counts-for-clicks/daily_counts_cv1.parquet'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7a3daa54",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T15:40:36.717632Z",
     "iopub.status.busy": "2023-03-24T15:40:36.717175Z",
     "iopub.status.idle": "2023-03-24T16:15:12.774641Z",
     "shell.execute_reply": "2023-03-24T16:15:12.772844Z"
    },
    "papermill": {
     "duration": 2076.070487,
     "end_time": "2023-03-24T16:15:12.778437",
     "exception": false,
     "start_time": "2023-03-24T15:40:36.707950",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "CPU times: user 1h 5min 5s, sys: 17min 6s, total: 1h 22min 11s\n",
      "Wall time: 34min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Imports candidates and builds features from co-visitation matrixes for the first cross-validation dataset.\n",
    "\n",
    "df_cand = pd.read_parquet(candidates_cv1)\n",
    "df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'order', True)\n",
    "\n",
    "df_cand = add_buy2buy_weight(trunked_sessions, buy2buy_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight_6(trunked_sessions, click2buy_matrice_cv, df_cand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c753d151",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:15:12.807818Z",
     "iopub.status.busy": "2023-03-24T16:15:12.807307Z",
     "iopub.status.idle": "2023-03-24T16:19:36.534585Z",
     "shell.execute_reply": "2023-03-24T16:19:36.532948Z"
    },
    "papermill": {
     "duration": 263.756644,
     "end_time": "2023-03-24T16:19:36.548376",
     "exception": false,
     "start_time": "2023-03-24T16:15:12.791732",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 37s, sys: 55.1 s, total: 4min 32s\n",
      "Wall time: 4min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Builds all the other features. Takes significantly less time than building features from co-visitation matrixes.\n",
    "\n",
    "#df_cand = pd.read_parquet(candidates_cv1)\n",
    "#df_cand = otto_common.cand_read_and_reduce(df_cand, 'order', True)\n",
    "\n",
    "df_cand = otto_common_fe.add_history_aid_features(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_history_agg_features(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_time_viewed(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_type_last(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages(counts_before_path_clicks,counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'order_predictions')\n",
    "df_cand = add_total_buys(full_sessions, df_cand)\n",
    "df_cand = add_total_orders(full_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_session_time(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.count_events_3hours(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_total_buys_in_session(trunked_sessions, df_cand)\n",
    "df_cand = add_buys_in_session(trunked_sessions, df_cand)\n",
    "df_cand = join_the_2order_conv(all2order_conv_cv, df_cand)\n",
    "df_cand = otto_common_fe.add_clicks_before_buy(click_before_buy_cv, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_cv, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_history_similarity(history_path_cv, df_cand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5e8d7bfd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:19:36.576848Z",
     "iopub.status.busy": "2023-03-24T16:19:36.576383Z",
     "iopub.status.idle": "2023-03-24T16:19:36.588440Z",
     "shell.execute_reply": "2023-03-24T16:19:36.587160Z"
    },
    "papermill": {
     "duration": 0.029268,
     "end_time": "2023-03-24T16:19:36.590988",
     "exception": false,
     "start_time": "2023-03-24T16:19:36.561720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n%%time\\n# Was used to check all aids that were not selected during candidate generation.\\n\\n#df_cand = pd.read_parquet(candidates_cv1)\\ndf_cand = cand_read_and_reduce_test_not_cands(candidates_cv1, True)\\ndf_cand = otto_common.add_history_aid_features(trunked_sessions, df_cand, 'order_predictions')\\ndf_cand = otto_common.add_history_agg_features(trunked_sessions, df_cand)\\ndf_cand = otto_common.add_time_viewed(trunked_sessions, df_cand, 'order_predictions')\\ndf_cand = otto_common.add_type_last(trunked_sessions, df_cand, 'order_predictions')\\ndf_cand = otto_common.add_daily_averages(counts_before_path_clicks,counts_during_path_clicks, df_cand, 'order_predictions')\\ndf_cand = otto_common.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'order_predictions')\\ndf_cand = add_total_buys(full_sessions, df_cand)\\ndf_cand = add_total_orders(full_sessions, df_cand)\\ndf_cand = otto_common.add_session_time(trunked_sessions, df_cand)\\ndf_cand = otto_common.count_events_3hours(trunked_sessions, df_cand)\\ndf_cand = otto_common.add_total_buys_in_session(trunked_sessions, df_cand)\\ndf_cand = add_buys_in_session(trunked_sessions, df_cand)\\ndf_cand = join_the_2order_conv(all2order_conv_cv, df_cand)\\ndf_cand = otto_common.add_clicks_before_buy(click_before_buy_cv, df_cand, 'order_predictions')\\ndf_cand = otto_common.add_median_time_viewed(median_time_viewed_cv, df_cand, 'order_predictions')\\ndf_cand = otto_common.add_history_similarity(history_path_cv, df_cand)\\n\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "# Was used to check all aids that were not selected during candidate generation.\n",
    "\n",
    "#df_cand = pd.read_parquet(candidates_cv1)\n",
    "df_cand = cand_read_and_reduce_test_not_cands(candidates_cv1, True)\n",
    "df_cand = otto_common.add_history_aid_features(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common.add_history_agg_features(trunked_sessions, df_cand)\n",
    "df_cand = otto_common.add_time_viewed(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common.add_type_last(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common.add_daily_averages(counts_before_path_clicks,counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "df_cand = otto_common.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'order_predictions')\n",
    "df_cand = add_total_buys(full_sessions, df_cand)\n",
    "df_cand = add_total_orders(full_sessions, df_cand)\n",
    "df_cand = otto_common.add_session_time(trunked_sessions, df_cand)\n",
    "df_cand = otto_common.count_events_3hours(trunked_sessions, df_cand)\n",
    "df_cand = otto_common.add_total_buys_in_session(trunked_sessions, df_cand)\n",
    "df_cand = add_buys_in_session(trunked_sessions, df_cand)\n",
    "df_cand = join_the_2order_conv(all2order_conv_cv, df_cand)\n",
    "df_cand = otto_common.add_clicks_before_buy(click_before_buy_cv, df_cand, 'order_predictions')\n",
    "df_cand = otto_common.add_median_time_viewed(median_time_viewed_cv, df_cand, 'order_predictions')\n",
    "df_cand = otto_common.add_history_similarity(history_path_cv, df_cand)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ab834ede",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:19:36.619718Z",
     "iopub.status.busy": "2023-03-24T16:19:36.619214Z",
     "iopub.status.idle": "2023-03-24T16:19:42.631754Z",
     "shell.execute_reply": "2023-03-24T16:19:42.630377Z"
    },
    "papermill": {
     "duration": 6.030806,
     "end_time": "2023-03-24T16:19:42.635203",
     "exception": false,
     "start_time": "2023-03-24T16:19:36.604397",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "862.0 MB\n"
     ]
    }
   ],
   "source": [
    "# Check resulting dataframe size and export to file.\n",
    "size = df_cand.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_cand.to_parquet('cv_features_order.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "066f891b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:19:42.666023Z",
     "iopub.status.busy": "2023-03-24T16:19:42.665569Z",
     "iopub.status.idle": "2023-03-24T16:19:42.671775Z",
     "shell.execute_reply": "2023-03-24T16:19:42.670404Z"
    },
    "papermill": {
     "duration": 0.024209,
     "end_time": "2023-03-24T16:19:42.674243",
     "exception": false,
     "start_time": "2023-03-24T16:19:42.650034",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for the second cross-validation dataset (only paths that are different between the cross-validation datasets).\n",
    "candidates_cv2 = '/kaggle/input/otto-validate-candidates-orders/candidates_order_cv2.parquet'\n",
    "trunked_sessions = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "history_path_cv2 = '/kaggle/input/create-counts-buys/history_change_cv2.parquet'\n",
    "\n",
    "counts_during_path = '/kaggle/input/create-counts-buys/daily_counts_orders_cv2.parquet'\n",
    "counts_during_path_clicks = '/kaggle/input/create-counts-for-clicks/daily_counts_cv2.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "497d2589",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:19:42.704476Z",
     "iopub.status.busy": "2023-03-24T16:19:42.703028Z",
     "iopub.status.idle": "2023-03-24T16:53:57.018762Z",
     "shell.execute_reply": "2023-03-24T16:53:57.017383Z"
    },
    "papermill": {
     "duration": 2054.35268,
     "end_time": "2023-03-24T16:53:57.040722",
     "exception": false,
     "start_time": "2023-03-24T16:19:42.688042",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "CPU times: user 1h 9min 41s, sys: 17min 44s, total: 1h 27min 25s\n",
      "Wall time: 34min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Import candidates and build features from co-visitation matrixes for the second dataset.\n",
    "\n",
    "df_cand = pd.read_parquet(candidates_cv2)\n",
    "df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'order', True)\n",
    "df_cand = add_buy2buy_weight(trunked_sessions, buy2buy_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_cv, df_cand)\n",
    "df_cand = add_click2buy_weight_6(trunked_sessions, click2buy_matrice_cv, df_cand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "73befd34",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:53:57.081442Z",
     "iopub.status.busy": "2023-03-24T16:53:57.080907Z",
     "iopub.status.idle": "2023-03-24T16:58:48.454678Z",
     "shell.execute_reply": "2023-03-24T16:58:48.453391Z"
    },
    "papermill": {
     "duration": 291.417649,
     "end_time": "2023-03-24T16:58:48.477220",
     "exception": false,
     "start_time": "2023-03-24T16:53:57.059571",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 55s, sys: 1min 19s, total: 5min 14s\n",
      "Wall time: 4min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Build all the other features for the second dataset.\n",
    "\n",
    "df_cand = otto_common_fe.add_history_aid_features(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_history_agg_features(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_time_viewed(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_type_last(trunked_sessions, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages(counts_before_path_clicks,counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'order_predictions')\n",
    "df_cand = add_total_buys(full_sessions, df_cand)\n",
    "df_cand = add_total_orders(full_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_session_time(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.count_events_3hours(trunked_sessions, df_cand)\n",
    "df_cand = otto_common_fe.add_total_buys_in_session(trunked_sessions, df_cand)\n",
    "df_cand = add_buys_in_session(trunked_sessions, df_cand)\n",
    "df_cand = join_the_2order_conv(all2order_conv_cv, df_cand)\n",
    "df_cand = otto_common_fe.add_clicks_before_buy(click_before_buy_cv, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_cv, df_cand, 'order_predictions')\n",
    "df_cand = otto_common_fe.add_history_similarity(history_path_cv2, df_cand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b6b23d67",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:58:48.519364Z",
     "iopub.status.busy": "2023-03-24T16:58:48.518808Z",
     "iopub.status.idle": "2023-03-24T16:58:54.537969Z",
     "shell.execute_reply": "2023-03-24T16:58:54.536639Z"
    },
    "papermill": {
     "duration": 6.043876,
     "end_time": "2023-03-24T16:58:54.540968",
     "exception": false,
     "start_time": "2023-03-24T16:58:48.497092",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "860.5 MB\n"
     ]
    }
   ],
   "source": [
    "# Check resulting dataframe size and export to file.\n",
    "size = df_cand.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_cand.to_parquet('cv2_features_order.parquet')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ab18b28",
   "metadata": {
    "papermill": {
     "duration": 0.019162,
     "end_time": "2023-03-24T16:58:54.579996",
     "exception": false,
     "start_time": "2023-03-24T16:58:54.560834",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature engineering for test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "00cec253",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:58:54.841051Z",
     "iopub.status.busy": "2023-03-24T16:58:54.840262Z",
     "iopub.status.idle": "2023-03-24T16:58:54.847631Z",
     "shell.execute_reply": "2023-03-24T16:58:54.846461Z"
    },
    "papermill": {
     "duration": 0.030998,
     "end_time": "2023-03-24T16:58:54.850337",
     "exception": false,
     "start_time": "2023-03-24T16:58:54.819339",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for test dataset.\n",
    "candidates_test = '/kaggle/input/otto-validate-candidates-orders/candidates_order_test.parquet'\n",
    "trunked_sessions = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "buy2buy_matrice_test = '/kaggle/input/otto-prepare-candidates-buys/matrix_buy2buy_test.parquet'\n",
    "\n",
    "click2buy_short_matrice_test = '/kaggle/input/otto-click2buy-short/matrix_click2buy_test_short.parquet'\n",
    "click2buy_matrice_test = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_test.parquet'\n",
    "all2order_conv_test = '/kaggle/input/create-counts-buys/2order_conversions_test.parquet'\n",
    "conversions_test = '/kaggle/input/create-counts-buys/conversions_test.parquet'\n",
    "click_before_buy_test = '/kaggle/input/create-counts-buys/clicks_before_buy_test.parquet'\n",
    "median_time_viewed_test = '/kaggle/input/create-counts-for-clicks/time_viewed_agg_test.parquet'\n",
    "history_path_test = '/kaggle/input/create-counts-buys/history_change_test.parquet'\n",
    "full_sessions = '/kaggle/input/otto-prepare-cv/train_full.parquet'\n",
    "\n",
    "counts_before_path = '/kaggle/input/create-counts-buys/daily_counts_orders_test_full.parquet'\n",
    "counts_during_path = '/kaggle/input/create-counts-buys/daily_counts_orders_test_trunked.parquet'\n",
    "counts_before_path_clicks = '/kaggle/input/create-counts-for-clicks/daily_counts_test_full.parquet'\n",
    "counts_during_path_clicks = '/kaggle/input/create-counts-for-clicks/daily_counts_test_trunked.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "619c96f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T16:58:54.891877Z",
     "iopub.status.busy": "2023-03-24T16:58:54.891354Z",
     "iopub.status.idle": "2023-03-24T19:45:36.916885Z",
     "shell.execute_reply": "2023-03-24T19:45:36.913243Z"
    },
    "papermill": {
     "duration": 10002.085672,
     "end_time": "2023-03-24T19:45:36.955668",
     "exception": false,
     "start_time": "2023-03-24T16:58:54.869996",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "start normalizing\n",
      "wgt_buy2buy\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "wgt_c2buy_short\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "start normalizing\n",
      "wgt_c2buy_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "wgt_c2buy_6_from_full\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "CPU times: user 4h 23min 26s, sys: 1h 14min 27s, total: 5h 37min 54s\n",
      "Wall time: 2h 46min 42s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Splits test dataset into 2 chunks and builds features for both chunks.\n",
    "n_splits=2\n",
    "\n",
    "for i in range(n_splits):\n",
    "    df = pd.read_parquet(candidates_test)\n",
    "    df_cand = otto_common.divide_df_by_column(df, n_splits, i, 'session')\n",
    "    del df\n",
    "    gc.collect()\n",
    "    df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'order', False)\n",
    "    df_cand = add_buy2buy_weight(trunked_sessions, buy2buy_matrice_test, df_cand)\n",
    "    df_cand = add_click2buy_short_weight(trunked_sessions, click2buy_short_matrice_test, df_cand)\n",
    "    df_cand = add_click2buy_weight(trunked_sessions, click2buy_matrice_test, df_cand)\n",
    "    df_cand = add_click2buy_weight_6(trunked_sessions, click2buy_matrice_test, df_cand)\n",
    "        \n",
    "    df_cand = otto_common_fe.add_history_aid_features(trunked_sessions, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_history_agg_features(trunked_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.add_time_viewed(trunked_sessions, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_type_last(trunked_sessions, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_daily_averages(counts_before_path_clicks,counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path_clicks, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'order_predictions')\n",
    "    df_cand = add_total_buys(full_sessions, df_cand)\n",
    "    df_cand = add_total_orders(full_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.add_session_time(trunked_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.count_events_3hours(trunked_sessions, df_cand)\n",
    "    df_cand = otto_common_fe.add_total_buys_in_session(trunked_sessions, df_cand)\n",
    "    df_cand = add_buys_in_session(trunked_sessions, df_cand)\n",
    "    df_cand = join_the_2order_conv(all2order_conv_test, df_cand)\n",
    "    df_cand = otto_common_fe.add_clicks_before_buy(click_before_buy_test, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_test, df_cand, 'order_predictions')\n",
    "    df_cand = otto_common_fe.add_history_similarity(history_path_test, df_cand)\n",
    "    string_i = 'test_features_order_part_' + str(i) + '.parquet'\n",
    "    df_cand.to_parquet(string_i)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 14739.421648,
   "end_time": "2023-03-24T19:45:38.680337",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-24T15:39:59.258689",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}