otto-feature-engineering-clicks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d05be78",
   "metadata": {
    "papermill": {
     "duration": 0.006621,
     "end_time": "2023-03-20T17:01:03.080993",
     "exception": false,
     "start_time": "2023-03-20T17:01:03.074372",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature engineering for clicks model\n",
    "This notebook builds most features for the clicks model (except for w2vec features that are calculated in a separate notebook).\n",
    "It takes as input already generated candidates, co-visitation matrixes and some pre-calculations made aside in create_counts_for_clicks notebook. Most of the features are built using functions common for all the three models that are defined in otto_common_fe notebook, but a few features unique to clicks model are built using functions defined in this notebook.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2d2ea837",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:03.094376Z",
     "iopub.status.busy": "2023-03-20T17:01:03.093716Z",
     "iopub.status.idle": "2023-03-20T17:01:17.758809Z",
     "shell.execute_reply": "2023-03-20T17:01:17.757571Z"
    },
    "papermill": {
     "duration": 14.675678,
     "end_time": "2023-03-20T17:01:17.762042",
     "exception": false,
     "start_time": "2023-03-20T17:01:03.086364",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting polars\r\n",
      "  Downloading polars-0.16.14-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.2 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.2/16.2 MB\u001b[0m \u001b[31m51.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.4.0)\r\n",
      "Installing collected packages: polars\r\n",
      "Successfully installed polars-0.16.14\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "     \n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common, otto_common_fe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5d66489d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:17.779311Z",
     "iopub.status.busy": "2023-03-20T17:01:17.778904Z",
     "iopub.status.idle": "2023-03-20T17:01:28.873647Z",
     "shell.execute_reply": "2023-03-20T17:01:28.872444Z"
    },
    "papermill": {
     "duration": 11.107808,
     "end_time": "2023-03-20T17:01:28.876340",
     "exception": false,
     "start_time": "2023-03-20T17:01:17.768532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: polars in /opt/conda/lib/python3.7/site-packages (0.16.14)\r\n",
      "Requirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.4.0)\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Polars library is used to generate features from co-visitation matrixes.\n",
    "!pip install polars\n",
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5616973b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:28.890492Z",
     "iopub.status.busy": "2023-03-20T17:01:28.890012Z",
     "iopub.status.idle": "2023-03-20T17:01:28.896758Z",
     "shell.execute_reply": "2023-03-20T17:01:28.895804Z"
    },
    "papermill": {
     "duration": 0.016795,
     "end_time": "2023-03-20T17:01:28.899107",
     "exception": false,
     "start_time": "2023-03-20T17:01:28.882312",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function was used for data exploration only.\n",
    "# It selects ground truth aids that were missed during candidate generation.\n",
    "def cand_read_and_reduce_test_not_cands(cand_path, is_cv):\n",
    "    df = pd.read_parquet(cand_path)\n",
    "    df = df.loc[df['pred_true'] == 0]\n",
    "    df = df[['session', 'clicks']]\n",
    "    df['session'] = df['session'].astype(np.int32)\n",
    "    df['click_predictions'] = df['clicks'].astype(np.int32)\n",
    "    del df['clicks']\n",
    "    gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7d859986",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:28.913099Z",
     "iopub.status.busy": "2023-03-20T17:01:28.912679Z",
     "iopub.status.idle": "2023-03-20T17:01:28.921535Z",
     "shell.execute_reply": "2023-03-20T17:01:28.920364Z"
    },
    "papermill": {
     "duration": 0.018792,
     "end_time": "2023-03-20T17:01:28.923962",
     "exception": false,
     "start_time": "2023-03-20T17:01:28.905170",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Apply co-visitation matrix to the last two aids in each truncated session.\n",
    "# Columns for the last two aids created by add_history_agg_features function.\n",
    "def add_matrix_calculations(input_path, df_candidate):\n",
    "    df_matrix = pd.read_parquet(input_path)\n",
    "    df_candidate = pd.merge(df_candidate, df_matrix, how='left', \n",
    "                   left_on=['first_aid','click_predictions'], right_on=['aid_x','aid_y'])\n",
    "    del df_candidate['aid_x'], df_candidate['aid_y']\n",
    "    gc.collect()\n",
    "    df_candidate = pd.merge(df_candidate, df_matrix, how='left', \n",
    "                   left_on=['second_aid','click_predictions'], right_on=['aid_x','aid_y'])\n",
    "    del df_candidate['aid_x'], df_candidate['aid_y']\n",
    "    gc.collect()\n",
    "    df_candidate = df_candidate.rename(columns={'wgt_x' : 'wgt_last', 'wgt_y' : 'wgt_before_last'})\n",
    "    df_candidate['wgt_last'] = df_candidate['wgt_last'].fillna(0)\n",
    "    df_candidate['wgt_before_last'] = df_candidate['wgt_before_last'].fillna(0)    \n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd0002ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:28.937595Z",
     "iopub.status.busy": "2023-03-20T17:01:28.937124Z",
     "iopub.status.idle": "2023-03-20T17:01:28.943500Z",
     "shell.execute_reply": "2023-03-20T17:01:28.942472Z"
    },
    "papermill": {
     "duration": 0.015988,
     "end_time": "2023-03-20T17:01:28.945724",
     "exception": false,
     "start_time": "2023-03-20T17:01:28.929736",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count total number of events for each aid and add this count as a feature.\n",
    "def add_total_counts(train_path, df_candidate):\n",
    "    total_counts = pd.read_parquet(train_path)\n",
    "    total_counts = total_counts.groupby('aid')['session'].nunique()\n",
    "    gc.collect()\n",
    "    total_counts.name = \"aid_counts\"\n",
    "    total_counts = total_counts.astype(np.int32)\n",
    "    df_candidate = pd.merge(df_candidate, total_counts, how='left', left_on='click_predictions', right_index=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e9f56c42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:28.959374Z",
     "iopub.status.busy": "2023-03-20T17:01:28.958955Z",
     "iopub.status.idle": "2023-03-20T17:01:28.967423Z",
     "shell.execute_reply": "2023-03-20T17:01:28.966485Z"
    },
    "papermill": {
     "duration": 0.018248,
     "end_time": "2023-03-20T17:01:28.969763",
     "exception": false,
     "start_time": "2023-03-20T17:01:28.951515",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# A function that performs join operation between polar dataframes chunk-by-chunk.\n",
    "def join_matrice_chunks(df, df_matrice, n_chunks, fields_df):\n",
    "    for j in range(n_chunks):\n",
    "        print('j = ' + str(j))\n",
    "        max_total_aid_x = df_matrice.select(pl.col(\"aid_x\").max())\n",
    "        aid_x_min = j*max_total_aid_x/n_chunks\n",
    "        if j + 1 == n_chunks:\n",
    "            aid_x_max = max_total_aid_x + 1\n",
    "        else:\n",
    "            aid_x_max = (j+1)*(max_total_aid_x)/n_chunks\n",
    "        df_matrice_chunk = df_matrice.filter((pl.col('aid_x') >= aid_x_min) & (pl.col('aid_x') < aid_x_max))\n",
    "        df = df.join(df_matrice_chunk, left_on=fields_df, how='left', right_on=['aid_x','aid_y'])\n",
    "        if j == 0:\n",
    "            df = df.rename({\"wgt\": \"wgt_all\"})\n",
    "        else:\n",
    "            df = df.with_columns(pl.col(\"wgt_all\").fill_null(pl.col(\"wgt\"))).drop('wgt')\n",
    "    df = df.rename({\"wgt_all\": \"wgt\"}).drop('aid')\n",
    "    return df\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "421758f0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:28.983850Z",
     "iopub.status.busy": "2023-03-20T17:01:28.983446Z",
     "iopub.status.idle": "2023-03-20T17:01:28.993808Z",
     "shell.execute_reply": "2023-03-20T17:01:28.992644Z"
    },
    "papermill": {
     "duration": 0.020637,
     "end_time": "2023-03-20T17:01:28.996377",
     "exception": false,
     "start_time": "2023-03-20T17:01:28.975740",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# A special function to build features using full click2click co-visitation matrix. This matrix is too big and causes memory crush if using a standard\n",
    "# function from otto_common_fe. So, I had to write another function that performs the join operation for click2click co-visitation matrix \n",
    "# chunk-by-chunk.\n",
    "def add_big_matrice_data_polars(df_test, count_matrice, df_candidate, col_name, n_max, prediction_col):\n",
    "    print('start loading matrice')\n",
    "    df_matrice = pl.read_parquet(count_matrice)\n",
    "    print('success loading matrice')\n",
    "    df_matrice = df_matrice.drop('__index_level_0__')\n",
    "    print(col_name)\n",
    "    for i in range(n_max):\n",
    "        print(str(i))\n",
    "        df_test_i = df_test.filter(pl.col(\"n\") == i).drop('n')\n",
    "        df_candidate = df_candidate.join(df_test_i, on='session', how='left')\n",
    "        df_candidate = join_matrice_chunks(df_candidate, df_matrice, 5, ['aid', prediction_col])\n",
    "        gc.collect()\n",
    "        if i == 0:\n",
    "            df_candidate = df_candidate.with_columns(pl.col(\"wgt\").fill_null(0).alias(col_name))\n",
    "        else:\n",
    "            df_candidate = df_candidate.with_columns((pl.col(\"wgt\").fill_null(0) + pl.col(col_name))\n",
    "                                                    .alias(col_name).cast(pl.Float32))\n",
    "        df_candidate = df_candidate.drop('wgt')\n",
    "        gc.collect()\n",
    "    df_candidate = df_candidate.to_pandas()\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2e40bb42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:29.011593Z",
     "iopub.status.busy": "2023-03-20T17:01:29.010290Z",
     "iopub.status.idle": "2023-03-20T17:01:29.018671Z",
     "shell.execute_reply": "2023-03-20T17:01:29.017240Z"
    },
    "papermill": {
     "duration": 0.018909,
     "end_time": "2023-03-20T17:01:29.021684",
     "exception": false,
     "start_time": "2023-03-20T17:01:29.002775",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Builds a feature from \"experimental\" co-visitation matrix using standard function add_matrice_data_polars from otto_common_fe.\n",
    "def add_history_experiment_features_pl(input_path, experiment_matrix_path, df_candidate):\n",
    "    n_max = 10\n",
    "    col_name = 'wgt_exp'\n",
    "    time_frame = 5 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = otto_common_fe.add_matrice_data_polars(df_test, experiment_matrix_path, df_candidate, col_name, n_max,\n",
    "                                                          'click_predictions', normalize=True, divide=True)\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "29d15859",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:29.037363Z",
     "iopub.status.busy": "2023-03-20T17:01:29.036961Z",
     "iopub.status.idle": "2023-03-20T17:01:29.044167Z",
     "shell.execute_reply": "2023-03-20T17:01:29.042886Z"
    },
    "papermill": {
     "duration": 0.018656,
     "end_time": "2023-03-20T17:01:29.046447",
     "exception": false,
     "start_time": "2023-03-20T17:01:29.027791",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Builds a feature from regular co-visitation matrix using a modified function, that performs join operation chunk-by-chunk to limit memory usage,\n",
    "# as regular click2click co-visitation matrix is larger than all the other co-visitation matrixes and using a straightforward join causes memory error.\n",
    "def add_matrix_weight_pl(input_path, matrix_path, df_candidate):\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    n_max = 5\n",
    "    col_name = 'wgt_matrix'\n",
    "    time_frame = 5 * 60\n",
    "    df_test = pd.read_parquet(input_path)\n",
    "    df_test= otto_common.filter_by_time_and_n_max(df_test, time_frame, n_max)\n",
    "    df_candidate = pl.from_pandas(df_candidate)\n",
    "    df_test = pl.from_pandas(df_test)\n",
    "    df_candidate = add_big_matrice_data_polars(df_test, matrix_path, df_candidate, col_name, n_max, 'click_predictions')\n",
    "    return df_candidate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eb0bea3",
   "metadata": {
    "papermill": {
     "duration": 0.005658,
     "end_time": "2023-03-20T17:01:29.058144",
     "exception": false,
     "start_time": "2023-03-20T17:01:29.052486",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature engineering for cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c9ede0e5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:29.071940Z",
     "iopub.status.busy": "2023-03-20T17:01:29.071539Z",
     "iopub.status.idle": "2023-03-20T17:01:29.077455Z",
     "shell.execute_reply": "2023-03-20T17:01:29.076300Z"
    },
    "papermill": {
     "duration": 0.015829,
     "end_time": "2023-03-20T17:01:29.079853",
     "exception": false,
     "start_time": "2023-03-20T17:01:29.064024",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for cross-validation dataset.\n",
    "cand_path = '/kaggle/input/otto-click-candidates-validation/candidates_click.parquet'\n",
    "cv_path = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "cv_matrix_path = '/kaggle/input/create-counts-for-clicks/matrix_exact_next_counts_cv.parquet'\n",
    "experiment_matrix_path = '/kaggle/input/otto-experiment-prepare-candidates-clicks/matrix_experimental_cv.parquet'\n",
    "matrix_path_cv = '/kaggle/input/otto-prepare-candidates-clicks/regular_click2click_matrix_cv.parquet'\n",
    "return_rate_path_cv = '/kaggle/input/create-counts-for-clicks/return_rate_cv.parquet'\n",
    "median_time_viewed_cv = '/kaggle/input/create-counts-for-clicks/time_viewed_agg_cv.parquet'\n",
    "\n",
    "counts_before_path = '/kaggle/input/create-counts-for-clicks/daily_counts_train.parquet'\n",
    "counts_during_path = '/kaggle/input/create-counts-for-clicks/daily_counts_cv1.parquet'\n",
    "train_path = '/kaggle/input/otto-prepare-cv/cv_train.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cd817679",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:01:29.093725Z",
     "iopub.status.busy": "2023-03-20T17:01:29.093307Z",
     "iopub.status.idle": "2023-03-20T17:40:47.533703Z",
     "shell.execute_reply": "2023-03-20T17:40:47.529988Z"
    },
    "papermill": {
     "duration": 2358.484038,
     "end_time": "2023-03-20T17:40:47.569876",
     "exception": false,
     "start_time": "2023-03-20T17:01:29.085838",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start loading matrice\n",
      "success loading matrice\n",
      "wgt_matrix\n",
      "0\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "1\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "2\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "3\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "4\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "start normalizing\n",
      "wgt_exp\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "CPU times: user 49min 53s, sys: 15min 8s, total: 1h 5min 1s\n",
      "Wall time: 39min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Building features for the first cross-validation dataset.\n",
    "\n",
    "df_cand = pd.read_parquet(cand_path)\n",
    "df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'click', True)\n",
    "df_cand = add_matrix_weight_pl(cv_path, matrix_path_cv, df_cand)\n",
    "df_cand = add_history_experiment_features_pl(cv_path, experiment_matrix_path, df_cand)\n",
    "df_cand = otto_common_fe.add_history_aid_features(cv_path, df_cand, 'click_predictions')\n",
    "df_cand = otto_common_fe.add_history_agg_features(cv_path, df_cand, False)\n",
    "df_cand = otto_common_fe.add_time_viewed(cv_path, df_cand, 'click_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path, df_cand, 'click_predictions')\n",
    "df_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\n",
    "df_cand = add_matrix_calculations(cv_matrix_path, df_cand)\n",
    "df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_cv, df_cand, 'click_predictions')\n",
    "df_cand = add_total_counts(train_path, df_cand)\n",
    "df_cand = otto_common_fe.add_type_last(cv_path, df_cand, 'click_predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1d37976e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:40:47.630926Z",
     "iopub.status.busy": "2023-03-20T17:40:47.630511Z",
     "iopub.status.idle": "2023-03-20T17:41:15.314821Z",
     "shell.execute_reply": "2023-03-20T17:41:15.313654Z"
    },
    "papermill": {
     "duration": 27.71981,
     "end_time": "2023-03-20T17:41:15.317487",
     "exception": false,
     "start_time": "2023-03-20T17:40:47.597677",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.0 GB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check file size and export to file.\n",
    "size = df_cand.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_cand.to_parquet('cv1_features.parquet')\n",
    "\n",
    "del df_cand\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4db30948",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:41:15.449207Z",
     "iopub.status.busy": "2023-03-20T17:41:15.447800Z",
     "iopub.status.idle": "2023-03-20T17:41:15.458572Z",
     "shell.execute_reply": "2023-03-20T17:41:15.457339Z"
    },
    "papermill": {
     "duration": 0.120045,
     "end_time": "2023-03-20T17:41:15.461857",
     "exception": false,
     "start_time": "2023-03-20T17:41:15.341812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n%%time\\n# this sell was used to see feature values for ground truth aids that were NOT selected during candidate generation\\n\\ndf_cand = pd.read_parquet(cand_path)\\ndf_cand = cand_read_and_reduce_test_not_cands(cand_path, True)\\ndf_cand = add_history_experiment_features_pl(cv_path, experiment_matrix_path, df_cand)\\ndf_cand = otto_common_fe.add_history_aid_features(cv_path, df_cand, 'click_predictions')\\ndf_cand = otto_common_fe.add_history_agg_features(cv_path, df_cand, False)\\ndf_cand = otto_common_fe.add_time_viewed(cv_path, df_cand, 'click_predictions')\\ndf_cand = otto_common_fe.add_daily_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\\ndf_cand = add_daily_averages_same_day(counts_during_path, df_cand)\\ndf_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\\ndf_cand = add_matrix_calculations(cv_matrix_path, df_cand)\\ndf_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_cv, df_cand, 'click_predictions')\\ndf_cand = add_total_counts(train_path, df_cand)\\ndf_cand = otto_common_fe.add_type_last(cv_path, df_cand, 'click_predictions')\\n\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "# this sell was used to see feature values for ground truth aids that were NOT selected during candidate generation\n",
    "\n",
    "df_cand = pd.read_parquet(cand_path)\n",
    "df_cand = cand_read_and_reduce_test_not_cands(cand_path, True)\n",
    "df_cand = add_history_experiment_features_pl(cv_path, experiment_matrix_path, df_cand)\n",
    "df_cand = otto_common_fe.add_history_aid_features(cv_path, df_cand, 'click_predictions')\n",
    "df_cand = otto_common_fe.add_history_agg_features(cv_path, df_cand, False)\n",
    "df_cand = otto_common_fe.add_time_viewed(cv_path, df_cand, 'click_predictions')\n",
    "df_cand = otto_common_fe.add_daily_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\n",
    "df_cand = add_daily_averages_same_day(counts_during_path, df_cand)\n",
    "df_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\n",
    "df_cand = add_matrix_calculations(cv_matrix_path, df_cand)\n",
    "df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_cv, df_cand, 'click_predictions')\n",
    "df_cand = add_total_counts(train_path, df_cand)\n",
    "df_cand = otto_common_fe.add_type_last(cv_path, df_cand, 'click_predictions')\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c7a4b6e",
   "metadata": {
    "papermill": {
     "duration": 0.00881,
     "end_time": "2023-03-20T17:41:15.559808",
     "exception": false,
     "start_time": "2023-03-20T17:41:15.550998",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature engineering for the test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6c9642b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:41:15.759677Z",
     "iopub.status.busy": "2023-03-20T17:41:15.758731Z",
     "iopub.status.idle": "2023-03-20T17:41:15.768187Z",
     "shell.execute_reply": "2023-03-20T17:41:15.766787Z"
    },
    "papermill": {
     "duration": 0.117118,
     "end_time": "2023-03-20T17:41:15.771838",
     "exception": false,
     "start_time": "2023-03-20T17:41:15.654720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Input paths for the test dataset.\n",
    "cand_test_path = '/kaggle/input/otto-click-candidates-validation/candidates_test.parquet'\n",
    "test_path = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "test_matrix_path = '/kaggle/input/create-counts-for-clicks/matrix_exact_next_counts_full.parquet'\n",
    "experiment_matrix_path = '/kaggle/input/otto-experiment-prepare-candidates-clicks/matrix_experimental_test.parquet'\n",
    "matrix_path_test = '/kaggle/input/otto-prepare-candidates-clicks/regular_click2click_matrix_test.parquet'\n",
    "return_rate_path_test = '/kaggle/input/create-counts-for-clicks/return_rate_test.parquet'\n",
    "median_time_viewed_test = '/kaggle/input/create-counts-for-clicks/time_viewed_agg_test.parquet'\n",
    "\n",
    "counts_before_path = '/kaggle/input/create-counts-for-clicks/daily_counts_test_full.parquet'\n",
    "counts_during_path = '/kaggle/input/create-counts-for-clicks/daily_counts_test_trunked.parquet'\n",
    "train_full_path = '/kaggle/input/otto-prepare-cv/train_full.parquet'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3193674c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T17:41:15.981098Z",
     "iopub.status.busy": "2023-03-20T17:41:15.980533Z",
     "iopub.status.idle": "2023-03-20T19:01:37.288271Z",
     "shell.execute_reply": "2023-03-20T19:01:37.286917Z"
    },
    "papermill": {
     "duration": 4821.424947,
     "end_time": "2023-03-20T19:01:37.292700",
     "exception": false,
     "start_time": "2023-03-20T17:41:15.867753",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start loading matrice\n",
      "success loading matrice\n",
      "wgt_matrix\n",
      "0\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "1\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "2\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "3\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "4\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "start normalizing\n",
      "wgt_exp\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "start loading matrice\n",
      "success loading matrice\n",
      "wgt_matrix\n",
      "0\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "1\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "2\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "3\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "4\n",
      "j = 0\n",
      "j = 1\n",
      "j = 2\n",
      "j = 3\n",
      "j = 4\n",
      "start normalizing\n",
      "wgt_exp\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "# Split test dataset into 2 chunks and build features for each chunk.\n",
    "n_splits=2\n",
    "\n",
    "for i in range(n_splits):\n",
    "    df = pd.read_parquet(cand_test_path)\n",
    "    df_cand = otto_common.divide_df_by_column(df, n_splits, i, 'session')\n",
    "    del df\n",
    "    gc.collect()\n",
    "    df_cand = otto_common_fe.cand_read_and_reduce(df_cand, 'click', False)\n",
    "    df_cand = add_matrix_weight_pl(test_path, matrix_path_test, df_cand)\n",
    "    df_cand = add_history_experiment_features_pl(test_path, experiment_matrix_path, df_cand)\n",
    "    df_cand = otto_common_fe.add_history_aid_features(test_path, df_cand, 'click_predictions')\n",
    "    df_cand = otto_common_fe.add_history_agg_features(test_path, df_cand, False)\n",
    "    df_cand = otto_common_fe.add_time_viewed(test_path, df_cand, 'click_predictions')\n",
    "    df_cand = otto_common_fe.add_daily_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\n",
    "    df_cand = otto_common_fe.add_daily_averages_same_day(counts_during_path, df_cand, 'click_predictions')\n",
    "    df_cand = otto_common_fe.add_weekly_averages(counts_before_path,counts_during_path, df_cand, 'click_predictions')\n",
    "    df_cand = add_matrix_calculations(test_matrix_path, df_cand)\n",
    "    df_cand = otto_common_fe.add_median_time_viewed(median_time_viewed_test, df_cand, 'click_predictions')\n",
    "    df_cand = add_total_counts(train_full_path, df_cand)\n",
    "    df_cand = otto_common_fe.add_type_last(test_path, df_cand, 'click_predictions')\n",
    "    gc.collect()\n",
    "    string_i = 'test_features_cart_part_' + str(i) + '.parquet'\n",
    "    df_cand.to_parquet(string_i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3ddf4c56",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-20T19:01:37.327137Z",
     "iopub.status.busy": "2023-03-20T19:01:37.325348Z",
     "iopub.status.idle": "2023-03-20T19:01:37.336828Z",
     "shell.execute_reply": "2023-03-20T19:01:37.335527Z"
    },
    "papermill": {
     "duration": 0.032083,
     "end_time": "2023-03-20T19:01:37.339322",
     "exception": false,
     "start_time": "2023-03-20T19:01:37.307239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.1 GB\n"
     ]
    }
   ],
   "source": [
    "size = df_cand.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 7247.467719,
   "end_time": "2023-03-20T19:01:39.609956",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-20T17:00:52.142237",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}