otto-click2click-experiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b31ae90b",
   "metadata": {
    "papermill": {
     "duration": 0.003676,
     "end_time": "2023-03-17T22:01:02.250144",
     "exception": false,
     "start_time": "2023-03-17T22:01:02.246468",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# \"Experimental\" click2click co-visitation matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0731860",
   "metadata": {
    "papermill": {
     "duration": 0.002571,
     "end_time": "2023-03-17T22:01:02.255708",
     "exception": false,
     "start_time": "2023-03-17T22:01:02.253137",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "This notebook prepares one of click-to-click co-visitation matrixes. It is used to calculate features for the click model.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "01ea4793",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-17T22:01:02.263073Z",
     "iopub.status.busy": "2023-03-17T22:01:02.262657Z",
     "iopub.status.idle": "2023-03-17T22:01:02.293778Z",
     "shell.execute_reply": "2023-03-17T22:01:02.292777Z"
    },
    "papermill": {
     "duration": 0.037654,
     "end_time": "2023-03-17T22:01:02.296123",
     "exception": false,
     "start_time": "2023-03-17T22:01:02.258469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "  \n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "553df1d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:01:02.304112Z",
     "iopub.status.busy": "2023-03-17T22:01:02.302937Z",
     "iopub.status.idle": "2023-03-17T22:01:02.313896Z",
     "shell.execute_reply": "2023-03-17T22:01:02.313284Z"
    },
    "papermill": {
     "duration": 0.016569,
     "end_time": "2023-03-17T22:01:02.315765",
     "exception": false,
     "start_time": "2023-03-17T22:01:02.299196",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CalculateCovisitationMatrixExperiment(otto_common.CalculateCovisitationMatrix):\n",
    "    '''\n",
    "    Child class with logic specific to this type of covisitation matrix.\n",
    "    It counts 20 aids after each aid if time between those events was less than 5 minutes.\n",
    "    Weight value is calculated in a way that makes later events more important.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, *args, timebase, **kwargs):\n",
    "        self.timebase = timebase\n",
    "        super().__init__(*args, **kwargs)\n",
    "\n",
    "    def calculate_weights(self, df_i):\n",
    "        df_i = df_i.sort_values(['session','ts'],ascending=[True,False])\n",
    "        df_i = df_i.reset_index(drop=True)\n",
    "        df_i['n'] = df_i.groupby('session').cumcount()\n",
    "        for j in range(1,20):\n",
    "            df_i2 = df_i.copy()\n",
    "            df_i2['n'] = df_i2['n'] - j\n",
    "            df_j = df_i.merge(df_i2,how='inner',on=['session', 'n'])\n",
    "            if j > 2:\n",
    "                df_j = df_j.loc[(df_j.ts_x - df_j.ts_y).abs() < 5 * 60]\n",
    "            df_j = df_j[['session', 'aid_x', 'aid_y','ts_x']].drop_duplicates(['session', 'aid_x', 'aid_y']) \n",
    "            df_j['wgt'] = (1 + 3*(df_j.ts_x - self.timebase)/3024000) / j   \n",
    "            df_j = df_j[['aid_x','aid_y','wgt']]\n",
    "            if j == 1:\n",
    "                df_j_all = df_j\n",
    "            else:\n",
    "                df_j_all = pd.concat([df_j_all, df_j], axis=0)\n",
    "        del df_i, df_j, df_i2\n",
    "        gc.collect()\n",
    "        df_j_all = df_j_all.loc[df_j_all['aid_x'].isin(self.aid_list)]\n",
    "        df_j_all.wgt = df_j_all.wgt.astype('float32')\n",
    "        df_j_all = self.groupby_reset_and_reduce(df_j_all)\n",
    "        return df_j_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c549ff5",
   "metadata": {
    "papermill": {
     "duration": 0.002637,
     "end_time": "2023-03-17T22:01:02.321419",
     "exception": false,
     "start_time": "2023-03-17T22:01:02.318782",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Co-visitation matrix for cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b06551aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:01:02.329394Z",
     "iopub.status.busy": "2023-03-17T22:01:02.328034Z",
     "iopub.status.idle": "2023-03-17T22:01:08.423737Z",
     "shell.execute_reply": "2023-03-17T22:01:08.422343Z"
    },
    "papermill": {
     "duration": 6.102047,
     "end_time": "2023-03-17T22:01:08.426191",
     "exception": false,
     "start_time": "2023-03-17T22:01:02.324144",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Make a list of AIDs, that appear in cross-validation sets.\n",
    "# aid_x is filtered to only include aids from the list, while aid_y is not filtered\n",
    "# This reduces size of resulting matrix.\n",
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "trunked_sessions_path2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "cv_list = otto_common.build_aid_list(trunked_sessions_path, trunked_sessions_path2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67db42e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:01:08.434030Z",
     "iopub.status.busy": "2023-03-17T22:01:08.433647Z",
     "iopub.status.idle": "2023-03-17T22:43:43.530054Z",
     "shell.execute_reply": "2023-03-17T22:43:43.528562Z"
    },
    "papermill": {
     "duration": 2555.103226,
     "end_time": "2023-03-17T22:43:43.532504",
     "exception": false,
     "start_time": "2023-03-17T22:01:08.429278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n"
     ]
    }
   ],
   "source": [
    "# Input parameters and matrix calculation itself.\n",
    "click_data_cv = '/kaggle/input/otto-prepare-cv/cv_train.parquet'\n",
    "n_splits = 150\n",
    "n_splits_concat = 20\n",
    "timebase_cv = 1658700000\n",
    "covisitation_cv = CalculateCovisitationMatrixExperiment(n_splits, n_splits_concat, cv_list, timebase=timebase_cv)\n",
    "df_matrix_cv = covisitation_cv.generate_covisitation_matrix(click_data_cv)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e7e4f8a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:43:43.556847Z",
     "iopub.status.busy": "2023-03-17T22:43:43.556444Z",
     "iopub.status.idle": "2023-03-17T22:43:51.264561Z",
     "shell.execute_reply": "2023-03-17T22:43:51.263734Z"
    },
    "papermill": {
     "duration": 7.722966,
     "end_time": "2023-03-17T22:43:51.266626",
     "exception": false,
     "start_time": "2023-03-17T22:43:43.543660",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.2 GB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check dataframe size\n",
    "size = df_matrix_cv.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_matrix_cv.to_parquet('matrix_experimental_cv.parquet')\n",
    "\n",
    "del df_matrix_cv, covisitation_cv, cv_list\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6689a4e",
   "metadata": {
    "papermill": {
     "duration": 0.010891,
     "end_time": "2023-03-17T22:43:51.289465",
     "exception": false,
     "start_time": "2023-03-17T22:43:51.278574",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Co-visitation matrix for test dataset\n",
    "The same steps to create a co-visitation matrix for the test dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9e6a08d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:43:51.313148Z",
     "iopub.status.busy": "2023-03-17T22:43:51.312762Z",
     "iopub.status.idle": "2023-03-17T22:43:54.169473Z",
     "shell.execute_reply": "2023-03-17T22:43:54.168394Z"
    },
    "papermill": {
     "duration": 2.871655,
     "end_time": "2023-03-17T22:43:54.172057",
     "exception": false,
     "start_time": "2023-03-17T22:43:51.300402",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "test_list = otto_common.build_aid_list(trunked_sessions_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "646c8827",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:43:54.196675Z",
     "iopub.status.busy": "2023-03-17T22:43:54.196308Z",
     "iopub.status.idle": "2023-03-17T23:47:55.917206Z",
     "shell.execute_reply": "2023-03-17T23:47:55.916165Z"
    },
    "papermill": {
     "duration": 3841.736512,
     "end_time": "2023-03-17T23:47:55.919840",
     "exception": false,
     "start_time": "2023-03-17T22:43:54.183328",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n"
     ]
    }
   ],
   "source": [
    "click_data_test = '/kaggle/input/otto-prepare-cv/train_full.parquet'\n",
    "n_splits = 200\n",
    "n_splits_concat = 25\n",
    "timebase_test = 1659304800\n",
    "covisitation_test = CalculateCovisitationMatrixExperiment(n_splits, n_splits_concat, test_list, timebase=timebase_test)\n",
    "df_matrix_test = covisitation_test.generate_covisitation_matrix(click_data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "19f92f15",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T23:47:55.969992Z",
     "iopub.status.busy": "2023-03-17T23:47:55.969456Z",
     "iopub.status.idle": "2023-03-17T23:48:06.792470Z",
     "shell.execute_reply": "2023-03-17T23:48:06.791017Z"
    },
    "papermill": {
     "duration": 10.853973,
     "end_time": "2023-03-17T23:48:06.795743",
     "exception": false,
     "start_time": "2023-03-17T23:47:55.941770",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.9 GB\n"
     ]
    }
   ],
   "source": [
    "# Check dataframe size\n",
    "size = df_matrix_test.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "\n",
    "df_matrix_test.to_parquet('matrix_experimental_test.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 6434.072611,
   "end_time": "2023-03-17T23:48:07.669885",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-17T22:00:53.597274",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}