tabular-with-image-submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "378426fd",
   "metadata": {
    "papermill": {
     "duration": 0.024272,
     "end_time": "2024-09-01T16:53:44.536925",
     "exception": false,
     "start_time": "2024-09-01T16:53:44.512653",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# IMPORT LIBRARIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9a968665",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:53:44.585501Z",
     "iopub.status.busy": "2024-09-01T16:53:44.585182Z",
     "iopub.status.idle": "2024-09-01T16:53:50.691972Z",
     "shell.execute_reply": "2024-09-01T16:53:50.691032Z"
    },
    "papermill": {
     "duration": 6.133883,
     "end_time": "2024-09-01T16:53:50.694343",
     "exception": false,
     "start_time": "2024-09-01T16:53:44.560460",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.model_selection import GroupKFold\n",
    "\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.pipeline import Pipeline\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "\n",
    "import lightgbm as lgb\n",
    "import catboost as cb\n",
    "import xgboost as xgb\n",
    "\n",
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "\n",
    "#downsampling techniques\n",
    "# they took long time, so we use RandomUnderSampler\n",
    "from imblearn.under_sampling import NearMiss\n",
    "from imblearn.under_sampling import ClusterCentroids\n",
    "from imblearn.under_sampling import TomekLinks\n",
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "from imblearn.pipeline import Pipeline as ImbPipeline\n",
    "from imblearn.under_sampling import NearMiss, TomekLinks\n",
    "from sklearn.impute import SimpleImputer\n",
    "from dataclasses import dataclass\n",
    "\n",
    "import time\n",
    "from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif, VarianceThreshold\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3182f847",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:53:50.743746Z",
     "iopub.status.busy": "2024-09-01T16:53:50.743170Z",
     "iopub.status.idle": "2024-09-01T16:53:50.748205Z",
     "shell.execute_reply": "2024-09-01T16:53:50.747376Z"
    },
    "papermill": {
     "duration": 0.03165,
     "end_time": "2024-09-01T16:53:50.750033",
     "exception": false,
     "start_time": "2024-09-01T16:53:50.718383",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class CFG_GLOBAL:\n",
    "    # 見落とし注意！\n",
    "    num_folds = 5\n",
    "    train_folds = [0, 1, 2, 3, 4]\n",
    "\n",
    "cfg_global = CFG_GLOBAL()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f28c526",
   "metadata": {
    "papermill": {
     "duration": 0.023478,
     "end_time": "2024-09-01T16:53:51.688564",
     "exception": false,
     "start_time": "2024-09-01T16:53:51.665086",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Image model 20240827234748"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a911e9aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:53:51.737061Z",
     "iopub.status.busy": "2024-09-01T16:53:51.736818Z",
     "iopub.status.idle": "2024-09-01T16:54:10.933354Z",
     "shell.execute_reply": "2024-09-01T16:54:10.932222Z"
    },
    "papermill": {
     "duration": 19.223512,
     "end_time": "2024-09-01T16:54:10.935626",
     "exception": false,
     "start_time": "2024-09-01T16:53:51.712114",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n",
      "This is convnextv2 family\n",
      "This is convnextv2 family\n",
      "This is convnextv2 family\n",
      "This is convnextv2 family\n",
      "This is convnextv2 family\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Predicting: 100%|██████████| 1/1 [00:01<00:00,  1.16s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ISIC_0015657</th>\n",
       "      <td>-0.501967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015729</th>\n",
       "      <td>-4.002757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015740</th>\n",
       "      <td>-3.307847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target\n",
       "isic_id               \n",
       "ISIC_0015657 -0.501967\n",
       "ISIC_0015729 -4.002757\n",
       "ISIC_0015740 -3.307847"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from pathlib import Path\n",
    "from dataclasses import dataclass\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class CFG:\n",
    "    # 見落とし注意！\n",
    "    run_id = \"20240827234748\"\n",
    "    model_name = \"convnextv2_nano.fcmae_ft_in22k_in1k\"\n",
    "    model_path = Path(f\"/kaggle/input/output-{run_id}\")\n",
    "    img_size = 224 #384 or 224\n",
    "    folds = [0,1,2,3,4]\n",
    "\n",
    "    # auto\n",
    "    oof_path = model_path / \"oof_predictions.csv\"\n",
    "\n",
    "    # --- desctiption\n",
    "#         \"cv\": 0.1599389,\n",
    "#         \"lb\": ,\n",
    "#         \"version\": \"\",\n",
    "#         \"config\": \"convnextv2_nano.fcmae_ft_in22k_in1k\"\",\n",
    "#         \"date\": 20240827234748,\n",
    "#         \"desc\": \"convnextv2_nano.fcmae_ft_in22k_in1k + lr0.0001 + warmup + train1:1-val1:10 + upsample-2 + AugumentMore + Dropout + CustomHead\",\n",
    "    # ---------------\n",
    "\n",
    "cfg = CFG()\n",
    "\n",
    "test_path = Path('/kaggle/input/isic-2024-challenge/test-metadata.csv')\n",
    "subm_path  = Path('/kaggle/input/isic-2024-challenge/sample_submission.csv')\n",
    "\n",
    "id_col = 'isic_id'\n",
    "\n",
    "def read_data(path, cfg):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .to_pandas()\n",
    "        .set_index(id_col)\n",
    "    )\n",
    "\n",
    "df_test = read_data(test_path, cfg)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "# === ImageNet inference code\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import h5py\n",
    "import timm\n",
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "import io\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "\n",
    "\n",
    "class ISICDataset(Dataset):\n",
    "    def __init__(self, hdf5_file, isic_ids, targets=None, transform=None):\n",
    "        self.hdf5_file = h5py.File(hdf5_file, 'r')  # Keep file open\n",
    "        self.isic_ids = isic_ids\n",
    "        self.targets = targets\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.isic_ids)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img_bytes = self.hdf5_file[self.isic_ids[idx]][()]\n",
    "        img = Image.open(io.BytesIO(img_bytes))\n",
    "        img = np.array(img)\n",
    "\n",
    "        if self.transform:\n",
    "            transformed = self.transform(image=img)\n",
    "            img = transformed['image']\n",
    "\n",
    "        target = self.targets[idx] if self.targets is not None else torch.tensor(-1)\n",
    "        return img, target\n",
    "\n",
    "    def __del__(self):\n",
    "        self.hdf5_file.close()  # Ensure file is closed when object is destroyed\n",
    "\n",
    "# Define the albumentations transformation\n",
    "base_transform = A.Compose([\n",
    "    A.Resize(cfg.img_size, cfg.img_size),\n",
    "    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "    ToTensorV2(),\n",
    "])\n",
    "\n",
    "class GeM(nn.Module):\n",
    "    def __init__(self, p=3, eps=1e-6):\n",
    "        super(GeM, self).__init__()\n",
    "        self.p = nn.Parameter(torch.ones(1) * p)\n",
    "        self.eps = eps\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gem(x, p=self.p, eps=self.eps)\n",
    "\n",
    "    def gem(self, x, p=3, eps=1e-6):\n",
    "        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return (\n",
    "            self.__class__.__name__\n",
    "            + \"(\"\n",
    "            + \"p=\"\n",
    "            + \"{:.4f}\".format(self.p.data.tolist()[0])\n",
    "            + \", \"\n",
    "            + \"eps=\"\n",
    "            + str(self.eps)\n",
    "            + \")\"\n",
    "        )\n",
    "\n",
    "class ISICModel(nn.Module):\n",
    "    def __init__(self, model_name, num_classes=1, pretrained=False, checkpoint_path=None):\n",
    "        super(ISICModel, self).__init__()\n",
    "        self.model_name = model_name\n",
    "        self.model = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n",
    "        self.model_org = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n",
    "        self.model_org.classifier = nn.Identity()\n",
    "        self.pooling = GeM()\n",
    "\n",
    "        if \"eva02\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "        elif \"efficientnetv2\" in self.model_name:\n",
    "            in_features = self.model.classifier.in_features\n",
    "            self.model.classifier = nn.Identity()\n",
    "            if self.pooling:  # My custom pooling\n",
    "                self.model.global_pool = nn.Identity()\n",
    "        elif \"convnextv2\" in self.model_name:\n",
    "            print(\"This is convnextv2 family\")\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.head = nn.Sequential(\n",
    "                nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(640, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "                nn.Dropout(0.5),  # ドロップアウト\n",
    "                nn.Linear(256, num_classes),  # 出力層（2クラス分類）\n",
    "            )\n",
    "        self.linear = nn.Linear(in_features, num_classes)\n",
    "        self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "\n",
    "    def forward(self, images):\n",
    "        features = self.model(images)\n",
    "\n",
    "        if \"efficientnetv2\" in self.model_name:\n",
    "            # Custom poolingがある場合\n",
    "            if self.pooling:\n",
    "                features = self.pooling(features).flatten(1)\n",
    "\n",
    "        if \"convnextv2\" in self.model_name:\n",
    "            output = self.head(features)\n",
    "        else:\n",
    "            for i, dropout in enumerate(self.dropouts):\n",
    "                if i == 0:\n",
    "                    output = self.linear(dropout(features))\n",
    "                else:\n",
    "                    output += self.linear(dropout(features))\n",
    "            output /= len(self.dropouts)\n",
    "\n",
    "        return output.squeeze()\n",
    "\n",
    "\n",
    "\n",
    "def get_latest_epoch_file(folder_path, target_fold):\n",
    "    # 正規表現パターン：fold_X_epoch_Y_score_Z.pth の形式に一致\n",
    "    pattern = re.compile(r\"fold_(\\d+)_epoch_(\\d+)_score_(\\d+\\.\\d+)\\.pth\") # 新パターン\n",
    "#     pattern = re.compile(r\"model_fold_(\\d+)_epoch_(\\d+)\\.pth\") # 旧パターン\n",
    "\n",
    "    max_epoch = -1\n",
    "    latest_file = None\n",
    "\n",
    "    for filename in os.listdir(folder_path):\n",
    "        match = pattern.match(filename)\n",
    "        if match:\n",
    "            fold, epoch, score = match.groups() # 新パターン\n",
    "#             fold, epoch = match.groups() # 旧パターン\n",
    "            fold = int(fold)\n",
    "            epoch = int(epoch)\n",
    "\n",
    "            if fold == target_fold and epoch > max_epoch:\n",
    "                max_epoch = epoch\n",
    "                latest_file = filename\n",
    "\n",
    "    if latest_file:\n",
    "        return os.path.join(folder_path, latest_file)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def load_models(folds, device):\n",
    "    models = []\n",
    "    for fold in folds:\n",
    "        model = ISICModel(cfg.model_name)\n",
    "        model.to(device)\n",
    "        model_w_path = get_latest_epoch_file(cfg.model_path, fold)\n",
    "        model.load_state_dict(torch.load(model_w_path, map_location=device))\n",
    "        model.eval()\n",
    "        models.append(model)\n",
    "    return models\n",
    "\n",
    "\n",
    "@torch.no_grad()  # Apply no_grad to the entire function\n",
    "def ensemble_predict(models, test_loader, device):\n",
    "    all_predictions = []\n",
    "    for inputs, _ in tqdm(test_loader, desc=\"Predicting\"):\n",
    "        inputs = inputs.to(device)\n",
    "#         fold_predictions = torch.stack([model(inputs).softmax(dim=1)[:, 1] for model in models])\n",
    "        fold_predictions = torch.stack([model(inputs) for model in models])\n",
    "        avg_predictions = fold_predictions.mean(dim=0)\n",
    "        all_predictions.extend(avg_predictions.cpu().numpy())\n",
    "    return all_predictions\n",
    "\n",
    "\n",
    "# === Do ImageNet inference on test data / merge df_test\n",
    "TEST_HDF5_FILE_PATH = '/kaggle/input/isic-2024-challenge/test-image.hdf5'\n",
    "\n",
    "# Set up CUDA if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "# folds to use for pred\n",
    "folds = cfg.folds\n",
    "\n",
    "models = load_models(folds, device)\n",
    "\n",
    "# Prepare your test dataset\n",
    "test_dataset = ISICDataset(\n",
    "    hdf5_file=TEST_HDF5_FILE_PATH,\n",
    "    isic_ids=df_test.index.values,  #minor change here from\n",
    "    transform=base_transform,\n",
    ")\n",
    "\n",
    "# Create test data loader\n",
    "test_loader = DataLoader(test_dataset, batch_size=512, shuffle=False, num_workers=4, pin_memory=True)\n",
    "\n",
    "# Run predictions\n",
    "predictions = ensemble_predict(models, test_loader, device)\n",
    "\n",
    "# Create a new DataFrame with predictions\n",
    "temp_df = pd.DataFrame({\"image_predict\": predictions}, index=df_test.index)\n",
    "\n",
    "# Join the predictions to df_test\n",
    "df_test = df_test.join(temp_df)\n",
    "\n",
    "df_subm[\"target\"] = df_test[\"image_predict\"]\n",
    "\n",
    "df_subm.to_csv(f'submission_{cfg.run_id}.csv')\n",
    "df_subm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08484fc8",
   "metadata": {
    "papermill": {
     "duration": 0.024514,
     "end_time": "2024-09-01T16:54:10.985594",
     "exception": false,
     "start_time": "2024-09-01T16:54:10.961080",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Image model 20240830205516"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5d22cc85",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:11.036802Z",
     "iopub.status.busy": "2024-09-01T16:54:11.036489Z",
     "iopub.status.idle": "2024-09-01T16:54:13.623608Z",
     "shell.execute_reply": "2024-09-01T16:54:13.622523Z"
    },
    "papermill": {
     "duration": 2.615607,
     "end_time": "2024-09-01T16:54:13.625671",
     "exception": false,
     "start_time": "2024-09-01T16:54:11.010064",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Predicting: 100%|██████████| 1/1 [00:00<00:00,  4.64it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ISIC_0015657</th>\n",
       "      <td>-2.714859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015729</th>\n",
       "      <td>-5.223221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015740</th>\n",
       "      <td>-4.123434</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target\n",
       "isic_id               \n",
       "ISIC_0015657 -2.714859\n",
       "ISIC_0015729 -5.223221\n",
       "ISIC_0015740 -4.123434"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from pathlib import Path\n",
    "from dataclasses import dataclass\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class CFG:\n",
    "    # 見落とし注意！\n",
    "    run_id = \"20240830205516\"\n",
    "    model_name = \"vit_tiny_patch16_224.augreg_in21k_ft_in1k\"\n",
    "    model_path = Path(f\"/kaggle/input/output-{run_id}\")\n",
    "    img_size = 224 #384 or 224\n",
    "    folds = [0,1,2,3,4]\n",
    "\n",
    "    # auto\n",
    "    oof_path = model_path / \"oof_predictions.csv\"\n",
    "\n",
    "    # --- desctiption\n",
    "        # \"cv\": 0.1612504,\n",
    "        # \"lb\": ,\n",
    "        # \"version\": \"\",\n",
    "        # \"config\": \"vit_tiny_patch16_224.augreg_in21k_ft_in1k\",\n",
    "        # \"date\": 20240830205516,\n",
    "        # \"desc\": \"lr1e-4 + warmup + train1:1-val1:10 + upsample-2 + AugumentMore + Dropoutx1 + CustomHead + weight_decay1e-3\",\n",
    "    # ---------------\n",
    "\n",
    "cfg = CFG()\n",
    "\n",
    "test_path = Path('/kaggle/input/isic-2024-challenge/test-metadata.csv')\n",
    "subm_path  = Path('/kaggle/input/isic-2024-challenge/sample_submission.csv')\n",
    "\n",
    "id_col = 'isic_id'\n",
    "\n",
    "def read_data(path, cfg):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .to_pandas()\n",
    "        .set_index(id_col)\n",
    "    )\n",
    "\n",
    "df_test = read_data(test_path, cfg)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "# === ImageNet inference code\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import h5py\n",
    "import timm\n",
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "import io\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "\n",
    "\n",
    "class ISICDataset(Dataset):\n",
    "    def __init__(self, hdf5_file, isic_ids, targets=None, transform=None):\n",
    "        self.hdf5_file = h5py.File(hdf5_file, 'r')  # Keep file open\n",
    "        self.isic_ids = isic_ids\n",
    "        self.targets = targets\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.isic_ids)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img_bytes = self.hdf5_file[self.isic_ids[idx]][()]\n",
    "        img = Image.open(io.BytesIO(img_bytes))\n",
    "        img = np.array(img)\n",
    "\n",
    "        if self.transform:\n",
    "            transformed = self.transform(image=img)\n",
    "            img = transformed['image']\n",
    "\n",
    "        target = self.targets[idx] if self.targets is not None else torch.tensor(-1)\n",
    "        return img, target\n",
    "\n",
    "    def __del__(self):\n",
    "        self.hdf5_file.close()  # Ensure file is closed when object is destroyed\n",
    "\n",
    "# Define the albumentations transformation\n",
    "base_transform = A.Compose([\n",
    "    A.Resize(cfg.img_size, cfg.img_size),\n",
    "    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "    ToTensorV2(),\n",
    "])\n",
    "\n",
    "class GeM(nn.Module):\n",
    "    def __init__(self, p=3, eps=1e-6):\n",
    "        super(GeM, self).__init__()\n",
    "        self.p = nn.Parameter(torch.ones(1) * p)\n",
    "        self.eps = eps\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gem(x, p=self.p, eps=self.eps)\n",
    "\n",
    "    def gem(self, x, p=3, eps=1e-6):\n",
    "        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return (\n",
    "            self.__class__.__name__\n",
    "            + \"(\"\n",
    "            + \"p=\"\n",
    "            + \"{:.4f}\".format(self.p.data.tolist()[0])\n",
    "            + \", \"\n",
    "            + \"eps=\"\n",
    "            + str(self.eps)\n",
    "            + \")\"\n",
    "        )\n",
    "\n",
    "class ISICModel(nn.Module):\n",
    "    def __init__(self, model_name, num_classes=1, pretrained=False, checkpoint_path=None):\n",
    "        super(ISICModel, self).__init__()\n",
    "        self.model_name = model_name\n",
    "        self.model = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n",
    "\n",
    "        if \"eva02\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.linear = nn.Linear(in_features, num_classes)\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "        elif \"efficientnetv2\" in self.model_name:\n",
    "            in_features = self.model.classifier.in_features\n",
    "            self.model.classifier = nn.Identity()\n",
    "            self.pooling = GeM()\n",
    "            if self.pooling:  # My custom pooling\n",
    "                self.model.global_pool = nn.Identity()\n",
    "            self.linear = nn.Linear(in_features, num_classes)\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "        elif \"convnextv2\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.head = nn.Sequential(\n",
    "                nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "                nn.Dropout(0.5),  # ドロップアウト\n",
    "                nn.Linear(256, num_classes),  # 出力層（2クラス分類）\n",
    "            )\n",
    "        elif \"swinv2\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                # nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                GeM(),\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(256, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"vit_tiny\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.norm = nn.Identity()\n",
    "            self.model.fc_norm = nn.Identity()\n",
    "            self.model.head_drop = nn.Identity()\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 64),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(1)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(64, num_classes)  # 出力層（2クラス分類）\n",
    "\n",
    "    def forward(self, images):\n",
    "        features = self.model(images)\n",
    "\n",
    "        if any(\n",
    "            [\n",
    "                \"efficientnetv2\" in self.model_name,\n",
    "                \"eva02\" in self.model_name,\n",
    "            ]\n",
    "        ):\n",
    "            # Custom poolingがある場合\n",
    "            if self.pooling:\n",
    "                features = self.pooling(features).flatten(1)\n",
    "            for i, dropout in enumerate(self.dropouts):\n",
    "                if i == 0:\n",
    "                    output = self.linear(dropout(features))\n",
    "                else:\n",
    "                    output += self.linear(dropout(features))\n",
    "            output /= len(self.dropouts)\n",
    "\n",
    "        if \"convnextv2\" in self.model_name:\n",
    "            output = self.head(features)\n",
    "\n",
    "        if \"swinv2\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"vit_tiny\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        return output.squeeze()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def get_latest_epoch_file(folder_path, target_fold):\n",
    "    # 正規表現パターン：fold_X_epoch_Y_score_Z.pth の形式に一致\n",
    "    pattern = re.compile(r\"fold_(\\d+)_epoch_(\\d+)_score_(\\d+\\.\\d+)\\.pth\") # 新パターン\n",
    "#     pattern = re.compile(r\"model_fold_(\\d+)_epoch_(\\d+)\\.pth\") # 旧パターン\n",
    "\n",
    "    max_epoch = -1\n",
    "    latest_file = None\n",
    "\n",
    "    for filename in os.listdir(folder_path):\n",
    "        match = pattern.match(filename)\n",
    "        if match:\n",
    "            fold, epoch, score = match.groups() # 新パターン\n",
    "#             fold, epoch = match.groups() # 旧パターン\n",
    "            fold = int(fold)\n",
    "            epoch = int(epoch)\n",
    "\n",
    "            if fold == target_fold and epoch > max_epoch:\n",
    "                max_epoch = epoch\n",
    "                latest_file = filename\n",
    "\n",
    "    if latest_file:\n",
    "        return os.path.join(folder_path, latest_file)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def load_models(folds, device):\n",
    "    models = []\n",
    "    for fold in folds:\n",
    "        model = ISICModel(cfg.model_name)\n",
    "        model.to(device)\n",
    "        model_w_path = get_latest_epoch_file(cfg.model_path, fold)\n",
    "        model.load_state_dict(torch.load(model_w_path, map_location=device))\n",
    "        model.eval()\n",
    "        models.append(model)\n",
    "    return models\n",
    "\n",
    "\n",
    "@torch.no_grad()  # Apply no_grad to the entire function\n",
    "def ensemble_predict(models, test_loader, device):\n",
    "    all_predictions = []\n",
    "    for inputs, _ in tqdm(test_loader, desc=\"Predicting\"):\n",
    "        inputs = inputs.to(device)\n",
    "#         fold_predictions = torch.stack([model(inputs).softmax(dim=1)[:, 1] for model in models])\n",
    "        fold_predictions = torch.stack([model(inputs) for model in models])\n",
    "        avg_predictions = fold_predictions.mean(dim=0)\n",
    "        all_predictions.extend(avg_predictions.cpu().numpy())\n",
    "    return all_predictions\n",
    "\n",
    "\n",
    "# === Do ImageNet inference on test data / merge df_test\n",
    "TEST_HDF5_FILE_PATH = '/kaggle/input/isic-2024-challenge/test-image.hdf5'\n",
    "\n",
    "# Set up CUDA if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "# folds to use for pred\n",
    "folds = cfg.folds\n",
    "\n",
    "models = load_models(folds, device)\n",
    "\n",
    "# Prepare your test dataset\n",
    "test_dataset = ISICDataset(\n",
    "    hdf5_file=TEST_HDF5_FILE_PATH,\n",
    "    isic_ids=df_test.index.values,  #minor change here from\n",
    "    transform=base_transform,\n",
    ")\n",
    "\n",
    "# Create test data loader\n",
    "test_loader = DataLoader(test_dataset, batch_size=512, shuffle=False, num_workers=4, pin_memory=True)\n",
    "\n",
    "# Run predictions\n",
    "predictions = ensemble_predict(models, test_loader, device)\n",
    "\n",
    "# Create a new DataFrame with predictions\n",
    "temp_df = pd.DataFrame({\"image_predict\": predictions}, index=df_test.index)\n",
    "\n",
    "# Join the predictions to df_test\n",
    "df_test = df_test.join(temp_df)\n",
    "\n",
    "df_subm[\"target\"] = df_test[\"image_predict\"]\n",
    "\n",
    "df_subm.to_csv(f'submission_{cfg.run_id}.csv')\n",
    "df_subm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c218bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-30T19:10:19.724754Z",
     "iopub.status.busy": "2024-08-30T19:10:19.724439Z",
     "iopub.status.idle": "2024-08-30T19:10:19.729474Z",
     "shell.execute_reply": "2024-08-30T19:10:19.728591Z",
     "shell.execute_reply.started": "2024-08-30T19:10:19.724728Z"
    },
    "papermill": {
     "duration": 0.025763,
     "end_time": "2024-09-01T16:54:13.678739",
     "exception": false,
     "start_time": "2024-09-01T16:54:13.652976",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Image model 20240831025049"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8bd32820",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:13.730890Z",
     "iopub.status.busy": "2024-09-01T16:54:13.730597Z",
     "iopub.status.idle": "2024-09-01T16:54:16.261875Z",
     "shell.execute_reply": "2024-09-01T16:54:16.260882Z"
    },
    "papermill": {
     "duration": 2.559889,
     "end_time": "2024-09-01T16:54:16.263769",
     "exception": false,
     "start_time": "2024-09-01T16:54:13.703880",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Predicting: 100%|██████████| 1/1 [00:00<00:00,  5.65it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ISIC_0015657</th>\n",
       "      <td>-1.637331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015729</th>\n",
       "      <td>-3.406027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015740</th>\n",
       "      <td>-4.912541</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target\n",
       "isic_id               \n",
       "ISIC_0015657 -1.637331\n",
       "ISIC_0015729 -3.406027\n",
       "ISIC_0015740 -4.912541"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from pathlib import Path\n",
    "from dataclasses import dataclass\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class CFG:\n",
    "    # 見落とし注意！\n",
    "    run_id = \"20240831025049\"\n",
    "    model_name = \"vit_tiny_patch16_224.augreg_in21k_ft_in1k\"\n",
    "    model_path = Path(f\"/kaggle/input/output-{run_id}\")\n",
    "    img_size = 224 #384 or 224\n",
    "    folds = [0,1,2,3,4]\n",
    "\n",
    "    # auto\n",
    "    oof_path = model_path / \"oof_predictions.csv\"\n",
    "\n",
    "    # --- desctiption\n",
    "        # \"cv\":  0.147796,\n",
    "        # \"lb\": ,\n",
    "        # \"version\": \"\",\n",
    "        # \"config\": \"vit_tiny_patch16_224.augreg_in21k_ft_in1k\",\n",
    "        # \"date\": 20240830233240,\n",
    "        # \"desc\": \"lr1e-4 + warmup + train1:1-val1:10 + upsample-2 + AugumentMore + Dropoutx1 + CustomHead + weight_decay1e-3\",\n",
    "        # \"desc\": \"remap case1\",\n",
    "    # ---------------\n",
    "\n",
    "cfg = CFG()\n",
    "\n",
    "test_path = Path('/kaggle/input/isic-2024-challenge/test-metadata.csv')\n",
    "subm_path  = Path('/kaggle/input/isic-2024-challenge/sample_submission.csv')\n",
    "\n",
    "id_col = 'isic_id'\n",
    "\n",
    "def read_data(path, cfg):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .to_pandas()\n",
    "        .set_index(id_col)\n",
    "    )\n",
    "\n",
    "df_test = read_data(test_path, cfg)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "# === ImageNet inference code\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import h5py\n",
    "import timm\n",
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "import io\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "\n",
    "\n",
    "class ISICDataset(Dataset):\n",
    "    def __init__(self, hdf5_file, isic_ids, targets=None, transform=None):\n",
    "        self.hdf5_file = h5py.File(hdf5_file, 'r')  # Keep file open\n",
    "        self.isic_ids = isic_ids\n",
    "        self.targets = targets\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.isic_ids)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img_bytes = self.hdf5_file[self.isic_ids[idx]][()]\n",
    "        img = Image.open(io.BytesIO(img_bytes))\n",
    "        img = np.array(img)\n",
    "\n",
    "        if self.transform:\n",
    "            transformed = self.transform(image=img)\n",
    "            img = transformed['image']\n",
    "\n",
    "        target = self.targets[idx] if self.targets is not None else torch.tensor(-1)\n",
    "        return img, target\n",
    "\n",
    "    def __del__(self):\n",
    "        self.hdf5_file.close()  # Ensure file is closed when object is destroyed\n",
    "\n",
    "# Define the albumentations transformation\n",
    "base_transform = A.Compose([\n",
    "    A.Resize(cfg.img_size, cfg.img_size),\n",
    "    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "    ToTensorV2(),\n",
    "])\n",
    "\n",
    "class GeM(nn.Module):\n",
    "    def __init__(self, p=3, eps=1e-6):\n",
    "        super(GeM, self).__init__()\n",
    "        self.p = nn.Parameter(torch.ones(1) * p)\n",
    "        self.eps = eps\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gem(x, p=self.p, eps=self.eps)\n",
    "\n",
    "    def gem(self, x, p=3, eps=1e-6):\n",
    "        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return (\n",
    "            self.__class__.__name__\n",
    "            + \"(\"\n",
    "            + \"p=\"\n",
    "            + \"{:.4f}\".format(self.p.data.tolist()[0])\n",
    "            + \", \"\n",
    "            + \"eps=\"\n",
    "            + str(self.eps)\n",
    "            + \")\"\n",
    "        )\n",
    "\n",
    "class ISICModel(nn.Module):\n",
    "    def __init__(self, model_name, num_classes=1, pretrained=False, checkpoint_path=None):\n",
    "        super(ISICModel, self).__init__()\n",
    "        self.model_name = model_name\n",
    "        self.model = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n",
    "\n",
    "        if \"eva02\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.linear = nn.Linear(in_features, num_classes)\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "        elif \"efficientnetv2\" in self.model_name:\n",
    "            in_features = self.model.classifier.in_features\n",
    "            self.model.classifier = nn.Identity()\n",
    "            self.pooling = GeM()\n",
    "            if self.pooling:  # My custom pooling\n",
    "                self.model.global_pool = nn.Identity()\n",
    "            self.linear = nn.Linear(in_features, num_classes)\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "        elif \"convnextv2\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.head = nn.Sequential(\n",
    "                nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "                nn.Dropout(0.5),  # ドロップアウト\n",
    "                nn.Linear(256, num_classes),  # 出力層（2クラス分類）\n",
    "            )\n",
    "        elif \"swinv2\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                # nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                GeM(),\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(256, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"vit_tiny\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.norm = nn.Identity()\n",
    "            self.model.fc_norm = nn.Identity()\n",
    "            self.model.head_drop = nn.Identity()\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 64),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(1)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(64, num_classes)  # 出力層（2クラス分類）\n",
    "\n",
    "    def forward(self, images):\n",
    "        features = self.model(images)\n",
    "\n",
    "        if any(\n",
    "            [\n",
    "                \"efficientnetv2\" in self.model_name,\n",
    "                \"eva02\" in self.model_name,\n",
    "            ]\n",
    "        ):\n",
    "            # Custom poolingがある場合\n",
    "            if self.pooling:\n",
    "                features = self.pooling(features).flatten(1)\n",
    "            for i, dropout in enumerate(self.dropouts):\n",
    "                if i == 0:\n",
    "                    output = self.linear(dropout(features))\n",
    "                else:\n",
    "                    output += self.linear(dropout(features))\n",
    "            output /= len(self.dropouts)\n",
    "\n",
    "        if \"convnextv2\" in self.model_name:\n",
    "            output = self.head(features)\n",
    "\n",
    "        if \"swinv2\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"vit_tiny\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        return output.squeeze()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def get_latest_epoch_file(folder_path, target_fold):\n",
    "    # 正規表現パターン：fold_X_epoch_Y_score_Z.pth の形式に一致\n",
    "    pattern = re.compile(r\"fold_(\\d+)_epoch_(\\d+)_score_(\\d+\\.\\d+)\\.pth\") # 新パターン\n",
    "#     pattern = re.compile(r\"model_fold_(\\d+)_epoch_(\\d+)\\.pth\") # 旧パターン\n",
    "\n",
    "    max_epoch = -1\n",
    "    latest_file = None\n",
    "\n",
    "    for filename in os.listdir(folder_path):\n",
    "        match = pattern.match(filename)\n",
    "        if match:\n",
    "            fold, epoch, score = match.groups() # 新パターン\n",
    "#             fold, epoch = match.groups() # 旧パターン\n",
    "            fold = int(fold)\n",
    "            epoch = int(epoch)\n",
    "\n",
    "            if fold == target_fold and epoch > max_epoch:\n",
    "                max_epoch = epoch\n",
    "                latest_file = filename\n",
    "\n",
    "    if latest_file:\n",
    "        return os.path.join(folder_path, latest_file)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def load_models(folds, device):\n",
    "    models = []\n",
    "    for fold in folds:\n",
    "        model = ISICModel(cfg.model_name)\n",
    "        model.to(device)\n",
    "        model_w_path = get_latest_epoch_file(cfg.model_path, fold)\n",
    "        model.load_state_dict(torch.load(model_w_path, map_location=device))\n",
    "        model.eval()\n",
    "        models.append(model)\n",
    "    return models\n",
    "\n",
    "\n",
    "@torch.no_grad()  # Apply no_grad to the entire function\n",
    "def ensemble_predict(models, test_loader, device):\n",
    "    all_predictions = []\n",
    "    for inputs, _ in tqdm(test_loader, desc=\"Predicting\"):\n",
    "        inputs = inputs.to(device)\n",
    "#         fold_predictions = torch.stack([model(inputs).softmax(dim=1)[:, 1] for model in models])\n",
    "        fold_predictions = torch.stack([model(inputs) for model in models])\n",
    "        avg_predictions = fold_predictions.mean(dim=0)\n",
    "        all_predictions.extend(avg_predictions.cpu().numpy())\n",
    "    return all_predictions\n",
    "\n",
    "\n",
    "# === Do ImageNet inference on test data / merge df_test\n",
    "TEST_HDF5_FILE_PATH = '/kaggle/input/isic-2024-challenge/test-image.hdf5'\n",
    "\n",
    "# Set up CUDA if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "# folds to use for pred\n",
    "folds = cfg.folds\n",
    "\n",
    "models = load_models(folds, device)\n",
    "\n",
    "# Prepare your test dataset\n",
    "test_dataset = ISICDataset(\n",
    "    hdf5_file=TEST_HDF5_FILE_PATH,\n",
    "    isic_ids=df_test.index.values,  #minor change here from\n",
    "    transform=base_transform,\n",
    ")\n",
    "\n",
    "# Create test data loader\n",
    "test_loader = DataLoader(test_dataset, batch_size=512, shuffle=False, num_workers=4, pin_memory=True)\n",
    "\n",
    "# Run predictions\n",
    "predictions = ensemble_predict(models, test_loader, device)\n",
    "\n",
    "# Create a new DataFrame with predictions\n",
    "temp_df = pd.DataFrame({\"image_predict\": predictions}, index=df_test.index)\n",
    "\n",
    "# Join the predictions to df_test\n",
    "df_test = df_test.join(temp_df)\n",
    "\n",
    "df_subm[\"target\"] = df_test[\"image_predict\"]\n",
    "\n",
    "df_subm.to_csv(f'submission_{cfg.run_id}.csv')\n",
    "df_subm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9594087d",
   "metadata": {
    "papermill": {
     "duration": 0.025788,
     "end_time": "2024-09-01T16:54:16.437350",
     "exception": false,
     "start_time": "2024-09-01T16:54:16.411562",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Image model 20240902001446"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "384b45a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:16.490403Z",
     "iopub.status.busy": "2024-09-01T16:54:16.490123Z",
     "iopub.status.idle": "2024-09-01T16:54:22.587230Z",
     "shell.execute_reply": "2024-09-01T16:54:22.586111Z"
    },
    "papermill": {
     "duration": 6.126192,
     "end_time": "2024-09-01T16:54:22.589198",
     "exception": false,
     "start_time": "2024-09-01T16:54:16.463006",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Predicting: 100%|██████████| 1/1 [00:00<00:00,  5.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ISIC_0015657</th>\n",
       "      <td>-2.898454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015729</th>\n",
       "      <td>-4.748341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015740</th>\n",
       "      <td>-6.205088</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target\n",
       "isic_id               \n",
       "ISIC_0015657 -2.898454\n",
       "ISIC_0015729 -4.748341\n",
       "ISIC_0015740 -6.205088"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from pathlib import Path\n",
    "from dataclasses import dataclass\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class CFG:\n",
    "    # 見落とし注意！\n",
    "    run_id = \"20240902001446\"\n",
    "    model_name = \"vit_small_patch16_224.augreg_in21k_ft_in1k\"\n",
    "    model_path = Path(f\"/kaggle/input/output-{run_id}\")\n",
    "    img_size = 224 #384 or 224\n",
    "    folds = [0,1,2,3,4]\n",
    "\n",
    "    # auto\n",
    "    oof_path = model_path / \"oof_predictions.csv\"\n",
    "\n",
    "    # --- desctiption\n",
    "        # \"cv\":  0.1486832,\n",
    "        # \"lb\": ,\n",
    "        # \"version\": \"\",\n",
    "        # \"config\": \"vit_small_patch16_224.augreg_in21k_ft_in1k\",\n",
    "        # \"date\": 20240902001446,\n",
    "        # \"desc\": \"lr1e-4 + warmup + train1:1-val1:10 + upsample-2 + AugumentMore + Dropoutx1 + CustomHead-64 + weight_decay1e-3\",\n",
    "        # \"desc\": \"remap case1\",\n",
    "    # ---------------\n",
    "\n",
    "cfg = CFG()\n",
    "\n",
    "test_path = Path('/kaggle/input/isic-2024-challenge/test-metadata.csv')\n",
    "subm_path  = Path('/kaggle/input/isic-2024-challenge/sample_submission.csv')\n",
    "\n",
    "id_col = 'isic_id'\n",
    "\n",
    "def read_data(path, cfg):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .to_pandas()\n",
    "        .set_index(id_col)\n",
    "    )\n",
    "\n",
    "df_test = read_data(test_path, cfg)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "# === ImageNet inference code\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import h5py\n",
    "import timm\n",
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "import io\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "\n",
    "\n",
    "class ISICDataset(Dataset):\n",
    "    def __init__(self, hdf5_file, isic_ids, targets=None, transform=None):\n",
    "        self.hdf5_file = h5py.File(hdf5_file, 'r')  # Keep file open\n",
    "        self.isic_ids = isic_ids\n",
    "        self.targets = targets\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.isic_ids)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img_bytes = self.hdf5_file[self.isic_ids[idx]][()]\n",
    "        img = Image.open(io.BytesIO(img_bytes))\n",
    "        img = np.array(img)\n",
    "\n",
    "        if self.transform:\n",
    "            transformed = self.transform(image=img)\n",
    "            img = transformed['image']\n",
    "\n",
    "        target = self.targets[idx] if self.targets is not None else torch.tensor(-1)\n",
    "        return img, target\n",
    "\n",
    "    def __del__(self):\n",
    "        self.hdf5_file.close()  # Ensure file is closed when object is destroyed\n",
    "\n",
    "# Define the albumentations transformation\n",
    "base_transform = A.Compose([\n",
    "    A.Resize(cfg.img_size, cfg.img_size),\n",
    "    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "    ToTensorV2(),\n",
    "])\n",
    "\n",
    "class GeM(nn.Module):\n",
    "    def __init__(self, p=3, eps=1e-6):\n",
    "        super(GeM, self).__init__()\n",
    "        self.p = nn.Parameter(torch.ones(1) * p)\n",
    "        self.eps = eps\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gem(x, p=self.p, eps=self.eps)\n",
    "\n",
    "    def gem(self, x, p=3, eps=1e-6):\n",
    "        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return (\n",
    "            self.__class__.__name__\n",
    "            + \"(\"\n",
    "            + \"p=\"\n",
    "            + \"{:.4f}\".format(self.p.data.tolist()[0])\n",
    "            + \", \"\n",
    "            + \"eps=\"\n",
    "            + str(self.eps)\n",
    "            + \")\"\n",
    "        )\n",
    "\n",
    "class ISICModel(nn.Module):\n",
    "    def __init__(self, model_name, num_classes=1, pretrained=False, checkpoint_path=None):\n",
    "        super(ISICModel, self).__init__()\n",
    "        self.model_name = model_name\n",
    "        self.model = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n",
    "\n",
    "        if \"eva02_small_patch14_336\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.linear = nn.Linear(in_features, num_classes)\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "        elif \"eva02_small_patch14_224\" in self.model_name:\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(192, 32),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(32, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"eva02_tiny_patch14_224\" in self.model_name:\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(192, 32),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(32, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"efficientnetv2\" in self.model_name:\n",
    "            in_features = self.model.classifier.in_features\n",
    "            self.model.classifier = nn.Identity()\n",
    "            self.pooling = GeM()\n",
    "            if self.pooling:  # My custom pooling\n",
    "                self.model.global_pool = nn.Identity()\n",
    "            self.linear = nn.Linear(in_features, num_classes)\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])\n",
    "        elif \"convnextv2_atto\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                # nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                GeM(),\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 32),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(32, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"convnextv2_nano\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                # nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                GeM(),\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(256, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"swinv2\" in self.model_name:\n",
    "            in_features = self.model.head.fc.in_features\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                # nn.AdaptiveAvgPool2d((1, 1)),  # Global Average Pooling\n",
    "                GeM(),\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 256),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(5)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(256, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"vit_tiny\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.norm = nn.Identity()\n",
    "            self.model.fc_norm = nn.Identity()\n",
    "            self.model.head_drop = nn.Identity()\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 64),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(1)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(64, num_classes)  # 出力層（2クラス分類）\n",
    "        elif \"vit_small\" in self.model_name:\n",
    "            in_features = self.model.head.in_features\n",
    "            self.model.norm = nn.Identity()\n",
    "            self.model.fc_norm = nn.Identity()\n",
    "            self.model.head_drop = nn.Identity()\n",
    "            self.model.head = nn.Identity()\n",
    "            self.feature_extractor = nn.Sequential(\n",
    "                nn.Flatten(),  # フラット化\n",
    "                nn.Linear(in_features, 64),  # 新しい全結合層1\n",
    "                nn.ReLU(),  # 活性化関数\n",
    "            )\n",
    "            self.dropouts = nn.ModuleList([nn.Dropout(0.5) for _ in range(1)])  # 5つのDropout\n",
    "            self.classifier = nn.Linear(64, num_classes)  # 出力層（2クラス分類）\n",
    "\n",
    "    def forward(self, images):\n",
    "        features = self.model(images)\n",
    "\n",
    "        if any(\n",
    "            [\n",
    "                \"efficientnetv2\" in self.model_name,\n",
    "                \"eva02_small_patch14_336\" in self.model_name,\n",
    "            ]\n",
    "        ):\n",
    "            # Custom poolingがある場合\n",
    "            if self.pooling:\n",
    "                features = self.pooling(features).flatten(1)\n",
    "            for i, dropout in enumerate(self.dropouts):\n",
    "                if i == 0:\n",
    "                    output = self.linear(dropout(features))\n",
    "                else:\n",
    "                    output += self.linear(dropout(features))\n",
    "            output /= len(self.dropouts)\n",
    "\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"eva02_small_patch14_224\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"eva02_tiny_patch14_224\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"convnextv2\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"swinv2\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        if \"vit_tiny\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "        if \"vit_small\" in self.model_name:\n",
    "            features = self.feature_extractor(features)\n",
    "            output = torch.mean(torch.stack([dropout(features) for dropout in self.dropouts]), dim=0)\n",
    "            output = self.classifier(output)\n",
    "\n",
    "        return output.squeeze()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def get_latest_epoch_file(folder_path, target_fold):\n",
    "    # 正規表現パターン：fold_X_epoch_Y_score_Z.pth の形式に一致\n",
    "    pattern = re.compile(r\"fold_(\\d+)_epoch_(\\d+)_score_(\\d+\\.\\d+)\\.pth\") # 新パターン\n",
    "#     pattern = re.compile(r\"model_fold_(\\d+)_epoch_(\\d+)\\.pth\") # 旧パターン\n",
    "\n",
    "    max_epoch = -1\n",
    "    latest_file = None\n",
    "\n",
    "    for filename in os.listdir(folder_path):\n",
    "        match = pattern.match(filename)\n",
    "        if match:\n",
    "            fold, epoch, score = match.groups() # 新パターン\n",
    "#             fold, epoch = match.groups() # 旧パターン\n",
    "            fold = int(fold)\n",
    "            epoch = int(epoch)\n",
    "\n",
    "            if fold == target_fold and epoch > max_epoch:\n",
    "                max_epoch = epoch\n",
    "                latest_file = filename\n",
    "\n",
    "    if latest_file:\n",
    "        return os.path.join(folder_path, latest_file)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def load_models(folds, device):\n",
    "    models = []\n",
    "    for fold in folds:\n",
    "        model = ISICModel(cfg.model_name)\n",
    "        model.to(device)\n",
    "        model_w_path = get_latest_epoch_file(cfg.model_path, fold)\n",
    "        model.load_state_dict(torch.load(model_w_path, map_location=device))\n",
    "        model.eval()\n",
    "        models.append(model)\n",
    "    return models\n",
    "\n",
    "\n",
    "@torch.no_grad()  # Apply no_grad to the entire function\n",
    "def ensemble_predict(models, test_loader, device):\n",
    "    all_predictions = []\n",
    "    for inputs, _ in tqdm(test_loader, desc=\"Predicting\"):\n",
    "        inputs = inputs.to(device)\n",
    "#         fold_predictions = torch.stack([model(inputs).softmax(dim=1)[:, 1] for model in models])\n",
    "        fold_predictions = torch.stack([model(inputs) for model in models])\n",
    "        avg_predictions = fold_predictions.mean(dim=0)\n",
    "        all_predictions.extend(avg_predictions.cpu().numpy())\n",
    "    return all_predictions\n",
    "\n",
    "\n",
    "# === Do ImageNet inference on test data / merge df_test\n",
    "TEST_HDF5_FILE_PATH = '/kaggle/input/isic-2024-challenge/test-image.hdf5'\n",
    "\n",
    "# Set up CUDA if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "# folds to use for pred\n",
    "folds = cfg.folds\n",
    "\n",
    "models = load_models(folds, device)\n",
    "\n",
    "# Prepare your test dataset\n",
    "test_dataset = ISICDataset(\n",
    "    hdf5_file=TEST_HDF5_FILE_PATH,\n",
    "    isic_ids=df_test.index.values,  #minor change here from\n",
    "    transform=base_transform,\n",
    ")\n",
    "\n",
    "# Create test data loader\n",
    "test_loader = DataLoader(test_dataset, batch_size=512, shuffle=False, num_workers=4, pin_memory=True)\n",
    "\n",
    "# Run predictions\n",
    "predictions = ensemble_predict(models, test_loader, device)\n",
    "\n",
    "# Create a new DataFrame with predictions\n",
    "temp_df = pd.DataFrame({\"image_predict\": predictions}, index=df_test.index)\n",
    "\n",
    "# Join the predictions to df_test\n",
    "df_test = df_test.join(temp_df)\n",
    "\n",
    "df_subm[\"target\"] = df_test[\"image_predict\"]\n",
    "\n",
    "df_subm.to_csv(f'submission_{cfg.run_id}.csv')\n",
    "df_subm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57ce8818",
   "metadata": {
    "papermill": {
     "duration": 0.026106,
     "end_time": "2024-09-01T16:54:22.642534",
     "exception": false,
     "start_time": "2024-09-01T16:54:22.616428",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# DATA PREPROCESS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0e24e04f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:22.696746Z",
     "iopub.status.busy": "2024-09-01T16:54:22.696451Z",
     "iopub.status.idle": "2024-09-01T16:54:22.712411Z",
     "shell.execute_reply": "2024-09-01T16:54:22.711578Z"
    },
    "papermill": {
     "duration": 0.045582,
     "end_time": "2024-09-01T16:54:22.714349",
     "exception": false,
     "start_time": "2024-09-01T16:54:22.668767",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "root = Path('/kaggle/input/isic-2024-challenge')\n",
    "\n",
    "train_path = root / 'train-metadata.csv'\n",
    "test_path = root / 'test-metadata.csv'\n",
    "subm_path = root / 'sample_submission.csv'\n",
    "\n",
    "id_col = 'isic_id'\n",
    "target_col = 'target'\n",
    "group_col = 'patient_id'\n",
    "\n",
    "err = 1e-5\n",
    "sampling_ratio = 0.01\n",
    "seed = 42\n",
    "\n",
    "num_cols = [\n",
    "    'age_approx',                        # Approximate age of patient at time of imaging.\n",
    "    'clin_size_long_diam_mm',            # Maximum diameter of the lesion (mm).+\n",
    "    'tbp_lv_A',                          # A inside  lesion.+\n",
    "    'tbp_lv_Aext',                       # A outside lesion.+\n",
    "    'tbp_lv_B',                          # B inside  lesion.+\n",
    "    'tbp_lv_Bext',                       # B outside lesion.+\n",
    "    'tbp_lv_C',                          # Chroma inside  lesion.+\n",
    "    'tbp_lv_Cext',                       # Chroma outside lesion.+\n",
    "    'tbp_lv_H',                          # Hue inside the lesion; calculated as the angle of A* and B* in LAB* color space. Typical values range from 25 (red) to 75 (brown).+\n",
    "    'tbp_lv_Hext',                       # Hue outside lesion.+\n",
    "    'tbp_lv_L',                          # L inside lesion.+\n",
    "    'tbp_lv_Lext',                       # L outside lesion.+\n",
    "    'tbp_lv_areaMM2',                    # Area of lesion (mm^2).+\n",
    "    'tbp_lv_area_perim_ratio',           # Border jaggedness, the ratio between lesions perimeter and area. Circular lesions will have low values; irregular shaped lesions will have higher values. Values range 0-10.+\n",
    "    'tbp_lv_color_std_mean',             # Color irregularity, calculated as the variance of colors within the lesion's boundary.\n",
    "    'tbp_lv_deltaA',                     # Average A contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaB',                     # Average B contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaL',                     # Average L contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaLB',                    #\n",
    "    'tbp_lv_deltaLBnorm',                # Contrast between the lesion and its immediate surrounding skin. Low contrast lesions tend to be faintly visible such as freckles; high contrast lesions tend to be those with darker pigment. Calculated as the average delta LB of the lesion relative to its immediate background in LAB* color space. Typical values range from 5.5 to 25.+\n",
    "    'tbp_lv_eccentricity',               # Eccentricity.+\n",
    "    'tbp_lv_minorAxisMM',                # Smallest lesion diameter (mm).+\n",
    "    'tbp_lv_nevi_confidence',            # Nevus confidence score (0-100 scale) is a convolutional neural network classifier estimated probability that the lesion is a nevus. The neural network was trained on approximately 57,000 lesions that were classified and labeled by a dermatologist.+,++\n",
    "    'tbp_lv_norm_border',                # Border irregularity (0-10 scale); the normalized average of border jaggedness and asymmetry.+\n",
    "    'tbp_lv_norm_color',                 # Color variation (0-10 scale); the normalized average of color asymmetry and color irregularity.+\n",
    "    'tbp_lv_perimeterMM',                # Perimeter of lesion (mm).+\n",
    "    'tbp_lv_radial_color_std_max',       # Color asymmetry, a measure of asymmetry of the spatial distribution of color within the lesion. This score is calculated by looking at the average standard deviation in LAB* color space within concentric rings originating from the lesion center. Values range 0-10.+\n",
    "    'tbp_lv_stdL',                       # Standard deviation of L inside  lesion.+\n",
    "    'tbp_lv_stdLExt',                    # Standard deviation of L outside lesion.+\n",
    "    'tbp_lv_symm_2axis',                 # Border asymmetry; a measure of asymmetry of the lesion's contour about an axis perpendicular to the lesion's most symmetric axis. Lesions with two axes of symmetry will therefore have low scores (more symmetric), while lesions with only one or zero axes of symmetry will have higher scores (less symmetric). This score is calculated by comparing opposite halves of the lesion contour over many degrees of rotation. The angle where the halves are most similar identifies the principal axis of symmetry, while the second axis of symmetry is perpendicular to the principal axis. Border asymmetry is reported as the asymmetry value about this second axis. Values range 0-10.+\n",
    "    'tbp_lv_symm_2axis_angle',           # Lesion border asymmetry angle.+\n",
    "    'tbp_lv_x',                          # X-coordinate of the lesion on 3D TBP.+\n",
    "    'tbp_lv_y',                          # Y-coordinate of the lesion on 3D TBP.+\n",
    "    'tbp_lv_z',                          # Z-coordinate of the lesion on 3D TBP.+\n",
    "]\n",
    "\n",
    "new_num_cols = [\n",
    "    'lesion_size_ratio',             # tbp_lv_minorAxisMM      / clin_size_long_diam_mm\n",
    "    'lesion_shape_index',            # tbp_lv_areaMM2          / tbp_lv_perimeterMM **2\n",
    "    'hue_contrast',                  # tbp_lv_H                - tbp_lv_Hext              abs\n",
    "    'luminance_contrast',            # tbp_lv_L                - tbp_lv_Lext              abs\n",
    "    'lesion_color_difference',       # tbp_lv_deltaA **2       + tbp_lv_deltaB **2 + tbp_lv_deltaL **2  sqrt\n",
    "    'border_complexity',             # tbp_lv_norm_border      + tbp_lv_symm_2axis\n",
    "    'color_uniformity',              # tbp_lv_color_std_mean   / tbp_lv_radial_color_std_max\n",
    "\n",
    "    'position_distance_3d',          # tbp_lv_x **2 + tbp_lv_y **2 + tbp_lv_z **2  sqrt\n",
    "    'perimeter_to_area_ratio',       # tbp_lv_perimeterMM      / tbp_lv_areaMM2\n",
    "    'area_to_perimeter_ratio',       # tbp_lv_areaMM2          / tbp_lv_perimeterMM\n",
    "    'lesion_visibility_score',       # tbp_lv_deltaLBnorm      + tbp_lv_norm_color\n",
    "    'symmetry_border_consistency',   # tbp_lv_symm_2axis       * tbp_lv_norm_border\n",
    "    'consistency_symmetry_border',   # tbp_lv_symm_2axis       * tbp_lv_norm_border / (tbp_lv_symm_2axis + tbp_lv_norm_border)\n",
    "\n",
    "    'color_consistency',             # tbp_lv_stdL             / tbp_lv_Lext\n",
    "    'consistency_color',             # tbp_lv_stdL*tbp_lv_Lext / tbp_lv_stdL + tbp_lv_Lext\n",
    "    'size_age_interaction',          # clin_size_long_diam_mm  * age_approx\n",
    "    'hue_color_std_interaction',     # tbp_lv_H                * tbp_lv_color_std_mean\n",
    "    'lesion_severity_index',         # tbp_lv_norm_border      + tbp_lv_norm_color + tbp_lv_eccentricity / 3\n",
    "    'shape_complexity_index',        # border_complexity       + lesion_shape_index\n",
    "    'color_contrast_index',          # tbp_lv_deltaA + tbp_lv_deltaB + tbp_lv_deltaL + tbp_lv_deltaLBnorm\n",
    "\n",
    "    'log_lesion_area',               # tbp_lv_areaMM2          + 1  np.log\n",
    "    'normalized_lesion_size',        # clin_size_long_diam_mm  / age_approx\n",
    "    'mean_hue_difference',           # tbp_lv_H                + tbp_lv_Hext    / 2\n",
    "    'std_dev_contrast',              # tbp_lv_deltaA **2 + tbp_lv_deltaB **2 + tbp_lv_deltaL **2   / 3  np.sqrt\n",
    "    'color_shape_composite_index',   # tbp_lv_color_std_mean   + bp_lv_area_perim_ratio + tbp_lv_symm_2axis   / 3\n",
    "    'lesion_orientation_3d',         # tbp_lv_y                , tbp_lv_x  np.arctan2\n",
    "    'overall_color_difference',      # tbp_lv_deltaA           + tbp_lv_deltaB + tbp_lv_deltaL   / 3\n",
    "\n",
    "    'symmetry_perimeter_interaction',# tbp_lv_symm_2axis       * tbp_lv_perimeterMM\n",
    "    'comprehensive_lesion_index',    # tbp_lv_area_perim_ratio + tbp_lv_eccentricity + bp_lv_norm_color + tbp_lv_symm_2axis   / 4\n",
    "    'color_variance_ratio',          # tbp_lv_color_std_mean   / tbp_lv_stdLExt\n",
    "    'border_color_interaction',      # tbp_lv_norm_border      * tbp_lv_norm_color\n",
    "    'border_color_interaction_2',\n",
    "    'size_color_contrast_ratio',     # clin_size_long_diam_mm  / tbp_lv_deltaLBnorm\n",
    "    'age_normalized_nevi_confidence',# tbp_lv_nevi_confidence  / age_approx\n",
    "    'age_normalized_nevi_confidence_2',\n",
    "    'color_asymmetry_index',         # tbp_lv_symm_2axis       * tbp_lv_radial_color_std_max\n",
    "\n",
    "    'volume_approximation_3d',       # tbp_lv_areaMM2          * sqrt(tbp_lv_x**2 + tbp_lv_y**2 + tbp_lv_z**2)\n",
    "    'color_range',                   # abs(tbp_lv_L - tbp_lv_Lext) + abs(tbp_lv_A - tbp_lv_Aext) + abs(tbp_lv_B - tbp_lv_Bext)\n",
    "    'shape_color_consistency',       # tbp_lv_eccentricity     * tbp_lv_color_std_mean\n",
    "    'border_length_ratio',           # tbp_lv_perimeterMM      / pi * sqrt(tbp_lv_areaMM2 / pi)\n",
    "    'age_size_symmetry_index',       # age_approx              * clin_size_long_diam_mm * tbp_lv_symm_2axis\n",
    "    'index_age_size_symmetry',       # age_approx              * tbp_lv_areaMM2 * tbp_lv_symm_2axis\n",
    "]\n",
    "\n",
    "cat_cols = ['sex', 'anatom_site_general', 'tbp_tile_type', 'tbp_lv_location', 'tbp_lv_location_simple', 'attribution']\n",
    "norm_cols = [f'{col}_patient_norm' for col in num_cols + new_num_cols]\n",
    "special_cols = ['count_per_patient']\n",
    "# image_cols = [\"target_3\",\"target_effnetv1b0\",\"target_eva02\"]\n",
    "#image_cols = [\"target_3\",\"target_effnetv1b0\"]\n",
    "# image_cols = [\"target_3\",\"target_effnetv1b0\",\"target_eva02\", \"image_20240818021241\"]\n",
    "# image_cols = [\"image_diff-20240823202308\", \"image_20240821002557\"]\n",
    "image_cols = [\n",
    "#     \"image_20240824125307\",\n",
    "#     \"image_20240825010806\",\n",
    "#     \"image_20240826023642\",\n",
    "#     \"image_20240827012429\",\n",
    "    \"image_20240827234748\",\n",
    "    \"image_20240830205516\",\n",
    "    \"image_20240831025049\",\n",
    "    \"image_20240902001446\",\n",
    "]\n",
    "\n",
    "#norm_cols += image_cols\n",
    "feature_cols = num_cols + new_num_cols + cat_cols + norm_cols + special_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "abd144b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:22.769291Z",
     "iopub.status.busy": "2024-09-01T16:54:22.769026Z",
     "iopub.status.idle": "2024-09-01T16:54:22.794612Z",
     "shell.execute_reply": "2024-09-01T16:54:22.793845Z"
    },
    "papermill": {
     "duration": 0.055082,
     "end_time": "2024-09-01T16:54:22.796405",
     "exception": false,
     "start_time": "2024-09-01T16:54:22.741323",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def read_data(path):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .with_columns(\n",
    "            pl.col('age_approx').cast(pl.String).replace('NA', np.nan).cast(pl.Float64),\n",
    "        )\n",
    "        .with_columns(\n",
    "            pl.col(pl.Float64).fill_nan(pl.col(pl.Float64).median()), # You may want to impute test data with train\n",
    "        )\n",
    "        .with_columns(\n",
    "            lesion_size_ratio              = pl.col('tbp_lv_minorAxisMM') / pl.col('clin_size_long_diam_mm'),\n",
    "            lesion_shape_index             = pl.col('tbp_lv_areaMM2') / (pl.col('tbp_lv_perimeterMM') ** 2),\n",
    "            hue_contrast                   = (pl.col('tbp_lv_H') - pl.col('tbp_lv_Hext')).abs(),\n",
    "            luminance_contrast             = (pl.col('tbp_lv_L') - pl.col('tbp_lv_Lext')).abs(),\n",
    "            lesion_color_difference        = (pl.col('tbp_lv_deltaA') ** 2 + pl.col('tbp_lv_deltaB') ** 2 + pl.col('tbp_lv_deltaL') ** 2).sqrt(),\n",
    "            border_complexity              = pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_symm_2axis'),\n",
    "            color_uniformity               = pl.col('tbp_lv_color_std_mean') / (pl.col('tbp_lv_radial_color_std_max') + err),\n",
    "        )\n",
    "        .with_columns(\n",
    "            position_distance_3d           = (pl.col('tbp_lv_x') ** 2 + pl.col('tbp_lv_y') ** 2 + pl.col('tbp_lv_z') ** 2).sqrt(),\n",
    "            perimeter_to_area_ratio        = pl.col('tbp_lv_perimeterMM') / pl.col('tbp_lv_areaMM2'),\n",
    "            area_to_perimeter_ratio        = pl.col('tbp_lv_areaMM2') / pl.col('tbp_lv_perimeterMM'),\n",
    "            lesion_visibility_score        = pl.col('tbp_lv_deltaLBnorm') + pl.col('tbp_lv_norm_color'),\n",
    "            combined_anatomical_site       = pl.col('anatom_site_general') + '_' + pl.col('tbp_lv_location'),\n",
    "            symmetry_border_consistency    = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_norm_border'),\n",
    "            consistency_symmetry_border    = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_norm_border') / (pl.col('tbp_lv_symm_2axis') + pl.col('tbp_lv_norm_border')),\n",
    "        )\n",
    "        .with_columns(\n",
    "            color_consistency              = pl.col('tbp_lv_stdL') / pl.col('tbp_lv_Lext'),\n",
    "            consistency_color              = pl.col('tbp_lv_stdL') * pl.col('tbp_lv_Lext') / (pl.col('tbp_lv_stdL') + pl.col('tbp_lv_Lext')),\n",
    "            size_age_interaction           = pl.col('clin_size_long_diam_mm') * pl.col('age_approx'),\n",
    "            hue_color_std_interaction      = pl.col('tbp_lv_H') * pl.col('tbp_lv_color_std_mean'),\n",
    "            lesion_severity_index          = (pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_norm_color') + pl.col('tbp_lv_eccentricity')) / 3,\n",
    "            shape_complexity_index         = pl.col('border_complexity') + pl.col('lesion_shape_index'),\n",
    "            color_contrast_index           = pl.col('tbp_lv_deltaA') + pl.col('tbp_lv_deltaB') + pl.col('tbp_lv_deltaL') + pl.col('tbp_lv_deltaLBnorm'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            log_lesion_area                = (pl.col('tbp_lv_areaMM2') + 1).log(),\n",
    "            normalized_lesion_size         = pl.col('clin_size_long_diam_mm') / pl.col('age_approx'),\n",
    "            mean_hue_difference            = (pl.col('tbp_lv_H') + pl.col('tbp_lv_Hext')) / 2,\n",
    "            std_dev_contrast               = ((pl.col('tbp_lv_deltaA') ** 2 + pl.col('tbp_lv_deltaB') ** 2 + pl.col('tbp_lv_deltaL') ** 2) / 3).sqrt(),\n",
    "            color_shape_composite_index    = (pl.col('tbp_lv_color_std_mean') + pl.col('tbp_lv_area_perim_ratio') + pl.col('tbp_lv_symm_2axis')) / 3,\n",
    "            lesion_orientation_3d          = pl.arctan2(pl.col('tbp_lv_y'), pl.col('tbp_lv_x')),\n",
    "            overall_color_difference       = (pl.col('tbp_lv_deltaA') + pl.col('tbp_lv_deltaB') + pl.col('tbp_lv_deltaL')) / 3,\n",
    "        )\n",
    "        .with_columns(\n",
    "            symmetry_perimeter_interaction = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_perimeterMM'),\n",
    "            comprehensive_lesion_index     = (pl.col('tbp_lv_area_perim_ratio') + pl.col('tbp_lv_eccentricity') + pl.col('tbp_lv_norm_color') + pl.col('tbp_lv_symm_2axis')) / 4,\n",
    "            color_variance_ratio           = pl.col('tbp_lv_color_std_mean') / pl.col('tbp_lv_stdLExt'),\n",
    "            border_color_interaction       = pl.col('tbp_lv_norm_border') * pl.col('tbp_lv_norm_color'),\n",
    "            border_color_interaction_2     = pl.col('tbp_lv_norm_border') * pl.col('tbp_lv_norm_color') / (pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_norm_color')),\n",
    "            size_color_contrast_ratio      = pl.col('clin_size_long_diam_mm') / pl.col('tbp_lv_deltaLBnorm'),\n",
    "            age_normalized_nevi_confidence = pl.col('tbp_lv_nevi_confidence') / pl.col('age_approx'),\n",
    "            age_normalized_nevi_confidence_2 = (pl.col('clin_size_long_diam_mm')**2 + pl.col('age_approx')**2).sqrt(),\n",
    "            color_asymmetry_index          = pl.col('tbp_lv_radial_color_std_max') * pl.col('tbp_lv_symm_2axis'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            volume_approximation_3d        = pl.col('tbp_lv_areaMM2') * (pl.col('tbp_lv_x')**2 + pl.col('tbp_lv_y')**2 + pl.col('tbp_lv_z')**2).sqrt(),\n",
    "            color_range                    = (pl.col('tbp_lv_L') - pl.col('tbp_lv_Lext')).abs() + (pl.col('tbp_lv_A') - pl.col('tbp_lv_Aext')).abs() + (pl.col('tbp_lv_B') - pl.col('tbp_lv_Bext')).abs(),\n",
    "            shape_color_consistency        = pl.col('tbp_lv_eccentricity') * pl.col('tbp_lv_color_std_mean'),\n",
    "            border_length_ratio            = pl.col('tbp_lv_perimeterMM') / (2 * np.pi * (pl.col('tbp_lv_areaMM2') / np.pi).sqrt()),\n",
    "            age_size_symmetry_index        = pl.col('age_approx') * pl.col('clin_size_long_diam_mm') * pl.col('tbp_lv_symm_2axis'),\n",
    "            index_age_size_symmetry        = pl.col('age_approx') * pl.col('tbp_lv_areaMM2') * pl.col('tbp_lv_symm_2axis'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            ((pl.col(col) - pl.col(col).mean().over('patient_id')) / (pl.col(col).std().over('patient_id') + err)).alias(f'{col}_patient_norm') for col in (num_cols + new_num_cols)\n",
    "        )\n",
    "        .with_columns(\n",
    "            count_per_patient = pl.col('isic_id').count().over('patient_id'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            pl.col(cat_cols).cast(pl.Categorical),\n",
    "        )\n",
    "        .to_pandas()\n",
    "#         .set_index(id_col)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "608f39b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:22.850442Z",
     "iopub.status.busy": "2024-09-01T16:54:22.849964Z",
     "iopub.status.idle": "2024-09-01T16:54:22.871508Z",
     "shell.execute_reply": "2024-09-01T16:54:22.870789Z"
    },
    "papermill": {
     "duration": 0.050758,
     "end_time": "2024-09-01T16:54:22.873304",
     "exception": false,
     "start_time": "2024-09-01T16:54:22.822546",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def preprocess(df_train, df_test):\n",
    "    global cat_cols\n",
    "\n",
    "    encoder = OneHotEncoder(sparse_output=False, dtype=np.int32, handle_unknown='ignore')\n",
    "    encoder.fit(df_train[cat_cols])\n",
    "\n",
    "    new_cat_cols = [f'onehot_{i}' for i in range(len(encoder.get_feature_names_out()))]\n",
    "\n",
    "    df_train[new_cat_cols] = encoder.transform(df_train[cat_cols])\n",
    "    df_train[new_cat_cols] = df_train[new_cat_cols].astype('category')\n",
    "\n",
    "    df_test[new_cat_cols] = encoder.transform(df_test[cat_cols])\n",
    "    df_test[new_cat_cols] = df_test[new_cat_cols].astype('category')\n",
    "\n",
    "\n",
    "    # === swd\n",
    "    run_id = \"20240827234748\"\n",
    "    df_train_image = pd.read_csv(f\"/kaggle/input/output-{run_id}/oof_predictions.csv\")\n",
    "\n",
    "#     # MinMax-scale\n",
    "#     for fold in [0, 1, 2, 3, 4]:\n",
    "#         filter = df_train_image[\"fold\"] == fold\n",
    "#         _df = df_train_image[filter]\n",
    "#         scaler = MinMaxScaler()\n",
    "#         df_train_image.loc[filter, \"oof_prediction\"]  = scaler.fit_transform(_df[\"oof_prediction\"].values.reshape(-1, 1))\n",
    "\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_train_image = df_train_image.reset_index(drop=True)\n",
    "    df_train[f\"image_{run_id}\"] = df_train_image[\"oof_prediction\"]\n",
    "\n",
    "    df_test_image = pd.read_csv(f\"submission_{run_id}.csv\")\n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_test_image = df_test_image.reset_index(drop=True)\n",
    "\n",
    "    df_test[f\"image_{run_id}\"] = df_test_image[\"target\"]\n",
    "\n",
    "    # === swd\n",
    "    run_id = \"20240830205516\"\n",
    "    df_train_image = pd.read_csv(f\"/kaggle/input/output-{run_id}/oof_predictions.csv\")\n",
    "\n",
    "#     # MinMax-scale\n",
    "#     for fold in [0, 1, 2, 3, 4]:\n",
    "#         filter = df_train_image[\"fold\"] == fold\n",
    "#         _df = df_train_image[filter]\n",
    "#         scaler = MinMaxScaler()\n",
    "#         df_train_image.loc[filter, \"oof_prediction\"]  = scaler.fit_transform(_df[\"oof_prediction\"].values.reshape(-1, 1))\n",
    "\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_train_image = df_train_image.reset_index(drop=True)\n",
    "    df_train[f\"image_{run_id}\"] = df_train_image[\"oof_prediction\"]\n",
    "\n",
    "    df_test_image = pd.read_csv(f\"submission_{run_id}.csv\")\n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_test_image = df_test_image.reset_index(drop=True)\n",
    "\n",
    "    df_test[f\"image_{run_id}\"] = df_test_image[\"target\"]\n",
    "\n",
    "\n",
    "    # === swd\n",
    "    run_id = \"20240831025049\"\n",
    "    df_train_image = pd.read_csv(f\"/kaggle/input/output-{run_id}/oof_predictions.csv\")\n",
    "\n",
    "#     # MinMax-scale\n",
    "#     for fold in [0, 1, 2, 3, 4]:\n",
    "#         filter = df_train_image[\"fold\"] == fold\n",
    "#         _df = df_train_image[filter]\n",
    "#         scaler = MinMaxScaler()\n",
    "#         df_train_image.loc[filter, \"oof_prediction\"]  = scaler.fit_transform(_df[\"oof_prediction\"].values.reshape(-1, 1))\n",
    "\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_train_image = df_train_image.reset_index(drop=True)\n",
    "    df_train[f\"image_{run_id}\"] = df_train_image[\"oof_prediction\"]\n",
    "\n",
    "    df_test_image = pd.read_csv(f\"submission_{run_id}.csv\")\n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_test_image = df_test_image.reset_index(drop=True)\n",
    "\n",
    "    df_test[f\"image_{run_id}\"] = df_test_image[\"target\"]\n",
    "\n",
    "\n",
    "    # === swd\n",
    "    run_id = \"20240902001446\"\n",
    "    df_train_image = pd.read_csv(f\"/kaggle/input/output-{run_id}/oof_predictions.csv\")\n",
    "\n",
    "#     # MinMax-scale\n",
    "#     for fold in [0, 1, 2, 3, 4]:\n",
    "#         filter = df_train_image[\"fold\"] == fold\n",
    "#         _df = df_train_image[filter]\n",
    "#         scaler = MinMaxScaler()\n",
    "#         df_train_image.loc[filter, \"oof_prediction\"]  = scaler.fit_transform(_df[\"oof_prediction\"].values.reshape(-1, 1))\n",
    "\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_train_image = df_train_image.reset_index(drop=True)\n",
    "    df_train[f\"image_{run_id}\"] = df_train_image[\"oof_prediction\"]\n",
    "\n",
    "    df_test_image = pd.read_csv(f\"submission_{run_id}.csv\")\n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_test_image = df_test_image.reset_index(drop=True)\n",
    "\n",
    "    df_test[f\"image_{run_id}\"] = df_test_image[\"target\"]\n",
    "\n",
    "\n",
    "\n",
    "# ==============================\n",
    "    for col in cat_cols:\n",
    "        feature_cols.remove(col)\n",
    "\n",
    "    feature_cols.extend(new_cat_cols)\n",
    "    cat_cols = new_cat_cols\n",
    "\n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ed6cc9af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:22.927120Z",
     "iopub.status.busy": "2024-09-01T16:54:22.926633Z",
     "iopub.status.idle": "2024-09-01T16:54:22.932025Z",
     "shell.execute_reply": "2024-09-01T16:54:22.931298Z"
    },
    "papermill": {
     "duration": 0.034299,
     "end_time": "2024-09-01T16:54:22.933849",
     "exception": false,
     "start_time": "2024-09-01T16:54:22.899550",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def custom_metric(y_hat, y_true):\n",
    "    # y_hat = estimator.predict_proba(X)[:, 1]\n",
    "    min_tpr = 0.80\n",
    "    max_fpr = abs(1 - min_tpr)\n",
    "\n",
    "    v_gt = abs(y_true - 1)\n",
    "    v_pred = np.array([1.0 - x for x in y_hat])\n",
    "\n",
    "    partial_auc_scaled = roc_auc_score(v_gt, v_pred, max_fpr=max_fpr)\n",
    "    partial_auc = 0.5 * max_fpr**2 + (max_fpr - 0.5 * max_fpr**2) / (1.0 - 0.5) * (partial_auc_scaled - 0.5)\n",
    "\n",
    "    return partial_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e84e6d63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:22.987092Z",
     "iopub.status.busy": "2024-09-01T16:54:22.986826Z",
     "iopub.status.idle": "2024-09-01T16:54:33.620069Z",
     "shell.execute_reply": "2024-09-01T16:54:33.619043Z"
    },
    "papermill": {
     "duration": 10.662641,
     "end_time": "2024-09-01T16:54:33.622616",
     "exception": false,
     "start_time": "2024-09-01T16:54:22.959975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_train = read_data(train_path)\n",
    "df_test = read_data(test_path)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "df_train, df_test = preprocess(df_train, df_test)\n",
    "\n",
    "df_train[\"iddx_2\"] = df_train[\"iddx_2\"].replace(\"\", np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e541cbeb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:33.677936Z",
     "iopub.status.busy": "2024-09-01T16:54:33.677113Z",
     "iopub.status.idle": "2024-09-01T16:54:34.910187Z",
     "shell.execute_reply": "2024-09-01T16:54:34.908989Z"
    },
    "papermill": {
     "duration": 1.262873,
     "end_time": "2024-09-01T16:54:34.912621",
     "exception": false,
     "start_time": "2024-09-01T16:54:33.649748",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold Summary (patients per fold):\n",
      "Fold 0: 206 patients\n",
      "Fold 1: 209 patients\n",
      "Fold 2: 208 patients\n",
      "Fold 3: 209 patients\n",
      "Fold 4: 210 patients\n",
      "Total patients: 1042\n"
     ]
    }
   ],
   "source": [
    "# Split folds\n",
    "\n",
    "df_fold = pd.read_csv(\"/kaggle/input/df-fold/df_fold.csv\")\n",
    "\n",
    "df_train = df_train.merge(df_fold, left_on=\"isic_id\", right_on=\"isic_id\", how=\"left\")\n",
    "\n",
    "# Add summary\n",
    "fold_summary = df_train.groupby(\"fold\")[\"patient_id\"].nunique().to_dict()\n",
    "total_patients = df_train[\"patient_id\"].nunique()\n",
    "\n",
    "print(f\"Fold Summary (patients per fold):\")\n",
    "for fold, count in fold_summary.items():\n",
    "    if fold != -1:  # Exclude the initialization value\n",
    "        print(f\"Fold {fold}: {count} patients\")\n",
    "print(f\"Total patients: {total_patients}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "271e6e6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:34.968875Z",
     "iopub.status.busy": "2024-09-01T16:54:34.968581Z",
     "iopub.status.idle": "2024-09-01T16:54:35.963638Z",
     "shell.execute_reply": "2024-09-01T16:54:35.962556Z"
    },
    "papermill": {
     "duration": 1.024891,
     "end_time": "2024-09-01T16:54:35.965787",
     "exception": false,
     "start_time": "2024-09-01T16:54:34.940896",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------\n",
      "Indeterminate の数\n",
      "fold  target\n",
      "0     0         17\n",
      "1     0         32\n",
      "2     0         12\n",
      "3     0         25\n",
      "4     0         28\n",
      "Name: count, dtype: int64\n",
      "--------------------\n",
      "iddx_2が存在する数\n",
      "fold  target\n",
      "0     0         128\n",
      "      1          77\n",
      "1     0         142\n",
      "      1          78\n",
      "2     0         130\n",
      "      1          80\n",
      "3     0         141\n",
      "      1          80\n",
      "4     0         134\n",
      "      1          78\n",
      "Name: count, dtype: int64\n",
      "--------------------\n",
      "処理の妥当性を確認（target=0におけるIndeterminateが消えていることを確認）\n",
      "fold  target  iddx_1   \n",
      "0     0       Benign       80009\n",
      "      1       Malignant       77\n",
      "1     0       Benign       79992\n",
      "      1       Malignant       78\n",
      "2     0       Benign       80001\n",
      "      1       Malignant       80\n",
      "3     0       Benign       79990\n",
      "      1       Malignant       80\n",
      "4     0       Benign       79999\n",
      "      1       Malignant       78\n",
      "Name: count, dtype: int64\n",
      "--------------------\n",
      "処理の妥当性を確認（target=0におけるIddx_2が消えていることを確認）\n",
      "fold  target  iddx_2                                                  \n",
      "0     1       Malignant adnexal epithelial proliferations - Follicular    41\n",
      "              Malignant melanocytic proliferations (Melanoma)             22\n",
      "              Malignant epidermal proliferations                          14\n",
      "1     1       Malignant adnexal epithelial proliferations - Follicular    35\n",
      "              Malignant melanocytic proliferations (Melanoma)             30\n",
      "              Malignant epidermal proliferations                          13\n",
      "2     1       Malignant melanocytic proliferations (Melanoma)             37\n",
      "              Malignant adnexal epithelial proliferations - Follicular    28\n",
      "              Malignant epidermal proliferations                          15\n",
      "3     1       Malignant adnexal epithelial proliferations - Follicular    32\n",
      "              Malignant melanocytic proliferations (Melanoma)             28\n",
      "              Malignant epidermal proliferations                          20\n",
      "4     1       Malignant melanocytic proliferations (Melanoma)             40\n",
      "              Malignant adnexal epithelial proliferations - Follicular    27\n",
      "              Malignant epidermal proliferations                          11\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Additional Filter\n",
    "print(\"-\"*20)\n",
    "print(\"Indeterminate の数\")\n",
    "print(df_train[df_train[\"iddx_1\"]==\"Indeterminate\"].groupby(\"fold\")[\"target\"].value_counts())\n",
    "\n",
    "print(\"-\"*20)\n",
    "print(\"iddx_2が存在する数\")\n",
    "print(df_train[df_train[\"iddx_2\"].notna()].groupby(\"fold\")[\"target\"].value_counts())\n",
    "\n",
    "exclude_isic_ids = []\n",
    "filter = (df_train[\"target\"] == 0) & (df_train[\"iddx_1\"] == \"Indeterminate\")\n",
    "exclude_isic_ids.extend(df_train[filter][\"isic_id\"].values.tolist())\n",
    "filter = (df_train[\"target\"] == 0) & (df_train[\"iddx_2\"].notna())\n",
    "exclude_isic_ids.extend(df_train[filter][\"isic_id\"].values.tolist())\n",
    "\n",
    "exclude_isic_ids = list(set(exclude_isic_ids))\n",
    "\n",
    "print(\"-\"*20)\n",
    "print(\"処理の妥当性を確認（target=0におけるIndeterminateが消えていることを確認）\")\n",
    "print(df_train[~df_train[\"isic_id\"].isin(exclude_isic_ids)].groupby([\"fold\", \"target\"])[\"iddx_1\"].value_counts())\n",
    "\n",
    "print(\"-\"*20)\n",
    "print(\"処理の妥当性を確認（target=0におけるIddx_2が消えていることを確認）\")\n",
    "print(df_train[~df_train[\"isic_id\"].isin(exclude_isic_ids)].groupby([\"fold\", \"target\"])[\"iddx_2\"].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "66563900",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.021620Z",
     "iopub.status.busy": "2024-09-01T16:54:36.021039Z",
     "iopub.status.idle": "2024-09-01T16:54:36.047994Z",
     "shell.execute_reply": "2024-09-01T16:54:36.047145Z"
    },
    "papermill": {
     "duration": 0.056799,
     "end_time": "2024-09-01T16:54:36.050059",
     "exception": false,
     "start_time": "2024-09-01T16:54:35.993260",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>isic_id</th>\n",
       "      <th>target</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>sex</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>clin_size_long_diam_mm</th>\n",
       "      <th>image_type</th>\n",
       "      <th>tbp_tile_type</th>\n",
       "      <th>tbp_lv_A</th>\n",
       "      <th>...</th>\n",
       "      <th>onehot_42</th>\n",
       "      <th>onehot_43</th>\n",
       "      <th>onehot_44</th>\n",
       "      <th>onehot_45</th>\n",
       "      <th>onehot_46</th>\n",
       "      <th>image_20240827234748</th>\n",
       "      <th>image_20240830205516</th>\n",
       "      <th>image_20240831025049</th>\n",
       "      <th>image_20240902001446</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0015670</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_1235828</td>\n",
       "      <td>60.0</td>\n",
       "      <td>male</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>3.04</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: white</td>\n",
       "      <td>20.244422</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.826628</td>\n",
       "      <td>-4.128582</td>\n",
       "      <td>-3.712779</td>\n",
       "      <td>-2.799356</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015845</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_8170065</td>\n",
       "      <td>60.0</td>\n",
       "      <td>male</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>1.10</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: white</td>\n",
       "      <td>31.712570</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-3.171355</td>\n",
       "      <td>-0.409820</td>\n",
       "      <td>-0.458478</td>\n",
       "      <td>0.351795</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0015864</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_6724798</td>\n",
       "      <td>60.0</td>\n",
       "      <td>male</td>\n",
       "      <td>posterior torso</td>\n",
       "      <td>3.40</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: XP</td>\n",
       "      <td>22.575830</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.405538</td>\n",
       "      <td>-5.672004</td>\n",
       "      <td>-5.391024</td>\n",
       "      <td>-6.028082</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0015902</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_4111386</td>\n",
       "      <td>65.0</td>\n",
       "      <td>male</td>\n",
       "      <td>anterior torso</td>\n",
       "      <td>3.22</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: XP</td>\n",
       "      <td>14.242329</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-3.528047</td>\n",
       "      <td>-3.939052</td>\n",
       "      <td>-2.017710</td>\n",
       "      <td>-2.314598</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0024200</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_8313778</td>\n",
       "      <td>55.0</td>\n",
       "      <td>male</td>\n",
       "      <td>anterior torso</td>\n",
       "      <td>2.73</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: white</td>\n",
       "      <td>24.725520</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.711912</td>\n",
       "      <td>-4.667692</td>\n",
       "      <td>-1.609399</td>\n",
       "      <td>-2.456065</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        isic_id  target  patient_id  age_approx   sex anatom_site_general  \\\n",
       "0  ISIC_0015670       0  IP_1235828        60.0  male     lower extremity   \n",
       "1  ISIC_0015845       0  IP_8170065        60.0  male           head/neck   \n",
       "2  ISIC_0015864       0  IP_6724798        60.0  male     posterior torso   \n",
       "3  ISIC_0015902       0  IP_4111386        65.0  male      anterior torso   \n",
       "4  ISIC_0024200       0  IP_8313778        55.0  male      anterior torso   \n",
       "\n",
       "   clin_size_long_diam_mm          image_type tbp_tile_type   tbp_lv_A  ...  \\\n",
       "0                    3.04  TBP tile: close-up     3D: white  20.244422  ...   \n",
       "1                    1.10  TBP tile: close-up     3D: white  31.712570  ...   \n",
       "2                    3.40  TBP tile: close-up        3D: XP  22.575830  ...   \n",
       "3                    3.22  TBP tile: close-up        3D: XP  14.242329  ...   \n",
       "4                    2.73  TBP tile: close-up     3D: white  24.725520  ...   \n",
       "\n",
       "   onehot_42  onehot_43  onehot_44  onehot_45  onehot_46  \\\n",
       "0          0          0          1          0          0   \n",
       "1          0          0          1          0          0   \n",
       "2          0          0          1          0          0   \n",
       "3          0          0          0          0          0   \n",
       "4          0          0          1          0          0   \n",
       "\n",
       "   image_20240827234748  image_20240830205516  image_20240831025049  \\\n",
       "0             -1.826628             -4.128582             -3.712779   \n",
       "1             -3.171355             -0.409820             -0.458478   \n",
       "2             -4.405538             -5.672004             -5.391024   \n",
       "3             -3.528047             -3.939052             -2.017710   \n",
       "4             -1.711912             -4.667692             -1.609399   \n",
       "\n",
       "   image_20240902001446  fold  \n",
       "0             -2.799356     3  \n",
       "1              0.351795     0  \n",
       "2             -6.028082     4  \n",
       "3             -2.314598     1  \n",
       "4             -2.456065     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9b44ccb9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.105628Z",
     "iopub.status.busy": "2024-09-01T16:54:36.105367Z",
     "iopub.status.idle": "2024-09-01T16:54:36.128314Z",
     "shell.execute_reply": "2024-09-01T16:54:36.127473Z"
    },
    "papermill": {
     "duration": 0.052939,
     "end_time": "2024-09-01T16:54:36.130433",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.077494",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>isic_id</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>sex</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>clin_size_long_diam_mm</th>\n",
       "      <th>image_type</th>\n",
       "      <th>tbp_tile_type</th>\n",
       "      <th>tbp_lv_A</th>\n",
       "      <th>tbp_lv_Aext</th>\n",
       "      <th>...</th>\n",
       "      <th>onehot_41</th>\n",
       "      <th>onehot_42</th>\n",
       "      <th>onehot_43</th>\n",
       "      <th>onehot_44</th>\n",
       "      <th>onehot_45</th>\n",
       "      <th>onehot_46</th>\n",
       "      <th>image_20240827234748</th>\n",
       "      <th>image_20240830205516</th>\n",
       "      <th>image_20240831025049</th>\n",
       "      <th>image_20240902001446</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0015657</td>\n",
       "      <td>IP_6074337</td>\n",
       "      <td>45.0</td>\n",
       "      <td>male</td>\n",
       "      <td>posterior torso</td>\n",
       "      <td>2.70</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: XP</td>\n",
       "      <td>22.80433</td>\n",
       "      <td>20.007270</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.501967</td>\n",
       "      <td>-2.714859</td>\n",
       "      <td>-1.637331</td>\n",
       "      <td>-2.898454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015729</td>\n",
       "      <td>IP_1664139</td>\n",
       "      <td>35.0</td>\n",
       "      <td>female</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>2.52</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: XP</td>\n",
       "      <td>16.64867</td>\n",
       "      <td>9.657964</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.002757</td>\n",
       "      <td>-5.223221</td>\n",
       "      <td>-3.406027</td>\n",
       "      <td>-4.748341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0015740</td>\n",
       "      <td>IP_7142616</td>\n",
       "      <td>65.0</td>\n",
       "      <td>male</td>\n",
       "      <td>posterior torso</td>\n",
       "      <td>3.16</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: XP</td>\n",
       "      <td>24.25384</td>\n",
       "      <td>19.937380</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-3.307847</td>\n",
       "      <td>-4.123434</td>\n",
       "      <td>-4.912541</td>\n",
       "      <td>-6.205088</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 215 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        isic_id  patient_id  age_approx     sex anatom_site_general  \\\n",
       "0  ISIC_0015657  IP_6074337        45.0    male     posterior torso   \n",
       "1  ISIC_0015729  IP_1664139        35.0  female     lower extremity   \n",
       "2  ISIC_0015740  IP_7142616        65.0    male     posterior torso   \n",
       "\n",
       "   clin_size_long_diam_mm          image_type tbp_tile_type  tbp_lv_A  \\\n",
       "0                    2.70  TBP tile: close-up        3D: XP  22.80433   \n",
       "1                    2.52  TBP tile: close-up        3D: XP  16.64867   \n",
       "2                    3.16  TBP tile: close-up        3D: XP  24.25384   \n",
       "\n",
       "   tbp_lv_Aext  ...  onehot_41  onehot_42  onehot_43  onehot_44  onehot_45  \\\n",
       "0    20.007270  ...          0          0          0          1          0   \n",
       "1     9.657964  ...          0          0          1          0          0   \n",
       "2    19.937380  ...          0          0          0          0          0   \n",
       "\n",
       "   onehot_46  image_20240827234748  image_20240830205516  \\\n",
       "0          0             -0.501967             -2.714859   \n",
       "1          0             -4.002757             -5.223221   \n",
       "2          0             -3.307847             -4.123434   \n",
       "\n",
       "   image_20240831025049  image_20240902001446  \n",
       "0             -1.637331             -2.898454  \n",
       "1             -3.406027             -4.748341  \n",
       "2             -4.912541             -6.205088  \n",
       "\n",
       "[3 rows x 215 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1b9b89a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.188361Z",
     "iopub.status.busy": "2024-09-01T16:54:36.187692Z",
     "iopub.status.idle": "2024-09-01T16:54:36.402643Z",
     "shell.execute_reply": "2024-09-01T16:54:36.401802Z"
    },
    "papermill": {
     "duration": 0.245506,
     "end_time": "2024-09-01T16:54:36.404906",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.159400",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#they are detected at the first run\n",
    "least_important_features = ['onehot_32', 'onehot_6', 'onehot_33', 'onehot_30', 'onehot_26', 'onehot_22', 'onehot_36', 'onehot_4']\n",
    "#they are detected after the least_important_features are removed and it has increased cv score also so I add it\n",
    "#least_important_features_2 = ['onehot_17', 'onehot_42', 'onehot_29', 'onehot_13', 'onehot_25']\n",
    "#least_important_features += least_important_features_2\n",
    "df_train.drop(columns =least_important_features,inplace = True)\n",
    "for feature in least_important_features:\n",
    "    cat_cols.remove(feature)\n",
    "    feature_cols.remove(feature)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d000b648",
   "metadata": {
    "papermill": {
     "duration": 0.027318,
     "end_time": "2024-09-01T16:54:36.460575",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.433257",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# MODEL INITIALIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "02800b71",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.516370Z",
     "iopub.status.busy": "2024-09-01T16:54:36.515986Z",
     "iopub.status.idle": "2024-09-01T16:54:36.522039Z",
     "shell.execute_reply": "2024-09-01T16:54:36.521315Z"
    },
    "papermill": {
     "duration": 0.036252,
     "end_time": "2024-09-01T16:54:36.524017",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.487765",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import copy\n",
    "\n",
    "feature_cols_without_image_cols = copy.copy(feature_cols)\n",
    "feature_cols += image_cols\n",
    "\n",
    "class SelectColumns(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, columns):\n",
    "        self.columns = columns\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6cd1a906",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.580115Z",
     "iopub.status.busy": "2024-09-01T16:54:36.579846Z",
     "iopub.status.idle": "2024-09-01T16:54:36.587082Z",
     "shell.execute_reply": "2024-09-01T16:54:36.586288Z"
    },
    "papermill": {
     "duration": 0.037341,
     "end_time": "2024-09-01T16:54:36.588927",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.551586",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "    'objective':        'binary',\n",
    "    'verbosity':        -1,\n",
    "    'n_iter':           200,\n",
    "    'boosting_type':    'gbdt',\n",
    "    'random_state':     seed,\n",
    "    'lambda_l1':        0.08758718919397321,\n",
    "    'lambda_l2':        0.0039689175176025465,\n",
    "    'learning_rate':    0.03231007103195577,\n",
    "    'max_depth':        4,\n",
    "    'num_leaves':       103,\n",
    "    'colsample_bytree': 0.8329551585827726,\n",
    "    'colsample_bynode': 0.4025961355653304,\n",
    "    'bagging_fraction': 0.7738954452473223,\n",
    "    'bagging_freq':     4,\n",
    "    'min_data_in_leaf': 85,\n",
    "    'scale_pos_weight': 2.7984184778875543,\n",
    "}\n",
    "\n",
    "\n",
    "sampling_ratio = 0.01\n",
    "seed =42\n",
    "\n",
    "lgb_model1 = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('filter', SelectColumns(feature_cols_without_image_cols)),\n",
    "    ('classifier', lgb.LGBMClassifier(**lgb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d5cb702a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.644912Z",
     "iopub.status.busy": "2024-09-01T16:54:36.644638Z",
     "iopub.status.idle": "2024-09-01T16:54:36.651330Z",
     "shell.execute_reply": "2024-09-01T16:54:36.650504Z"
    },
    "papermill": {
     "duration": 0.036763,
     "end_time": "2024-09-01T16:54:36.653124",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.616361",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "    'objective':        'binary',\n",
    "    'verbosity':        -1,\n",
    "    'n_iter':           200,\n",
    "    'boosting_type':    'gbdt',\n",
    "    'random_state':     seed,\n",
    "    'lambda_l1':        0.08758718919397321,\n",
    "    'lambda_l2':        0.0039689175176025465,\n",
    "    'learning_rate':    0.03231007103195577,\n",
    "    'max_depth':        4,\n",
    "    'num_leaves':       103,\n",
    "    'colsample_bytree': 0.8329551585827726,\n",
    "    'colsample_bynode': 0.4025961355653304,\n",
    "    'bagging_fraction': 0.7738954452473223,\n",
    "    'bagging_freq':     4,\n",
    "    'min_data_in_leaf': 85,\n",
    "    'scale_pos_weight': 2.7984184778875543,\n",
    "}\n",
    "\n",
    "\n",
    "sampling_ratio = 0.01\n",
    "seed =42\n",
    "\n",
    "lgb_model2 = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('classifier', lgb.LGBMClassifier(**lgb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "97fa0201",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.708582Z",
     "iopub.status.busy": "2024-09-01T16:54:36.708312Z",
     "iopub.status.idle": "2024-09-01T16:54:36.716799Z",
     "shell.execute_reply": "2024-09-01T16:54:36.716075Z"
    },
    "papermill": {
     "duration": 0.03824,
     "end_time": "2024-09-01T16:54:36.718605",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.680365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cb_params = {\n",
    "    'loss_function':     'Logloss',\n",
    "    'iterations':        250,\n",
    "    'verbose':           False,\n",
    "    'random_state':      seed,\n",
    "    'max_depth':         7,\n",
    "    'learning_rate':     0.06936242010150652,\n",
    "    'scale_pos_weight':  2.6149345838209532,\n",
    "    'l2_leaf_reg':       6.216113851699493,\n",
    "    'subsample':         0.6249261779711819,\n",
    "    'min_data_in_leaf':  24,\n",
    "    'cat_features':      cat_cols,\n",
    "}\n",
    "cb_model1 = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('filter', SelectColumns(feature_cols_without_image_cols)),\n",
    "    ('classifier', cb.CatBoostClassifier(**cb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2e2b3eef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.774136Z",
     "iopub.status.busy": "2024-09-01T16:54:36.773867Z",
     "iopub.status.idle": "2024-09-01T16:54:36.779823Z",
     "shell.execute_reply": "2024-09-01T16:54:36.778988Z"
    },
    "papermill": {
     "duration": 0.035883,
     "end_time": "2024-09-01T16:54:36.781730",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.745847",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cb_params = {\n",
    "    'loss_function':     'Logloss',\n",
    "    'iterations':        250,\n",
    "    'verbose':           False,\n",
    "    'random_state':      seed,\n",
    "    'max_depth':         7,\n",
    "    'learning_rate':     0.06936242010150652,\n",
    "    'scale_pos_weight':  2.6149345838209532,\n",
    "    'l2_leaf_reg':       6.216113851699493,\n",
    "    'subsample':         0.6249261779711819,\n",
    "    'min_data_in_leaf':  24,\n",
    "    'cat_features':      cat_cols,\n",
    "}\n",
    "cb_model2 = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('classifier', cb.CatBoostClassifier(**cb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8453cca4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.838767Z",
     "iopub.status.busy": "2024-09-01T16:54:36.838077Z",
     "iopub.status.idle": "2024-09-01T16:54:36.844492Z",
     "shell.execute_reply": "2024-09-01T16:54:36.843654Z"
    },
    "papermill": {
     "duration": 0.037031,
     "end_time": "2024-09-01T16:54:36.846406",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.809375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'enable_categorical': True,\n",
    "    'tree_method':        'hist',\n",
    "    'random_state':       seed,\n",
    "    'learning_rate':      0.08501257473292347,\n",
    "    'lambda':             8.879624125465703,\n",
    "    'alpha':              0.6779926606782505,\n",
    "    'max_depth':          6,\n",
    "    'subsample':          0.6012681388711075,\n",
    "    'colsample_bytree':   0.8437772277074493,\n",
    "    'colsample_bylevel':  0.5476090898823716,\n",
    "    'colsample_bynode':   0.9928601203635129,\n",
    "    'scale_pos_weight':   3.29440313334688,\n",
    "}\n",
    "\n",
    "xgb_model1 = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('filter', SelectColumns(feature_cols_without_image_cols)),\n",
    "    ('classifier', xgb.XGBClassifier(**xgb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8941d888",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:36.902396Z",
     "iopub.status.busy": "2024-09-01T16:54:36.902093Z",
     "iopub.status.idle": "2024-09-01T16:54:36.908247Z",
     "shell.execute_reply": "2024-09-01T16:54:36.907448Z"
    },
    "papermill": {
     "duration": 0.036303,
     "end_time": "2024-09-01T16:54:36.910128",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.873825",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'enable_categorical': True,\n",
    "    'tree_method':        'hist',\n",
    "    'random_state':       seed,\n",
    "    'learning_rate':      0.08501257473292347,\n",
    "    'lambda':             8.879624125465703,\n",
    "    'alpha':              0.6779926606782505,\n",
    "    'max_depth':          6,\n",
    "    'subsample':          0.6012681388711075,\n",
    "    'colsample_bytree':   0.8437772277074493,\n",
    "    'colsample_bylevel':  0.5476090898823716,\n",
    "    'colsample_bynode':   0.9928601203635129,\n",
    "    'scale_pos_weight':   3.29440313334688,\n",
    "}\n",
    "\n",
    "xgb_model2 = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('classifier', xgb.XGBClassifier(**xgb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75244e3d",
   "metadata": {
    "papermill": {
     "duration": 0.027746,
     "end_time": "2024-09-01T16:54:36.965181",
     "exception": false,
     "start_time": "2024-09-01T16:54:36.937435",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# CROSS VALIDATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c5c8ac1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:37.268471Z",
     "iopub.status.busy": "2024-09-01T16:54:37.268157Z",
     "iopub.status.idle": "2024-09-01T16:54:37.277084Z",
     "shell.execute_reply": "2024-09-01T16:54:37.276208Z"
    },
    "papermill": {
     "duration": 0.039307,
     "end_time": "2024-09-01T16:54:37.279001",
     "exception": false,
     "start_time": "2024-09-01T16:54:37.239694",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['age_approx',\n",
       " 'clin_size_long_diam_mm',\n",
       " 'tbp_lv_A',\n",
       " 'tbp_lv_Aext',\n",
       " 'tbp_lv_B',\n",
       " 'tbp_lv_Bext',\n",
       " 'tbp_lv_C',\n",
       " 'tbp_lv_Cext',\n",
       " 'tbp_lv_H',\n",
       " 'tbp_lv_Hext',\n",
       " 'tbp_lv_L',\n",
       " 'tbp_lv_Lext',\n",
       " 'tbp_lv_areaMM2',\n",
       " 'tbp_lv_area_perim_ratio',\n",
       " 'tbp_lv_color_std_mean',\n",
       " 'tbp_lv_deltaA',\n",
       " 'tbp_lv_deltaB',\n",
       " 'tbp_lv_deltaL',\n",
       " 'tbp_lv_deltaLB',\n",
       " 'tbp_lv_deltaLBnorm',\n",
       " 'tbp_lv_eccentricity',\n",
       " 'tbp_lv_minorAxisMM',\n",
       " 'tbp_lv_nevi_confidence',\n",
       " 'tbp_lv_norm_border',\n",
       " 'tbp_lv_norm_color',\n",
       " 'tbp_lv_perimeterMM',\n",
       " 'tbp_lv_radial_color_std_max',\n",
       " 'tbp_lv_stdL',\n",
       " 'tbp_lv_stdLExt',\n",
       " 'tbp_lv_symm_2axis',\n",
       " 'tbp_lv_symm_2axis_angle',\n",
       " 'tbp_lv_x',\n",
       " 'tbp_lv_y',\n",
       " 'tbp_lv_z',\n",
       " 'lesion_size_ratio',\n",
       " 'lesion_shape_index',\n",
       " 'hue_contrast',\n",
       " 'luminance_contrast',\n",
       " 'lesion_color_difference',\n",
       " 'border_complexity',\n",
       " 'color_uniformity',\n",
       " 'position_distance_3d',\n",
       " 'perimeter_to_area_ratio',\n",
       " 'area_to_perimeter_ratio',\n",
       " 'lesion_visibility_score',\n",
       " 'symmetry_border_consistency',\n",
       " 'consistency_symmetry_border',\n",
       " 'color_consistency',\n",
       " 'consistency_color',\n",
       " 'size_age_interaction',\n",
       " 'hue_color_std_interaction',\n",
       " 'lesion_severity_index',\n",
       " 'shape_complexity_index',\n",
       " 'color_contrast_index',\n",
       " 'log_lesion_area',\n",
       " 'normalized_lesion_size',\n",
       " 'mean_hue_difference',\n",
       " 'std_dev_contrast',\n",
       " 'color_shape_composite_index',\n",
       " 'lesion_orientation_3d',\n",
       " 'overall_color_difference',\n",
       " 'symmetry_perimeter_interaction',\n",
       " 'comprehensive_lesion_index',\n",
       " 'color_variance_ratio',\n",
       " 'border_color_interaction',\n",
       " 'border_color_interaction_2',\n",
       " 'size_color_contrast_ratio',\n",
       " 'age_normalized_nevi_confidence',\n",
       " 'age_normalized_nevi_confidence_2',\n",
       " 'color_asymmetry_index',\n",
       " 'volume_approximation_3d',\n",
       " 'color_range',\n",
       " 'shape_color_consistency',\n",
       " 'border_length_ratio',\n",
       " 'age_size_symmetry_index',\n",
       " 'index_age_size_symmetry',\n",
       " 'age_approx_patient_norm',\n",
       " 'clin_size_long_diam_mm_patient_norm',\n",
       " 'tbp_lv_A_patient_norm',\n",
       " 'tbp_lv_Aext_patient_norm',\n",
       " 'tbp_lv_B_patient_norm',\n",
       " 'tbp_lv_Bext_patient_norm',\n",
       " 'tbp_lv_C_patient_norm',\n",
       " 'tbp_lv_Cext_patient_norm',\n",
       " 'tbp_lv_H_patient_norm',\n",
       " 'tbp_lv_Hext_patient_norm',\n",
       " 'tbp_lv_L_patient_norm',\n",
       " 'tbp_lv_Lext_patient_norm',\n",
       " 'tbp_lv_areaMM2_patient_norm',\n",
       " 'tbp_lv_area_perim_ratio_patient_norm',\n",
       " 'tbp_lv_color_std_mean_patient_norm',\n",
       " 'tbp_lv_deltaA_patient_norm',\n",
       " 'tbp_lv_deltaB_patient_norm',\n",
       " 'tbp_lv_deltaL_patient_norm',\n",
       " 'tbp_lv_deltaLB_patient_norm',\n",
       " 'tbp_lv_deltaLBnorm_patient_norm',\n",
       " 'tbp_lv_eccentricity_patient_norm',\n",
       " 'tbp_lv_minorAxisMM_patient_norm',\n",
       " 'tbp_lv_nevi_confidence_patient_norm',\n",
       " 'tbp_lv_norm_border_patient_norm',\n",
       " 'tbp_lv_norm_color_patient_norm',\n",
       " 'tbp_lv_perimeterMM_patient_norm',\n",
       " 'tbp_lv_radial_color_std_max_patient_norm',\n",
       " 'tbp_lv_stdL_patient_norm',\n",
       " 'tbp_lv_stdLExt_patient_norm',\n",
       " 'tbp_lv_symm_2axis_patient_norm',\n",
       " 'tbp_lv_symm_2axis_angle_patient_norm',\n",
       " 'tbp_lv_x_patient_norm',\n",
       " 'tbp_lv_y_patient_norm',\n",
       " 'tbp_lv_z_patient_norm',\n",
       " 'lesion_size_ratio_patient_norm',\n",
       " 'lesion_shape_index_patient_norm',\n",
       " 'hue_contrast_patient_norm',\n",
       " 'luminance_contrast_patient_norm',\n",
       " 'lesion_color_difference_patient_norm',\n",
       " 'border_complexity_patient_norm',\n",
       " 'color_uniformity_patient_norm',\n",
       " 'position_distance_3d_patient_norm',\n",
       " 'perimeter_to_area_ratio_patient_norm',\n",
       " 'area_to_perimeter_ratio_patient_norm',\n",
       " 'lesion_visibility_score_patient_norm',\n",
       " 'symmetry_border_consistency_patient_norm',\n",
       " 'consistency_symmetry_border_patient_norm',\n",
       " 'color_consistency_patient_norm',\n",
       " 'consistency_color_patient_norm',\n",
       " 'size_age_interaction_patient_norm',\n",
       " 'hue_color_std_interaction_patient_norm',\n",
       " 'lesion_severity_index_patient_norm',\n",
       " 'shape_complexity_index_patient_norm',\n",
       " 'color_contrast_index_patient_norm',\n",
       " 'log_lesion_area_patient_norm',\n",
       " 'normalized_lesion_size_patient_norm',\n",
       " 'mean_hue_difference_patient_norm',\n",
       " 'std_dev_contrast_patient_norm',\n",
       " 'color_shape_composite_index_patient_norm',\n",
       " 'lesion_orientation_3d_patient_norm',\n",
       " 'overall_color_difference_patient_norm',\n",
       " 'symmetry_perimeter_interaction_patient_norm',\n",
       " 'comprehensive_lesion_index_patient_norm',\n",
       " 'color_variance_ratio_patient_norm',\n",
       " 'border_color_interaction_patient_norm',\n",
       " 'border_color_interaction_2_patient_norm',\n",
       " 'size_color_contrast_ratio_patient_norm',\n",
       " 'age_normalized_nevi_confidence_patient_norm',\n",
       " 'age_normalized_nevi_confidence_2_patient_norm',\n",
       " 'color_asymmetry_index_patient_norm',\n",
       " 'volume_approximation_3d_patient_norm',\n",
       " 'color_range_patient_norm',\n",
       " 'shape_color_consistency_patient_norm',\n",
       " 'border_length_ratio_patient_norm',\n",
       " 'age_size_symmetry_index_patient_norm',\n",
       " 'index_age_size_symmetry_patient_norm',\n",
       " 'count_per_patient',\n",
       " 'onehot_0',\n",
       " 'onehot_1',\n",
       " 'onehot_2',\n",
       " 'onehot_3',\n",
       " 'onehot_5',\n",
       " 'onehot_7',\n",
       " 'onehot_8',\n",
       " 'onehot_9',\n",
       " 'onehot_10',\n",
       " 'onehot_11',\n",
       " 'onehot_12',\n",
       " 'onehot_13',\n",
       " 'onehot_14',\n",
       " 'onehot_15',\n",
       " 'onehot_16',\n",
       " 'onehot_17',\n",
       " 'onehot_18',\n",
       " 'onehot_19',\n",
       " 'onehot_20',\n",
       " 'onehot_21',\n",
       " 'onehot_23',\n",
       " 'onehot_24',\n",
       " 'onehot_25',\n",
       " 'onehot_27',\n",
       " 'onehot_28',\n",
       " 'onehot_29',\n",
       " 'onehot_31',\n",
       " 'onehot_34',\n",
       " 'onehot_35',\n",
       " 'onehot_37',\n",
       " 'onehot_38',\n",
       " 'onehot_39',\n",
       " 'onehot_40',\n",
       " 'onehot_41',\n",
       " 'onehot_42',\n",
       " 'onehot_43',\n",
       " 'onehot_44',\n",
       " 'onehot_45',\n",
       " 'onehot_46',\n",
       " 'image_20240827234748',\n",
       " 'image_20240830205516',\n",
       " 'image_20240831025049',\n",
       " 'image_20240902001446']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "aa7f50d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:54:37.335493Z",
     "iopub.status.busy": "2024-09-01T16:54:37.335199Z",
     "iopub.status.idle": "2024-09-01T16:59:27.209614Z",
     "shell.execute_reply": "2024-09-01T16:59:27.208644Z"
    },
    "papermill": {
     "duration": 289.934327,
     "end_time": "2024-09-01T16:59:27.241263",
     "exception": false,
     "start_time": "2024-09-01T16:54:37.306936",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 score: 0.19238429832487994\n",
      "Fold 0 score: 0.1922283414328287\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 score: 0.18112400983688107\n",
      "Fold 1 score: 0.18089984532318618\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 2 score: 0.17752934338320767\n",
      "Fold 2 score: 0.17732962274275868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 3 score: 0.1848868608576072\n",
      "Fold 3 score: 0.18467883840211652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4 score: 0.18054818633950864\n",
      "Fold 4 score: 0.18035864733854323\n"
     ]
    }
   ],
   "source": [
    "scores_filtered = []\n",
    "scores = []\n",
    "estimators = []\n",
    "\n",
    "all_val_targets, all_val_outputs = [], []\n",
    "oof_predictions = np.zeros(len(df_train))\n",
    "for fold in cfg_global.train_folds:\n",
    "\n",
    "    # model\n",
    "    estimator = VotingClassifier([\n",
    "        ('lgb1', lgb_model1),\n",
    "        ('lgb2', lgb_model2),\n",
    "        # ('cb1', cb_model1),\n",
    "        ('cb2', cb_model2),\n",
    "        # ('xgb1', xgb_model1),\n",
    "        ('xgb2', xgb_model2),\n",
    "    ], voting='soft')\n",
    "\n",
    "    # Split data for current fold\n",
    "    exclude_filter = ~df_train[\"isic_id\"].isin(exclude_isic_ids)\n",
    "    X_train = df_train[exclude_filter & (df_train[\"fold\"] != fold)][feature_cols]\n",
    "    X_test = df_train[exclude_filter & (df_train[\"fold\"] == fold)][feature_cols]\n",
    "    y_train = df_train[exclude_filter & (df_train[\"fold\"] != fold)][target_col]\n",
    "    y_test = df_train[exclude_filter & (df_train[\"fold\"] == fold)][target_col]\n",
    "\n",
    "    # モデルのトレーニング\n",
    "    estimator.fit(X_train, y_train)\n",
    "\n",
    "    # モデルの評価\n",
    "    y_hat = estimator.predict_proba(X_test)[:, 1]\n",
    "    score_filtered = custom_metric(y_hat, y_test.values)\n",
    "\n",
    "    # OOFを再計算（すべてのデータポイントで）\n",
    "    X_test = df_train[df_train[\"fold\"] == fold][feature_cols]\n",
    "    y_test = df_train[df_train[\"fold\"] == fold][target_col]\n",
    "    y_hat = estimator.predict_proba(X_test)[:, 1]\n",
    "    score = custom_metric(y_hat, y_test.values)\n",
    "    oof_predictions[y_test.index] = y_hat\n",
    "    all_val_targets.extend(y_test)\n",
    "    all_val_outputs.extend(y_hat)\n",
    "\n",
    "    # スコアとestimatorをリストに保存\n",
    "    scores_filtered.append(score_filtered)\n",
    "    scores.append(score)\n",
    "    estimators.append(estimator)\n",
    "\n",
    "    print(f\"Fold {fold} score: {score_filtered}\")\n",
    "    print(f\"Fold {fold} score: {score}\")\n",
    "\n",
    "all_val_outputs = np.array(all_val_outputs)\n",
    "all_val_targets = np.array(all_val_targets)\n",
    "oof_predictions = np.array(oof_predictions)\n",
    "# print(custom_metric(all_val_outputs, all_val_targets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d6e74533",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:27.302579Z",
     "iopub.status.busy": "2024-09-01T16:59:27.302227Z",
     "iopub.status.idle": "2024-09-01T16:59:27.312959Z",
     "shell.execute_reply": "2024-09-01T16:59:27.312256Z"
    },
    "papermill": {
     "duration": 0.04315,
     "end_time": "2024-09-01T16:59:27.314924",
     "exception": false,
     "start_time": "2024-09-01T16:59:27.271774",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "oof_df = pd.DataFrame(\n",
    "    {\n",
    "        \"isic_id\": df_train[\"isic_id\"],\n",
    "        \"target\": df_train[\"target\"],\n",
    "        \"fold\": df_train[\"fold\"],\n",
    "        \"oof_prediction\": oof_predictions,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "77f75823",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:27.373799Z",
     "iopub.status.busy": "2024-09-01T16:59:27.373531Z",
     "iopub.status.idle": "2024-09-01T16:59:27.859470Z",
     "shell.execute_reply": "2024-09-01T16:59:27.858552Z"
    },
    "papermill": {
     "duration": 0.51798,
     "end_time": "2024-09-01T16:59:27.861702",
     "exception": false,
     "start_time": "2024-09-01T16:59:27.343722",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------\n",
      "OOF filtered score\n",
      "0.1833008201972076\n",
      "--------------------\n",
      "OOF all score\n",
      "0.1831055452976138\n",
      "--------------------\n",
      "OOF filtered score folds mean\n",
      "0.1832945397484169\n",
      "--------------------\n",
      "OOF score folds mean\n",
      "0.18309905904788665\n"
     ]
    }
   ],
   "source": [
    "print(\"-\"*20)\n",
    "print(\"OOF filtered score\")\n",
    "y_hat = oof_df[~oof_df[\"isic_id\"].isin(exclude_isic_ids)][\"oof_prediction\"]\n",
    "y_true = oof_df[~oof_df[\"isic_id\"].isin(exclude_isic_ids)][\"target\"]\n",
    "print(custom_metric(y_hat,y_true))\n",
    "\n",
    "print(\"-\"*20)\n",
    "print(\"OOF all score\")\n",
    "y_hat = oof_df[\"oof_prediction\"]\n",
    "y_true = oof_df[\"target\"]\n",
    "print(custom_metric(y_hat, y_true))\n",
    "\n",
    "print(\"-\"*20)\n",
    "print(\"OOF filtered score folds mean\")\n",
    "print(np.mean(scores_filtered))\n",
    "print(\"-\"*20)\n",
    "print(\"OOF score folds mean\")\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f826526",
   "metadata": {
    "papermill": {
     "duration": 0.028674,
     "end_time": "2024-09-01T16:59:27.920254",
     "exception": false,
     "start_time": "2024-09-01T16:59:27.891580",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# HYPERPARAMETER TUNING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "87a51cea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:27.979400Z",
     "iopub.status.busy": "2024-09-01T16:59:27.979060Z",
     "iopub.status.idle": "2024-09-01T16:59:27.983003Z",
     "shell.execute_reply": "2024-09-01T16:59:27.982237Z"
    },
    "papermill": {
     "duration": 0.035662,
     "end_time": "2024-09-01T16:59:27.984980",
     "exception": false,
     "start_time": "2024-09-01T16:59:27.949318",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DO_TUNING = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5c9fc473",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.043863Z",
     "iopub.status.busy": "2024-09-01T16:59:28.043587Z",
     "iopub.status.idle": "2024-09-01T16:59:28.052315Z",
     "shell.execute_reply": "2024-09-01T16:59:28.051460Z"
    },
    "papermill": {
     "duration": 0.040147,
     "end_time": "2024-09-01T16:59:28.054090",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.013943",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cb_objective(trial):\n",
    "    params = {\n",
    "        'loss_function':     'Logloss',\n",
    "        'iterations':        200,\n",
    "        'verbose':           False,\n",
    "        'random_state':      seed,\n",
    "        'learning_rate':     trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n",
    "        'max_depth':         trial.suggest_int('max_depth', 4, 8),\n",
    "        'l2_leaf_reg':       trial.suggest_float('l2_leaf_reg', 1e-3, 10.0, log=True),\n",
    "        'subsample':         trial.suggest_float('subsample', 0.4, 1.0),\n",
    "        'colsample_bylevel': trial.suggest_float('colsample_bylevel', 0.4, 1.0),\n",
    "        'min_data_in_leaf':  trial.suggest_int('min_data_in_leaf', 5, 100),\n",
    "        'scale_pos_weight':  trial.suggest_float('scale_pos_weight', 0.8, 4.0),\n",
    "        'bootstrap_type':    'Bayesian',  # Optional: depending on your use case, you may want to tune this as well\n",
    "    }\n",
    "\n",
    "    estimator = Pipeline([\n",
    "        ('sampler', RandomUnderSampler(sampling_strategy=sampling_ratio)),\n",
    "        ('classifier', cb.CatBoostClassifier(**params)),\n",
    "    ])\n",
    "\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "    groups = df_train[group_col]\n",
    "    cv = StratifiedGroupKFold(5, shuffle=True)\n",
    "\n",
    "    val_score = cross_val_score(\n",
    "        estimator=estimator,\n",
    "        X=X, y=y,\n",
    "        cv=cv,\n",
    "        groups=groups,\n",
    "        scoring=custom_metric,\n",
    "    )\n",
    "\n",
    "    return np.mean(val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5311a960",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.114175Z",
     "iopub.status.busy": "2024-09-01T16:59:28.113532Z",
     "iopub.status.idle": "2024-09-01T16:59:28.123247Z",
     "shell.execute_reply": "2024-09-01T16:59:28.122407Z"
    },
    "papermill": {
     "duration": 0.04153,
     "end_time": "2024-09-01T16:59:28.125087",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.083557",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def lgb_objective(trial):\n",
    "    params = {\n",
    "        'objective':         'binary',\n",
    "        'verbosity':         -1,\n",
    "        'n_iter': 200,\n",
    "        'boosting_type':  'gbdt',\n",
    "        'lambda_l1':         trial.suggest_float('lambda_l1', 1e-3, 10.0, log=True),\n",
    "        'lambda_l2':         trial.suggest_float('lambda_l2', 1e-3, 10.0, log=True),\n",
    "        'learning_rate':     trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n",
    "        'max_depth':         trial.suggest_int('max_depth', 4, 8),\n",
    "        'num_leaves':        trial.suggest_int('num_leaves', 16, 256),\n",
    "        'colsample_bytree':  trial.suggest_float('colsample_bytree', 0.4, 1.0),\n",
    "        'colsample_bynode':  trial.suggest_float('colsample_bynode', 0.4, 1.0),\n",
    "        'bagging_fraction':  trial.suggest_float('bagging_fraction', 0.4, 1.0),\n",
    "        'bagging_freq':      trial.suggest_int('bagging_freq', 1, 7),\n",
    "        'min_data_in_leaf':  trial.suggest_int('min_data_in_leaf', 5, 100),\n",
    "        'scale_pos_weight' : trial.suggest_float('scale_pos_weight', 0.8, 4.0),\n",
    "    }\n",
    "\n",
    "    estimator = Pipeline([\n",
    "        ('sampler', RandomUnderSampler(sampling_strategy=sampling_ratio)),\n",
    "        ('classifier', lgb.LGBMClassifier(**params)),\n",
    "    ])\n",
    "\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "    groups = df_train[group_col]\n",
    "    cv = StratifiedGroupKFold(5, shuffle=True)\n",
    "\n",
    "    val_score = cross_val_score(\n",
    "        estimator=estimator,\n",
    "        X=X, y=y,\n",
    "        cv=cv,\n",
    "        groups=groups,\n",
    "        scoring=custom_metric,\n",
    "    )\n",
    "\n",
    "    return np.mean(val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f970e084",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.184652Z",
     "iopub.status.busy": "2024-09-01T16:59:28.184159Z",
     "iopub.status.idle": "2024-09-01T16:59:28.192657Z",
     "shell.execute_reply": "2024-09-01T16:59:28.191822Z"
    },
    "papermill": {
     "duration": 0.040232,
     "end_time": "2024-09-01T16:59:28.194521",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.154289",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def xgb_objective(trial):\n",
    "    params = {\n",
    "        'objective':          'binary:logistic',\n",
    "        'n_estimators':       200,\n",
    "        'tree_method':        'hist',\n",
    "        'random_state':       seed,\n",
    "        'learning_rate':      trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n",
    "        'max_depth':          trial.suggest_int('max_depth', 4, 8),\n",
    "        'lambda':             trial.suggest_float('lambda', 1e-3, 10.0, log=True),\n",
    "        'alpha':              trial.suggest_float('alpha', 1e-3, 10.0, log=True),\n",
    "        'subsample':          trial.suggest_float('subsample', 0.4, 1.0),\n",
    "        'colsample_bytree':   trial.suggest_float('colsample_bytree', 0.4, 1.0),\n",
    "        'colsample_bynode':   trial.suggest_float('colsample_bynode', 0.4, 1.0),\n",
    "        'scale_pos_weight':   trial.suggest_float('scale_pos_weight', 0.8, 4.0),\n",
    "    }\n",
    "\n",
    "    estimator = Pipeline([\n",
    "        ('sampler', RandomUnderSampler(sampling_strategy=sampling_ratio)),\n",
    "        ('classifier', xgb.XGBClassifier(**params)),\n",
    "    ])\n",
    "\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "    groups = df_train[group_col]\n",
    "    cv = StratifiedGroupKFold(5, shuffle=True)\n",
    "\n",
    "    val_score = cross_val_score(\n",
    "        estimator=estimator,\n",
    "        X=X, y=y,\n",
    "        cv=cv,\n",
    "        groups=groups,\n",
    "        scoring=custom_metric,\n",
    "    )\n",
    "\n",
    "    return np.mean(val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "718100f5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.253762Z",
     "iopub.status.busy": "2024-09-01T16:59:28.253496Z",
     "iopub.status.idle": "2024-09-01T16:59:28.258308Z",
     "shell.execute_reply": "2024-09-01T16:59:28.257535Z"
    },
    "papermill": {
     "duration": 0.036638,
     "end_time": "2024-09-01T16:59:28.260091",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.223453",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # LightGBM\n",
    "    start_time = time.time()\n",
    "    study_lgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=seed))\n",
    "    study_lgb.optimize(objective_lgb, n_trials=100)\n",
    "    end_time = time.time()\n",
    "    elapsed_time_lgb = end_time - start_time\n",
    "    print(f\"LightGBM tuning took {elapsed_time_lgb:.2f} seconds.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "6ba0a88f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.319932Z",
     "iopub.status.busy": "2024-09-01T16:59:28.319659Z",
     "iopub.status.idle": "2024-09-01T16:59:28.324596Z",
     "shell.execute_reply": "2024-09-01T16:59:28.323724Z"
    },
    "papermill": {
     "duration": 0.036408,
     "end_time": "2024-09-01T16:59:28.326434",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.290026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # CatBoost\n",
    "    start_time = time.time()\n",
    "    study_cb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=seed))\n",
    "    study_cb.optimize(objective_cb, n_trials=100)\n",
    "    end_time = time.time()\n",
    "    elapsed_time_cb = end_time - start_time\n",
    "    print(f\"CatBoost tuning took {elapsed_time_cb:.2f} seconds.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "746b2294",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.386719Z",
     "iopub.status.busy": "2024-09-01T16:59:28.386139Z",
     "iopub.status.idle": "2024-09-01T16:59:28.391160Z",
     "shell.execute_reply": "2024-09-01T16:59:28.390323Z"
    },
    "papermill": {
     "duration": 0.037153,
     "end_time": "2024-09-01T16:59:28.393006",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.355853",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # XGBoost\n",
    "    start_time = time.time()\n",
    "    study_xgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=seed))\n",
    "    study_xgb.optimize(objective_xgb, n_trials=100)\n",
    "    end_time = time.time()\n",
    "    elapsed_time_xgb = end_time - start_time\n",
    "    print(f\"XGBoost tuning took {elapsed_time_xgb:.2f} seconds.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ae072215",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.451703Z",
     "iopub.status.busy": "2024-09-01T16:59:28.451426Z",
     "iopub.status.idle": "2024-09-01T16:59:28.455833Z",
     "shell.execute_reply": "2024-09-01T16:59:28.454980Z"
    },
    "papermill": {
     "duration": 0.035856,
     "end_time": "2024-09-01T16:59:28.457659",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.421803",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # Print best parameters for each study\n",
    "    print(\"Best LGBM trial:\", study_lgb.best_trial)\n",
    "    print(\"Best CatBoost trial:\", study_cb.best_trial)\n",
    "    print(\"Best XGBoost trial:\", study_xgb.best_trial)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31cd0021",
   "metadata": {
    "papermill": {
     "duration": 0.028652,
     "end_time": "2024-09-01T16:59:28.515290",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.486638",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TRAINING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6f8b578a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T16:59:28.574656Z",
     "iopub.status.busy": "2024-09-01T16:59:28.574046Z",
     "iopub.status.idle": "2024-09-01T17:00:34.846632Z",
     "shell.execute_reply": "2024-09-01T17:00:34.845740Z"
    },
    "papermill": {
     "duration": 66.334994,
     "end_time": "2024-09-01T17:00:34.879429",
     "exception": false,
     "start_time": "2024-09-01T16:59:28.544435",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>VotingClassifier(estimators=[(&#x27;lgb1&#x27;,\n",
       "                              Pipeline(steps=[(&#x27;sampler_1&#x27;,\n",
       "                                               RandomOverSampler(random_state=42,\n",
       "                                                                 sampling_strategy=0.003)),\n",
       "                                              (&#x27;sampler_2&#x27;,\n",
       "                                               RandomUnderSampler(random_state=42,\n",
       "                                                                  sampling_strategy=0.01)),\n",
       "                                              (&#x27;filter&#x27;,\n",
       "                                               SelectColumns(columns=[&#x27;age_approx&#x27;,\n",
       "                                                                      &#x27;clin_size_long_diam_mm&#x27;,\n",
       "                                                                      &#x27;tbp_lv_A&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Aext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_B&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Bext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_C&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Cext&#x27;,\n",
       "                                                                      &#x27;tbp_...\n",
       "                                                             importance_type=None,\n",
       "                                                             interaction_constraints=None,\n",
       "                                                             lambda=8.879624125465703,\n",
       "                                                             learning_rate=0.08501257473292347,\n",
       "                                                             max_bin=None,\n",
       "                                                             max_cat_threshold=None,\n",
       "                                                             max_cat_to_onehot=None,\n",
       "                                                             max_delta_step=None,\n",
       "                                                             max_depth=6,\n",
       "                                                             max_leaves=None,\n",
       "                                                             min_child_weight=None,\n",
       "                                                             missing=nan,\n",
       "                                                             monotone_constraints=None,\n",
       "                                                             multi_strategy=None,\n",
       "                                                             n_estimators=None,\n",
       "                                                             n_jobs=None, ...))]))],\n",
       "                 voting=&#x27;soft&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">VotingClassifier</label><div class=\"sk-toggleable__content\"><pre>VotingClassifier(estimators=[(&#x27;lgb1&#x27;,\n",
       "                              Pipeline(steps=[(&#x27;sampler_1&#x27;,\n",
       "                                               RandomOverSampler(random_state=42,\n",
       "                                                                 sampling_strategy=0.003)),\n",
       "                                              (&#x27;sampler_2&#x27;,\n",
       "                                               RandomUnderSampler(random_state=42,\n",
       "                                                                  sampling_strategy=0.01)),\n",
       "                                              (&#x27;filter&#x27;,\n",
       "                                               SelectColumns(columns=[&#x27;age_approx&#x27;,\n",
       "                                                                      &#x27;clin_size_long_diam_mm&#x27;,\n",
       "                                                                      &#x27;tbp_lv_A&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Aext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_B&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Bext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_C&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Cext&#x27;,\n",
       "                                                                      &#x27;tbp_...\n",
       "                                                             importance_type=None,\n",
       "                                                             interaction_constraints=None,\n",
       "                                                             lambda=8.879624125465703,\n",
       "                                                             learning_rate=0.08501257473292347,\n",
       "                                                             max_bin=None,\n",
       "                                                             max_cat_threshold=None,\n",
       "                                                             max_cat_to_onehot=None,\n",
       "                                                             max_delta_step=None,\n",
       "                                                             max_depth=6,\n",
       "                                                             max_leaves=None,\n",
       "                                                             min_child_weight=None,\n",
       "                                                             missing=nan,\n",
       "                                                             monotone_constraints=None,\n",
       "                                                             multi_strategy=None,\n",
       "                                                             n_estimators=None,\n",
       "                                                             n_jobs=None, ...))]))],\n",
       "                 voting=&#x27;soft&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>lgb1</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SelectColumns</label><div class=\"sk-toggleable__content\"><pre>SelectColumns(columns=[&#x27;age_approx&#x27;, &#x27;clin_size_long_diam_mm&#x27;, &#x27;tbp_lv_A&#x27;,\n",
       "                       &#x27;tbp_lv_Aext&#x27;, &#x27;tbp_lv_B&#x27;, &#x27;tbp_lv_Bext&#x27;, &#x27;tbp_lv_C&#x27;,\n",
       "                       &#x27;tbp_lv_Cext&#x27;, &#x27;tbp_lv_H&#x27;, &#x27;tbp_lv_Hext&#x27;, &#x27;tbp_lv_L&#x27;,\n",
       "                       &#x27;tbp_lv_Lext&#x27;, &#x27;tbp_lv_areaMM2&#x27;,\n",
       "                       &#x27;tbp_lv_area_perim_ratio&#x27;, &#x27;tbp_lv_color_std_mean&#x27;,\n",
       "                       &#x27;tbp_lv_deltaA&#x27;, &#x27;tbp_lv_deltaB&#x27;, &#x27;tbp_lv_deltaL&#x27;,\n",
       "                       &#x27;tbp_lv_deltaLB&#x27;, &#x27;tbp_lv_deltaLBnorm&#x27;,\n",
       "                       &#x27;tbp_lv_eccentricity&#x27;, &#x27;tbp_lv_minorAxisMM&#x27;,\n",
       "                       &#x27;tbp_lv_nevi_confidence&#x27;, &#x27;tbp_lv_norm_border&#x27;,\n",
       "                       &#x27;tbp_lv_norm_color&#x27;, &#x27;tbp_lv_perimeterMM&#x27;,\n",
       "                       &#x27;tbp_lv_radial_color_std_max&#x27;, &#x27;tbp_lv_stdL&#x27;,\n",
       "                       &#x27;tbp_lv_stdLExt&#x27;, &#x27;tbp_lv_symm_2axis&#x27;, ...])</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(bagging_fraction=0.7738954452473223, bagging_freq=4,\n",
       "               colsample_bynode=0.4025961355653304,\n",
       "               colsample_bytree=0.8329551585827726,\n",
       "               lambda_l1=0.08758718919397321, lambda_l2=0.0039689175176025465,\n",
       "               learning_rate=0.03231007103195577, max_depth=4,\n",
       "               min_data_in_leaf=85, n_iter=200, num_leaves=103,\n",
       "               objective=&#x27;binary&#x27;, random_state=42,\n",
       "               scale_pos_weight=2.7984184778875543, verbosity=-1)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>lgb2</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(bagging_fraction=0.7738954452473223, bagging_freq=4,\n",
       "               colsample_bynode=0.4025961355653304,\n",
       "               colsample_bytree=0.8329551585827726,\n",
       "               lambda_l1=0.08758718919397321, lambda_l2=0.0039689175176025465,\n",
       "               learning_rate=0.03231007103195577, max_depth=4,\n",
       "               min_data_in_leaf=85, n_iter=200, num_leaves=103,\n",
       "               objective=&#x27;binary&#x27;, random_state=42,\n",
       "               scale_pos_weight=2.7984184778875543, verbosity=-1)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>cb2</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CatBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>&lt;catboost.core.CatBoostClassifier object at 0x7f8679d29390&gt;</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>xgb2</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-13\" type=\"checkbox\" ><label for=\"sk-estimator-id-13\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-14\" type=\"checkbox\" ><label for=\"sk-estimator-id-14\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(alpha=0.6779926606782505, base_score=None, booster=None,\n",
       "              callbacks=None, colsample_bylevel=0.5476090898823716,\n",
       "              colsample_bynode=0.9928601203635129,\n",
       "              colsample_bytree=0.8437772277074493, device=None,\n",
       "              early_stopping_rounds=None, enable_categorical=True,\n",
       "              eval_metric=None, feature_types=None, gamma=None,\n",
       "              grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, lambda=8.879624125465703,\n",
       "              learning_rate=0.08501257473292347, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=6, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              multi_strategy=None, n_estimators=None, n_jobs=None, ...)</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "VotingClassifier(estimators=[('lgb1',\n",
       "                              Pipeline(steps=[('sampler_1',\n",
       "                                               RandomOverSampler(random_state=42,\n",
       "                                                                 sampling_strategy=0.003)),\n",
       "                                              ('sampler_2',\n",
       "                                               RandomUnderSampler(random_state=42,\n",
       "                                                                  sampling_strategy=0.01)),\n",
       "                                              ('filter',\n",
       "                                               SelectColumns(columns=['age_approx',\n",
       "                                                                      'clin_size_long_diam_mm',\n",
       "                                                                      'tbp_lv_A',\n",
       "                                                                      'tbp_lv_Aext',\n",
       "                                                                      'tbp_lv_B',\n",
       "                                                                      'tbp_lv_Bext',\n",
       "                                                                      'tbp_lv_C',\n",
       "                                                                      'tbp_lv_Cext',\n",
       "                                                                      'tbp_...\n",
       "                                                             importance_type=None,\n",
       "                                                             interaction_constraints=None,\n",
       "                                                             lambda=8.879624125465703,\n",
       "                                                             learning_rate=0.08501257473292347,\n",
       "                                                             max_bin=None,\n",
       "                                                             max_cat_threshold=None,\n",
       "                                                             max_cat_to_onehot=None,\n",
       "                                                             max_delta_step=None,\n",
       "                                                             max_depth=6,\n",
       "                                                             max_leaves=None,\n",
       "                                                             min_child_weight=None,\n",
       "                                                             missing=nan,\n",
       "                                                             monotone_constraints=None,\n",
       "                                                             multi_strategy=None,\n",
       "                                                             n_estimators=None,\n",
       "                                                             n_jobs=None, ...))]))],\n",
       "                 voting='soft')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = df_train[feature_cols], df_train[target_col]\n",
    "\n",
    "final_estimator = VotingClassifier([\n",
    "    ('lgb1', lgb_model1),\n",
    "    ('lgb2', lgb_model2),\n",
    "    # ('cb1', cb_model1),\n",
    "    ('cb2', cb_model2),\n",
    "    # ('xgb1', xgb_model1),\n",
    "    ('xgb2', xgb_model2),\n",
    "], voting='soft')\n",
    "\n",
    "final_estimator.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3ee3414",
   "metadata": {
    "papermill": {
     "duration": 0.029207,
     "end_time": "2024-09-01T17:00:34.938248",
     "exception": false,
     "start_time": "2024-09-01T17:00:34.909041",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LOOKING FOR FEATURE IMPORTANCE(lgb + xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "ade512aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:34.998482Z",
     "iopub.status.busy": "2024-09-01T17:00:34.998158Z",
     "iopub.status.idle": "2024-09-01T17:00:35.002150Z",
     "shell.execute_reply": "2024-09-01T17:00:35.001319Z"
    },
    "papermill": {
     "duration": 0.036493,
     "end_time": "2024-09-01T17:00:35.004054",
     "exception": false,
     "start_time": "2024-09-01T17:00:34.967561",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DO_FEATURE_IMPORTANCE_MODELS = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "3ded7a32",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.063704Z",
     "iopub.status.busy": "2024-09-01T17:00:35.063257Z",
     "iopub.status.idle": "2024-09-01T17:00:35.069420Z",
     "shell.execute_reply": "2024-09-01T17:00:35.068579Z"
    },
    "papermill": {
     "duration": 0.038022,
     "end_time": "2024-09-01T17:00:35.071259",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.033237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "    lgb_model = estimator.named_estimators_['lgb'].named_steps['classifier']\n",
    "    lgb_feature_importance = lgb_model.booster_.feature_importance(importance_type='gain')\n",
    "    lgb_feature_importance_df = pd.DataFrame({\n",
    "        'feature': X.columns,\n",
    "        'importance': lgb_feature_importance\n",
    "    }).sort_values(by='importance', ascending=False)\n",
    "\n",
    "\n",
    "    xgb_model = estimator.named_estimators_['xgb'].named_steps['classifier']\n",
    "    xgb_feature_importance = xgb_model.get_booster().get_score(importance_type='weight')\n",
    "    xgb_feature_importance_df = pd.DataFrame({\n",
    "        'feature': xgb_feature_importance.keys(),\n",
    "        'importance': xgb_feature_importance.values()\n",
    "    }).sort_values(by='importance', ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1000831c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.131261Z",
     "iopub.status.busy": "2024-09-01T17:00:35.131018Z",
     "iopub.status.idle": "2024-09-01T17:00:35.134910Z",
     "shell.execute_reply": "2024-09-01T17:00:35.134150Z"
    },
    "papermill": {
     "duration": 0.036106,
     "end_time": "2024-09-01T17:00:35.136669",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.100563",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "\n",
    "    print(lgb_feature_importance_df)\n",
    "    print(xgb_feature_importance_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c8496fe",
   "metadata": {
    "papermill": {
     "duration": 0.029709,
     "end_time": "2024-09-01T17:00:35.195897",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.166188",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LEAST IMPORTANT FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "beff4d45",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.255667Z",
     "iopub.status.busy": "2024-09-01T17:00:35.255395Z",
     "iopub.status.idle": "2024-09-01T17:00:35.260396Z",
     "shell.execute_reply": "2024-09-01T17:00:35.259582Z"
    },
    "papermill": {
     "duration": 0.037054,
     "end_time": "2024-09-01T17:00:35.262227",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.225173",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "\n",
    "    # Assuming lgb_feature_importance_df is already created and contains the feature importances\n",
    "    least_important_lgb = lgb_feature_importance_df.sort_values(by='importance').head(24)\n",
    "\n",
    "    print(\"Least Important Features in LightGBM:\")\n",
    "    print(least_important_lgb)\n",
    "\n",
    "    # Convert the xgb_feature_importance to a DataFrame for easier manipulation\n",
    "    least_important_xgb = xgb_feature_importance_df.sort_values(by = \"importance\").head(6)\n",
    "\n",
    "\n",
    "    print(\"\\nLeast Important Features in XGBoost:\")\n",
    "    print(least_important_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d1f36a93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.323034Z",
     "iopub.status.busy": "2024-09-01T17:00:35.322738Z",
     "iopub.status.idle": "2024-09-01T17:00:35.327589Z",
     "shell.execute_reply": "2024-09-01T17:00:35.326755Z"
    },
    "papermill": {
     "duration": 0.037611,
     "end_time": "2024-09-01T17:00:35.329528",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.291917",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "\n",
    "    # Extract the least important feature names from both LightGBM and XGBoost\n",
    "    least_important_lgb_features = least_important_lgb['feature'].tolist()\n",
    "    least_important_xgb_features = least_important_xgb['feature'].tolist()\n",
    "\n",
    "    # Find the intersection of the two lists\n",
    "    common_least_important_features = list(set(least_important_lgb_features) & set(least_important_xgb_features))\n",
    "\n",
    "    print(\"Common Least Important Features in Both LightGBM and XGBoost:\")\n",
    "    print(common_least_important_features)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dd492ce",
   "metadata": {
    "papermill": {
     "duration": 0.029318,
     "end_time": "2024-09-01T17:00:35.388912",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.359594",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LOOKING FOR FEATURE IMPORTANCE(Tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "5c6df87a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.449056Z",
     "iopub.status.busy": "2024-09-01T17:00:35.448813Z",
     "iopub.status.idle": "2024-09-01T17:00:35.452593Z",
     "shell.execute_reply": "2024-09-01T17:00:35.451729Z"
    },
    "papermill": {
     "duration": 0.036195,
     "end_time": "2024-09-01T17:00:35.454569",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.418374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DO_FEATURE_IMPORTANCE_TEST = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "0cac1cd1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.516447Z",
     "iopub.status.busy": "2024-09-01T17:00:35.515770Z",
     "iopub.status.idle": "2024-09-01T17:00:35.526707Z",
     "shell.execute_reply": "2024-09-01T17:00:35.525848Z"
    },
    "papermill": {
     "duration": 0.044334,
     "end_time": "2024-09-01T17:00:35.528471",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.484137",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_TEST:\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "\n",
    "    # Separate continuous and categorical features\n",
    "    continuous_features = num_cols + norm_cols + new_num_cols\n",
    "    # Fill null values of continuous features with their median values\n",
    "    X[continuous_features] = X[continuous_features].apply(lambda x: x.fillna(x.median()))\n",
    "\n",
    "\n",
    "    # Correlation Matrix for continuous features\n",
    "    corr_matrix = df_train[continuous_features + ['target']].corr()\n",
    "    threshold = 0.01\n",
    "    relevant_features_corr = corr_matrix[abs(corr_matrix['target']) > threshold].index\n",
    "    selected_features_corr = relevant_features_corr.drop('target')\n",
    "    print(\"Selected continuous features based on correlation threshold:\")\n",
    "    print(selected_features_corr)\n",
    "    print(len(selected_features_corr))\n",
    "\n",
    "    # Chi-Square Test for categorical features\n",
    "    chi2_selector = SelectKBest(chi2, k=15)\n",
    "    chi2_selector.fit_transform(X[cat_cols], y)\n",
    "    selected_features_chi2 = X[cat_cols].columns[chi2_selector.get_support()]\n",
    "    print(\"Selected categorical features based on Chi-Square Test:\")\n",
    "    print(selected_features_chi2)\n",
    "\n",
    "    # Mutual Information for all features\n",
    "    mi_selector = SelectKBest(mutual_info_classif, k=15)\n",
    "    mi_selector.fit_transform(X, y)\n",
    "    selected_features_mi = X.columns[mi_selector.get_support()]\n",
    "    print(\"Selected features based on Mutual Information:\")\n",
    "    print(selected_features_mi)\n",
    "\n",
    "    # Variance Threshold for continuous features\n",
    "    threshold = 0.05\n",
    "    var_threshold = VarianceThreshold(threshold=threshold)\n",
    "    var_threshold.fit_transform(X[continuous_features])\n",
    "    selected_features_var = X[continuous_features].columns[var_threshold.get_support()]\n",
    "    print(\"Selected continuous features based on Variance Threshold:\")\n",
    "    print(selected_features_var)\n",
    "    print(len(selected_features_var))\n",
    "\n",
    "\n",
    "    # Combine all selected features\n",
    "    selected_features_all = set(selected_features_corr) | set(selected_features_chi2) | set(selected_features_mi) | set(selected_features_var)\n",
    "\n",
    "    # Original features\n",
    "    original_features = set(X.columns)\n",
    "\n",
    "    # Find features not selected by any method\n",
    "    least_selected_features = original_features - selected_features_all\n",
    "\n",
    "    boosting_selected_features = set()\n",
    "\n",
    "    least_selected_features_list = list(least_selected_features | boosting_selected_features)\n",
    "\n",
    "    print(least_selected_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a12ca368",
   "metadata": {
    "papermill": {
     "duration": 0.029184,
     "end_time": "2024-09-01T17:00:35.587079",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.557895",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TEST PREDICTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "7e212243",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-01T17:00:35.777827Z",
     "iopub.status.busy": "2024-09-01T17:00:35.777568Z",
     "iopub.status.idle": "2024-09-01T17:00:35.921945Z",
     "shell.execute_reply": "2024-09-01T17:00:35.921257Z"
    },
    "papermill": {
     "duration": 0.176513,
     "end_time": "2024-09-01T17:00:35.923887",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.747374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ISIC_0015657</th>\n",
       "      <td>0.274951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015729</th>\n",
       "      <td>0.218071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015740</th>\n",
       "      <td>0.249403</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target\n",
       "isic_id               \n",
       "ISIC_0015657  0.274951\n",
       "ISIC_0015729  0.218071\n",
       "ISIC_0015740  0.249403"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 再学習したものを使い場合\n",
    "\n",
    "df_subm['target'] = final_estimator.predict_proba(df_test[feature_cols])[:, 1]\n",
    "\n",
    "df_subm.to_csv('submission.csv')\n",
    "df_subm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dcd7fa9",
   "metadata": {
    "papermill": {
     "duration": 0.029385,
     "end_time": "2024-09-01T17:00:35.984065",
     "exception": false,
     "start_time": "2024-09-01T17:00:35.954680",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 9094797,
     "sourceId": 63056,
     "sourceType": "competition"
    },
    {
     "datasetId": 5406640,
     "sourceId": 8982084,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5415918,
     "sourceId": 8991790,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5547006,
     "sourceId": 9178024,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5554340,
     "sourceId": 9188295,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5557308,
     "sourceId": 9192711,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5557325,
     "sourceId": 9192730,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5570381,
     "sourceId": 9212230,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5574872,
     "sourceId": 9218742,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5584877,
     "sourceId": 9233358,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5585278,
     "sourceId": 9233973,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5588816,
     "sourceId": 9239367,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5589059,
     "sourceId": 9239750,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5594036,
     "sourceId": 9247162,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5598115,
     "sourceId": 9253005,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5608770,
     "sourceId": 9268358,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5619116,
     "sourceId": 9283137,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5619591,
     "sourceId": 9283765,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5620073,
     "sourceId": 9284569,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5627973,
     "sourceId": 9295669,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 186147615,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 186149019,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 187730674,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188543089,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188543756,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188602899,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188603204,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188603902,
     "sourceType": "kernelVersion"
    }
   ],
   "dockerImageVersionId": 30747,
   "isGpuEnabled": true,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 417.422882,
   "end_time": "2024-09-01T17:00:39.131887",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-09-01T16:53:41.709005",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}