notebooks/compute_optimal_thres.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6f5a3f5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import json\n",
    "import copy\n",
    "import socket\n",
    "import getpass\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "sys.path.append('../')\n",
    "import pickle\n",
    "import warnings\n",
    "from sklearn.metrics import recall_score, matthews_corrcoef, roc_auc_score, f1_score\n",
    "from collections import defaultdict\n",
    "import json\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)\n",
    "\n",
    "def flatten(l):\n",
    "    return [item for sublist in l for item in sublist]\n",
    "\n",
    "def compute_opt_thres(target, pred):\n",
    "    opt_thres = 0\n",
    "    opt_f1 = 0\n",
    "    for i in np.arange(0.001, 0.999, 0.001):\n",
    "        f1 = f1_score(target, pred >= i)\n",
    "        if f1 >= opt_f1:\n",
    "            opt_thres = i\n",
    "            opt_f1 = f1\n",
    "    return opt_thres\n",
    "\n",
    "plt.rcParams.update({'font.size': 20})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0e5a759f",
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = Path(f'/path/to/your/root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2c78959d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if Path('opt_thres.pkl').is_file():\n",
    "    already_computed = set(pickle.load(Path('opt_thres.pkl').open('rb')).keys())\n",
    "else:\n",
    "    already_computed = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cef4d188",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6155it [00:14, 430.52it/s]\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for i in tqdm(root_dir.glob('**/done')):\n",
    "    args = json.load((i.parent/'args.json').open('r'))\n",
    "    if (args['dataset'][0], args['task'], args['attr'], args['algorithm']) in already_computed:\n",
    "        continue\n",
    "    \n",
    "    final_res = pickle.load((i.parent/'final_results.pkl').open('rb'))\n",
    "    \n",
    "    ssets = ['va', 'te', 'MIMIC-sex-te', 'CheXpert-sex-te', 'NIH-sex-te', \n",
    "             'PadChest-sex-te', 'VinDr-sex-te']\n",
    "    if args['task'] == 'Pneumothorax':\n",
    "        ssets.append('SIIM-sex-te')\n",
    "    \n",
    "    for sset in ssets:\n",
    "        if sset in final_res:\n",
    "            args[f'{sset}_auroc'] = final_res[sset]['overall']['AUROC']\n",
    "            if sset == 'va':\n",
    "                args[f'{sset}_min_attr_auroc'] = final_res[sset]['min_attr']['AUROC']\n",
    "    args['va_y'] = final_res['va']['y']\n",
    "    args['va_preds'] = final_res['va']['preds']\n",
    "    \n",
    "    results.append(args)\n",
    "df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6c7af53",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['dataset'] = df['dataset'].apply(lambda x: x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e3c4d1ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(663, 32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2a361c5",
   "metadata": {},
   "source": [
    "## Optimal Threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ee6e1166",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_models = df.groupby(['dataset', 'task', 'attr', 'algorithm']).apply(lambda x: x.loc[x['va_min_attr_auroc'].idxmax()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b40cb05d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [00:29<00:00,  1.23s/it]\n"
     ]
    }
   ],
   "source": [
    "opt_thres = {}\n",
    "for idx, row in tqdm(best_models.iterrows(), total = len(best_models)):\n",
    "    dataset, task, attr, algorithm = idx\n",
    "#     if dataset not in opt_thres:\n",
    "#         opt_thres[dataset] = {}\n",
    "    opt_thres[(dataset, task, attr, algorithm)] = np.round(compute_opt_thres(row['va_y'], row['va_preds']), 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c0da7061",
   "metadata": {},
   "outputs": [],
   "source": [
    "if Path('opt_thres.pkl').is_file():\n",
    "    old_file = pickle.load(Path('opt_thres.pkl').open('rb'))\n",
    "else:\n",
    "    old_file = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4c02c62b",
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_thres = {**old_file, **opt_thres}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3854db77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('CheXpert', 'Cardiomegaly', 'age', 'CDANN'): 0.305,\n",
       " ('CheXpert', 'Cardiomegaly', 'age', 'DANN'): 0.217,\n",
       " ('CheXpert', 'Cardiomegaly', 'age', 'ERM'): 0.342,\n",
       " ('CheXpert', 'Cardiomegaly', 'age', 'GroupDRO'): 0.759,\n",
       " ('CheXpert', 'Cardiomegaly', 'age', 'MA'): 0.342,\n",
       " ('CheXpert', 'Cardiomegaly', 'age', 'ReSample'): 0.811,\n",
       " ('CheXpert', 'Cardiomegaly', 'ethnicity', 'CDANN'): 0.255,\n",
       " ('CheXpert', 'Cardiomegaly', 'ethnicity', 'DANN'): 0.332,\n",
       " ('CheXpert', 'Cardiomegaly', 'ethnicity', 'ERM'): 0.333,\n",
       " ('CheXpert', 'Cardiomegaly', 'ethnicity', 'GroupDRO'): 0.83,\n",
       " ('CheXpert', 'Cardiomegaly', 'ethnicity', 'MA'): 0.342,\n",
       " ('CheXpert', 'Cardiomegaly', 'ethnicity', 'ReSample'): 0.753,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex', 'CDANN'): 0.207,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex', 'DANN'): 0.197,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex', 'ERM'): 0.255,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex', 'GroupDRO'): 0.822,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex', 'MA'): 0.345,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex', 'ReSample'): 0.726,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex_ethnicity', 'CDANN'): 0.334,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex_ethnicity', 'DANN'): 0.245,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex_ethnicity', 'ERM'): 0.278,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex_ethnicity', 'GroupDRO'): 0.717,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex_ethnicity', 'MA'): 0.349,\n",
       " ('CheXpert', 'Cardiomegaly', 'sex_ethnicity', 'ReSample'): 0.751,\n",
       " ('CheXpert', 'Effusion', 'age', 'CDANN'): 0.352,\n",
       " ('CheXpert', 'Effusion', 'age', 'DANN'): 0.471,\n",
       " ('CheXpert', 'Effusion', 'age', 'ERM'): 0.36,\n",
       " ('CheXpert', 'Effusion', 'age', 'GroupDRO'): 0.509,\n",
       " ('CheXpert', 'Effusion', 'age', 'MA'): 0.412,\n",
       " ('CheXpert', 'Effusion', 'age', 'ReSample'): 0.475,\n",
       " ('CheXpert', 'Effusion', 'ethnicity', 'CDANN'): 0.451,\n",
       " ('CheXpert', 'Effusion', 'ethnicity', 'DANN'): 0.436,\n",
       " ('CheXpert', 'Effusion', 'ethnicity', 'ERM'): 0.36,\n",
       " ('CheXpert', 'Effusion', 'ethnicity', 'GroupDRO'): 0.539,\n",
       " ('CheXpert', 'Effusion', 'ethnicity', 'MA'): 0.412,\n",
       " ('CheXpert', 'Effusion', 'ethnicity', 'ReSample'): 0.438,\n",
       " ('CheXpert', 'Effusion', 'sex', 'CDANN'): 0.309,\n",
       " ('CheXpert', 'Effusion', 'sex', 'DANN'): 0.416,\n",
       " ('CheXpert', 'Effusion', 'sex', 'ERM'): 0.383,\n",
       " ('CheXpert', 'Effusion', 'sex', 'GroupDRO'): 0.499,\n",
       " ('CheXpert', 'Effusion', 'sex', 'MA'): 0.407,\n",
       " ('CheXpert', 'Effusion', 'sex', 'ReSample'): 0.46,\n",
       " ('CheXpert', 'Effusion', 'sex_ethnicity', 'CDANN'): 0.469,\n",
       " ('CheXpert', 'Effusion', 'sex_ethnicity', 'DANN'): 0.382,\n",
       " ('CheXpert', 'Effusion', 'sex_ethnicity', 'ERM'): 0.383,\n",
       " ('CheXpert', 'Effusion', 'sex_ethnicity', 'GroupDRO'): 0.566,\n",
       " ('CheXpert', 'Effusion', 'sex_ethnicity', 'MA'): 0.373,\n",
       " ('CheXpert', 'Effusion', 'sex_ethnicity', 'ReSample'): 0.325,\n",
       " ('CheXpert', 'No Finding', 'age', 'CDANN'): 0.239,\n",
       " ('CheXpert', 'No Finding', 'age', 'DANN'): 0.414,\n",
       " ('CheXpert', 'No Finding', 'age', 'ERM'): 0.268,\n",
       " ('CheXpert', 'No Finding', 'age', 'GroupDRO'): 0.612,\n",
       " ('CheXpert', 'No Finding', 'age', 'MA'): 0.299,\n",
       " ('CheXpert', 'No Finding', 'age', 'ReSample'): 0.677,\n",
       " ('CheXpert', 'No Finding', 'ethnicity', 'CDANN'): 0.256,\n",
       " ('CheXpert', 'No Finding', 'ethnicity', 'DANN'): 0.373,\n",
       " ('CheXpert', 'No Finding', 'ethnicity', 'ERM'): 0.242,\n",
       " ('CheXpert', 'No Finding', 'ethnicity', 'GroupDRO'): 0.814,\n",
       " ('CheXpert', 'No Finding', 'ethnicity', 'MA'): 0.35,\n",
       " ('CheXpert', 'No Finding', 'ethnicity', 'ReSample'): 0.571,\n",
       " ('CheXpert', 'No Finding', 'sex', 'CDANN'): 0.198,\n",
       " ('CheXpert', 'No Finding', 'sex', 'DANN'): 0.251,\n",
       " ('CheXpert', 'No Finding', 'sex', 'ERM'): 0.173,\n",
       " ('CheXpert', 'No Finding', 'sex', 'GroupDRO'): 0.78,\n",
       " ('CheXpert', 'No Finding', 'sex', 'MA'): 0.273,\n",
       " ('CheXpert', 'No Finding', 'sex', 'ReSample'): 0.775,\n",
       " ('CheXpert', 'No Finding', 'sex_ethnicity', 'CDANN'): 0.2,\n",
       " ('CheXpert', 'No Finding', 'sex_ethnicity', 'DANN'): 0.177,\n",
       " ('CheXpert', 'No Finding', 'sex_ethnicity', 'ERM'): 0.162,\n",
       " ('CheXpert', 'No Finding', 'sex_ethnicity', 'GroupDRO'): 0.653,\n",
       " ('CheXpert', 'No Finding', 'sex_ethnicity', 'MA'): 0.291,\n",
       " ('CheXpert', 'No Finding', 'sex_ethnicity', 'ReSample'): 0.793,\n",
       " ('CheXpert', 'Pneumothorax', 'age', 'CDANN'): 0.183,\n",
       " ('CheXpert', 'Pneumothorax', 'age', 'DANN'): 0.319,\n",
       " ('CheXpert', 'Pneumothorax', 'age', 'ERM'): 0.255,\n",
       " ('CheXpert', 'Pneumothorax', 'age', 'GroupDRO'): 0.615,\n",
       " ('CheXpert', 'Pneumothorax', 'age', 'MA'): 0.231,\n",
       " ('CheXpert', 'Pneumothorax', 'age', 'ReSample'): 0.712,\n",
       " ('CheXpert', 'Pneumothorax', 'ethnicity', 'CDANN'): 0.226,\n",
       " ('CheXpert', 'Pneumothorax', 'ethnicity', 'DANN'): 0.207,\n",
       " ('CheXpert', 'Pneumothorax', 'ethnicity', 'ERM'): 0.255,\n",
       " ('CheXpert', 'Pneumothorax', 'ethnicity', 'GroupDRO'): 0.738,\n",
       " ('CheXpert', 'Pneumothorax', 'ethnicity', 'MA'): 0.22,\n",
       " ('CheXpert', 'Pneumothorax', 'ethnicity', 'ReSample'): 0.715,\n",
       " ('CheXpert', 'Pneumothorax', 'sex', 'CDANN'): 0.144,\n",
       " ('CheXpert', 'Pneumothorax', 'sex', 'DANN'): 0.212,\n",
       " ('CheXpert', 'Pneumothorax', 'sex', 'ERM'): 0.226,\n",
       " ('CheXpert', 'Pneumothorax', 'sex', 'GroupDRO'): 0.735,\n",
       " ('CheXpert', 'Pneumothorax', 'sex', 'MA'): 0.227,\n",
       " ('CheXpert', 'Pneumothorax', 'sex', 'ReSample'): 0.701,\n",
       " ('CheXpert', 'Pneumothorax', 'sex_ethnicity', 'CDANN'): 0.336,\n",
       " ('CheXpert', 'Pneumothorax', 'sex_ethnicity', 'DANN'): 0.205,\n",
       " ('CheXpert', 'Pneumothorax', 'sex_ethnicity', 'ERM'): 0.231,\n",
       " ('CheXpert', 'Pneumothorax', 'sex_ethnicity', 'GroupDRO'): 0.793,\n",
       " ('CheXpert', 'Pneumothorax', 'sex_ethnicity', 'MA'): 0.235,\n",
       " ('CheXpert', 'Pneumothorax', 'sex_ethnicity', 'ReSample'): 0.72,\n",
       " ('MIMIC', 'Cardiomegaly', 'age', 'CDANN'): 0.211,\n",
       " ('MIMIC', 'Cardiomegaly', 'age', 'DANN'): 0.196,\n",
       " ('MIMIC', 'Cardiomegaly', 'age', 'ERM'): 0.228,\n",
       " ('MIMIC', 'Cardiomegaly', 'age', 'GroupDRO'): 0.598,\n",
       " ('MIMIC', 'Cardiomegaly', 'age', 'MA'): 0.277,\n",
       " ('MIMIC', 'Cardiomegaly', 'age', 'ReSample'): 0.614,\n",
       " ('MIMIC', 'Cardiomegaly', 'ethnicity', 'CDANN'): 0.224,\n",
       " ('MIMIC', 'Cardiomegaly', 'ethnicity', 'DANN'): 0.225,\n",
       " ('MIMIC', 'Cardiomegaly', 'ethnicity', 'ERM'): 0.223,\n",
       " ('MIMIC', 'Cardiomegaly', 'ethnicity', 'GroupDRO'): 0.591,\n",
       " ('MIMIC', 'Cardiomegaly', 'ethnicity', 'MA'): 0.252,\n",
       " ('MIMIC', 'Cardiomegaly', 'ethnicity', 'ReSample'): 0.693,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex', 'CDANN'): 0.251,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex', 'DANN'): 0.23,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex', 'ERM'): 0.179,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex', 'GroupDRO'): 0.599,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex', 'MA'): 0.253,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex', 'ReSample'): 0.637,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex_ethnicity', 'CDANN'): 0.198,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex_ethnicity', 'DANN'): 0.183,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex_ethnicity', 'ERM'): 0.179,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex_ethnicity', 'GroupDRO'): 0.569,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex_ethnicity', 'MA'): 0.253,\n",
       " ('MIMIC', 'Cardiomegaly', 'sex_ethnicity', 'ReSample'): 0.707,\n",
       " ('MIMIC', 'Effusion', 'age', 'CDANN'): 0.435,\n",
       " ('MIMIC', 'Effusion', 'age', 'DANN'): 0.443,\n",
       " ('MIMIC', 'Effusion', 'age', 'ERM'): 0.346,\n",
       " ('MIMIC', 'Effusion', 'age', 'GroupDRO'): 0.684,\n",
       " ('MIMIC', 'Effusion', 'age', 'MA'): 0.378,\n",
       " ('MIMIC', 'Effusion', 'age', 'ReSample'): 0.587,\n",
       " ('MIMIC', 'Effusion', 'ethnicity', 'CDANN'): 0.384,\n",
       " ('MIMIC', 'Effusion', 'ethnicity', 'DANN'): 0.379,\n",
       " ('MIMIC', 'Effusion', 'ethnicity', 'ERM'): 0.294,\n",
       " ('MIMIC', 'Effusion', 'ethnicity', 'GroupDRO'): 0.533,\n",
       " ('MIMIC', 'Effusion', 'ethnicity', 'MA'): 0.352,\n",
       " ('MIMIC', 'Effusion', 'ethnicity', 'ReSample'): 0.663,\n",
       " ('MIMIC', 'Effusion', 'sex', 'CDANN'): 0.345,\n",
       " ('MIMIC', 'Effusion', 'sex', 'DANN'): 0.298,\n",
       " ('MIMIC', 'Effusion', 'sex', 'ERM'): 0.309,\n",
       " ('MIMIC', 'Effusion', 'sex', 'GroupDRO'): 0.719,\n",
       " ('MIMIC', 'Effusion', 'sex', 'MA'): 0.36,\n",
       " ('MIMIC', 'Effusion', 'sex', 'ReSample'): 0.651,\n",
       " ('MIMIC', 'Effusion', 'sex_ethnicity', 'CDANN'): 0.371,\n",
       " ('MIMIC', 'Effusion', 'sex_ethnicity', 'DANN'): 0.256,\n",
       " ('MIMIC', 'Effusion', 'sex_ethnicity', 'ERM'): 0.397,\n",
       " ('MIMIC', 'Effusion', 'sex_ethnicity', 'GroupDRO'): 0.581,\n",
       " ('MIMIC', 'Effusion', 'sex_ethnicity', 'MA'): 0.36,\n",
       " ('MIMIC', 'Effusion', 'sex_ethnicity', 'ReSample'): 0.685,\n",
       " ('MIMIC', 'No Finding', 'age', 'CDANN'): 0.374,\n",
       " ('MIMIC', 'No Finding', 'age', 'DANN'): 0.354,\n",
       " ('MIMIC', 'No Finding', 'age', 'ERM'): 0.402,\n",
       " ('MIMIC', 'No Finding', 'age', 'GroupDRO'): 0.644,\n",
       " ('MIMIC', 'No Finding', 'age', 'MA'): 0.394,\n",
       " ('MIMIC', 'No Finding', 'age', 'ReSample'): 0.585,\n",
       " ('MIMIC', 'No Finding', 'ethnicity', 'CDANN'): 0.439,\n",
       " ('MIMIC', 'No Finding', 'ethnicity', 'DANN'): 0.426,\n",
       " ('MIMIC', 'No Finding', 'ethnicity', 'ERM'): 0.323,\n",
       " ('MIMIC', 'No Finding', 'ethnicity', 'GroupDRO'): 0.475,\n",
       " ('MIMIC', 'No Finding', 'ethnicity', 'MA'): 0.395,\n",
       " ('MIMIC', 'No Finding', 'ethnicity', 'ReSample'): 0.355,\n",
       " ('MIMIC', 'No Finding', 'sex', 'CDANN'): 0.399,\n",
       " ('MIMIC', 'No Finding', 'sex', 'DANN'): 0.375,\n",
       " ('MIMIC', 'No Finding', 'sex', 'ERM'): 0.332,\n",
       " ('MIMIC', 'No Finding', 'sex', 'GroupDRO'): 0.513,\n",
       " ('MIMIC', 'No Finding', 'sex', 'MA'): 0.384,\n",
       " ('MIMIC', 'No Finding', 'sex', 'ReSample'): 0.512,\n",
       " ('MIMIC', 'No Finding', 'sex_ethnicity', 'CDANN'): 0.366,\n",
       " ('MIMIC', 'No Finding', 'sex_ethnicity', 'DANN'): 0.351,\n",
       " ('MIMIC', 'No Finding', 'sex_ethnicity', 'ERM'): 0.332,\n",
       " ('MIMIC', 'No Finding', 'sex_ethnicity', 'GroupDRO'): 0.478,\n",
       " ('MIMIC', 'No Finding', 'sex_ethnicity', 'MA'): 0.384,\n",
       " ('MIMIC', 'No Finding', 'sex_ethnicity', 'ReSample'): 0.427,\n",
       " ('MIMIC', 'Pneumothorax', 'age', 'CDANN'): 0.109,\n",
       " ('MIMIC', 'Pneumothorax', 'age', 'DANN'): 0.182,\n",
       " ('MIMIC', 'Pneumothorax', 'age', 'ERM'): 0.228,\n",
       " ('MIMIC', 'Pneumothorax', 'age', 'GroupDRO'): 0.79,\n",
       " ('MIMIC', 'Pneumothorax', 'age', 'MA'): 0.157,\n",
       " ('MIMIC', 'Pneumothorax', 'age', 'ReSample'): 0.839,\n",
       " ('MIMIC', 'Pneumothorax', 'ethnicity', 'CDANN'): 0.074,\n",
       " ('MIMIC', 'Pneumothorax', 'ethnicity', 'DANN'): 0.164,\n",
       " ('MIMIC', 'Pneumothorax', 'ethnicity', 'ERM'): 0.134,\n",
       " ('MIMIC', 'Pneumothorax', 'ethnicity', 'GroupDRO'): 0.761,\n",
       " ('MIMIC', 'Pneumothorax', 'ethnicity', 'MA'): 0.146,\n",
       " ('MIMIC', 'Pneumothorax', 'ethnicity', 'ReSample'): 0.923,\n",
       " ('MIMIC', 'Pneumothorax', 'sex', 'CDANN'): 0.138,\n",
       " ('MIMIC', 'Pneumothorax', 'sex', 'DANN'): 0.159,\n",
       " ('MIMIC', 'Pneumothorax', 'sex', 'ERM'): 0.129,\n",
       " ('MIMIC', 'Pneumothorax', 'sex', 'GroupDRO'): 0.766,\n",
       " ('MIMIC', 'Pneumothorax', 'sex', 'MA'): 0.137,\n",
       " ('MIMIC', 'Pneumothorax', 'sex', 'ReSample'): 0.89,\n",
       " ('MIMIC', 'Pneumothorax', 'sex_ethnicity', 'CDANN'): 0.118,\n",
       " ('MIMIC', 'Pneumothorax', 'sex_ethnicity', 'DANN'): 0.134,\n",
       " ('MIMIC', 'Pneumothorax', 'sex_ethnicity', 'ERM'): 0.12,\n",
       " ('MIMIC', 'Pneumothorax', 'sex_ethnicity', 'GroupDRO'): 0.757,\n",
       " ('MIMIC', 'Pneumothorax', 'sex_ethnicity', 'MA'): 0.152,\n",
       " ('MIMIC', 'Pneumothorax', 'sex_ethnicity', 'ReSample'): 0.777,\n",
       " ('ISIC', 'No Finding', 'age', 'CDANN'): 0.074,\n",
       " ('ISIC', 'No Finding', 'age', 'DANN'): 0.63,\n",
       " ('ISIC', 'No Finding', 'age', 'ERM'): 0.127,\n",
       " ('ISIC', 'No Finding', 'age', 'GroupDRO'): 0.008,\n",
       " ('ISIC', 'No Finding', 'age', 'MA'): 0.55,\n",
       " ('ISIC', 'No Finding', 'age', 'ReSample'): 0.001,\n",
       " ('ISIC', 'No Finding', 'sex', 'CDANN'): 0.667,\n",
       " ('ISIC', 'No Finding', 'sex', 'DANN'): 0.387,\n",
       " ('ISIC', 'No Finding', 'sex', 'ERM'): 0.127,\n",
       " ('ISIC', 'No Finding', 'sex', 'GroupDRO'): 0.008,\n",
       " ('ISIC', 'No Finding', 'sex', 'MA'): 0.345,\n",
       " ('ISIC', 'No Finding', 'sex', 'ReSample'): 0.007,\n",
       " ('ODIR', 'Retinopathy', 'age', 'CDANN'): 0.497,\n",
       " ('ODIR', 'Retinopathy', 'age', 'DANN'): 0.576,\n",
       " ('ODIR', 'Retinopathy', 'age', 'ERM'): 0.365,\n",
       " ('ODIR', 'Retinopathy', 'age', 'GroupDRO'): 0.664,\n",
       " ('ODIR', 'Retinopathy', 'age', 'MA'): 0.365,\n",
       " ('ODIR', 'Retinopathy', 'age', 'ReSample'): 0.548,\n",
       " ('ODIR', 'Retinopathy', 'sex', 'CDANN'): 0.398,\n",
       " ('ODIR', 'Retinopathy', 'sex', 'DANN'): 0.42,\n",
       " ('ODIR', 'Retinopathy', 'sex', 'ERM'): 0.433,\n",
       " ('ODIR', 'Retinopathy', 'sex', 'GroupDRO'): 0.454,\n",
       " ('ODIR', 'Retinopathy', 'sex', 'MA'): 0.433,\n",
       " ('ODIR', 'Retinopathy', 'sex', 'ReSample'): 0.477}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt_thres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c7acdaf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(opt_thres, open('opt_thres.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ec1aeed",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}