notebooks/model_scratch.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python385jvsc74a57bd0f15a5c76ea51587da210030115844771c17b4abfe429728161a28bde3ed3abfd",
   "display_name": "Python 3.8.5 64-bit ('venv')"
  },
  "metadata": {
   "interpreter": {
    "hash": "f15a5c76ea51587da210030115844771c17b4abfe429728161a28bde3ed3abfd"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from imblearn.over_sampling import SMOTE\n",
    "import xgboost as xgb\n",
    "import joblib"
   ]
  },
  {
   "source": [
    "## Load Dataset"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(303, 14)"
      ]
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "dtypes = {\n",
    "    'age': 'int', \n",
    "    'sex': 'str', \n",
    "    'cp': 'str', \n",
    "    'trestbps': 'int', \n",
    "    'chol': 'int', \n",
    "    'fbs': 'str', \n",
    "    'restecg': 'str', \n",
    "    'thalach': 'int', \n",
    "    'exang': 'str', \n",
    "    'oldpeak': 'float', \n",
    "    'slope': 'str', \n",
    "    'ca': 'str', \n",
    "    'thal': 'str', \n",
    "    'target': 'int',\n",
    "}\n",
    "\n",
    "df_data = pd.read_csv(\"../data/heart.xls\", dtype=dtypes)\n",
    "df_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   age sex cp  trestbps  chol fbs restecg  thalach exang  oldpeak slope ca  \\\n",
       "0   63   1  3       145   233   1       0      150     0      2.3     0  0   \n",
       "1   37   1  2       130   250   0       1      187     0      3.5     0  0   \n",
       "2   41   0  1       130   204   0       0      172     0      1.4     2  0   \n",
       "3   56   1  1       120   236   0       1      178     0      0.8     2  0   \n",
       "4   57   0  0       120   354   0       1      163     1      0.6     2  0   \n",
       "\n",
       "  thal  target  \n",
       "0    1       1  \n",
       "1    2       1  \n",
       "2    2       1  \n",
       "3    2       1  \n",
       "4    2       1  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>age</th>\n      <th>sex</th>\n      <th>cp</th>\n      <th>trestbps</th>\n      <th>chol</th>\n      <th>fbs</th>\n      <th>restecg</th>\n      <th>thalach</th>\n      <th>exang</th>\n      <th>oldpeak</th>\n      <th>slope</th>\n      <th>ca</th>\n      <th>thal</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>63</td>\n      <td>1</td>\n      <td>3</td>\n      <td>145</td>\n      <td>233</td>\n      <td>1</td>\n      <td>0</td>\n      <td>150</td>\n      <td>0</td>\n      <td>2.3</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>37</td>\n      <td>1</td>\n      <td>2</td>\n      <td>130</td>\n      <td>250</td>\n      <td>0</td>\n      <td>1</td>\n      <td>187</td>\n      <td>0</td>\n      <td>3.5</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>41</td>\n      <td>0</td>\n      <td>1</td>\n      <td>130</td>\n      <td>204</td>\n      <td>0</td>\n      <td>0</td>\n      <td>172</td>\n      <td>0</td>\n      <td>1.4</td>\n      <td>2</td>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>56</td>\n      <td>1</td>\n      <td>1</td>\n      <td>120</td>\n      <td>236</td>\n      <td>0</td>\n      <td>1</td>\n      <td>178</td>\n      <td>0</td>\n      <td>0.8</td>\n      <td>2</td>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>57</td>\n      <td>0</td>\n      <td>0</td>\n      <td>120</td>\n      <td>354</td>\n      <td>0</td>\n      <td>1</td>\n      <td>163</td>\n      <td>1</td>\n      <td>0.6</td>\n      <td>2</td>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "df_data.head()"
   ]
  },
  {
   "source": [
    "Attribute Information:\n",
    "\n",
    "Only 14 attributes used:\n",
    "\n",
    "1. age: age in years (**numeric**)\n",
    "2. sex: (1 = male; 0 = female) (**categorical**)\n",
    "3. cp: chest pain type (**categorical**)\n",
    "    - Value 1: typical angina\n",
    "    - Value 2: atypical angina\n",
    "    - Value 3: non-anginal pain\n",
    "    - Value 4: asymptomatic \n",
    "4. trestbps: resting blood pressure (in mm Hg on admission to the hospital) (**numeric**)\n",
    "5. chol: serum cholestoral in mg/dl (**numeric**)\n",
    "6. fbs: (fasting blood sugar > 120 mg/dl) (1 = true; 0 = false) (**categorical**)\n",
    "7. restecg: resting electrocardiographic results (**categorical**)\n",
    "    - Value 0: normal\n",
    "    - Value 1: having ST-T wave abnormality (T wave inversions and/or ST elevation or depression of > 0.05 mV)\n",
    "    - Value 2: showing probable or definite left ventricular hypertrophy by Estes' criteria \n",
    "8. thalach: maximum heart rate achieved (**numeric**)\n",
    "9. exang: exercise induced angina (1 = yes; 0 = no) (**categorical**)\n",
    "10. oldpeak = ST depression induced by exercise relative to rest (**numeric**)\n",
    "11. slope: the slope of the peak exercise ST segment (**categorical**)\n",
    "    - Value 1: upsloping\n",
    "    - Value 2: flat\n",
    "    - Value 3: downsloping \n",
    "12. ca: number of major vessels (0-3) colored by flourosopy (**categorical**)\n",
    "13. thal: 3 = normal; 6 = fixed defect; 7 = reversable defect (**categorical**)\n",
    "14. num: diagnosis of heart disease (angiographic disease status) (**categorical**) (the predicted attribute) \n",
    "    - Value 0: < 50% diameter narrowing\n",
    "    - Value 1: > 50% diameter narrowing\n",
    "    - (in any major vessel: attributes 59 through 68 are vessels)\n",
    "    \n",
    "    In short, \n",
    "    - 0 means absence of heart disease;\n",
    "    - 1 means presence of heart disease;\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 303 entries, 0 to 302\nData columns (total 14 columns):\n #   Column    Non-Null Count  Dtype  \n---  ------    --------------  -----  \n 0   age       303 non-null    int64  \n 1   sex       303 non-null    object \n 2   cp        303 non-null    object \n 3   trestbps  303 non-null    int64  \n 4   chol      303 non-null    int64  \n 5   fbs       303 non-null    object \n 6   restecg   303 non-null    object \n 7   thalach   303 non-null    int64  \n 8   exang     303 non-null    object \n 9   oldpeak   303 non-null    float64\n 10  slope     303 non-null    object \n 11  ca        303 non-null    object \n 12  thal      303 non-null    object \n 13  target    303 non-null    int64  \ndtypes: float64(1), int64(5), object(8)\nmemory usage: 33.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "1    165\n",
       "0    138\n",
       "Name: target, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 131
    }
   ],
   "source": [
    "df_data[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(303, 13)\n(303, 1)\n"
     ]
    }
   ],
   "source": [
    "X, y = df_data.drop(columns=[\"target\"]), df_data[[\"target\"]]\n",
    "print(X.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "source": [
    "## Split dataset"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(212, 13)\n(91, 13)\n(212, 1)\n(91, 1)\n"
     ]
    }
   ],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=2, test_size=0.3, random_state=123)\n",
    "\n",
    "for train_index, test_index in sss.split(X, y):\n",
    "    X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "\n",
    "# reset the suffled indexes\n",
    "X_train = X_train.reset_index(drop=True)\n",
    "X_test = X_test.reset_index(drop=True)\n",
    "y_train = y_train.reset_index(drop=True)\n",
    "y_test = y_test.reset_index(drop=True)\n",
    "\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "target\n",
      "1         115\n",
      "0          97\n",
      "dtype: int64\n",
      "target\n",
      "1         50\n",
      "0         41\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y_train.value_counts())\n",
    "print(y_test.value_counts())"
   ]
  },
  {
   "source": [
    "## Balance the Training Dataset"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sm = SMOTE(random_state=123)\n",
    "#X_train_res, y_train_res = sm.fit_resample(X_train, y_train)\n",
    "#print(X_train_res.shape)\n",
    "#print(y_train_res.shape)"
   ]
  },
  {
   "source": [],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['sex' 'cp' 'fbs' 'restecg' 'exang' 'slope' 'ca' 'thal']\n"
     ]
    }
   ],
   "source": [
    "cat_col = X_train.select_dtypes(include=['object']).columns.values\n",
    "print(cat_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'sex': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'cp': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'fbs': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'restecg': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'exang': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'slope': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'ca': OneHotEncoder(handle_unknown='ignore', sparse=False),\n",
       " 'thal': OneHotEncoder(handle_unknown='ignore', sparse=False)}"
      ]
     },
     "metadata": {},
     "execution_count": 137
    }
   ],
   "source": [
    "ohe = {\n",
    "    c: OneHotEncoder(sparse=False, handle_unknown='ignore')\n",
    "    for c in cat_col\n",
    "}\n",
    "ohe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "sex\t->\t[array(['0', '1'], dtype=object)]\ncp\t->\t[array(['0', '1', '2', '3'], dtype=object)]\nfbs\t->\t[array(['0', '1'], dtype=object)]\nrestecg\t->\t[array(['0', '1', '2'], dtype=object)]\nexang\t->\t[array(['0', '1'], dtype=object)]\nslope\t->\t[array(['0', '1', '2'], dtype=object)]\nca\t->\t[array(['0', '1', '2', '3', '4'], dtype=object)]\nthal\t->\t[array(['0', '1', '2', '3'], dtype=object)]\n"
     ]
    }
   ],
   "source": [
    "for c in cat_col:\n",
    "    ohe[c].fit(X_train[c].values.reshape(-1, 1))\n",
    "    print(f\"{c}\\t->\\t{ohe[c].categories_}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(212, 25)\n"
     ]
    }
   ],
   "source": [
    "X_transformed = None\n",
    "for c in cat_col:\n",
    "    feat_arr = ohe[c].transform(X_train[c].values.reshape(-1, 1))\n",
    "\n",
    "    if X_transformed is None:\n",
    "        X_transformed = pd.DataFrame(feat_arr)\n",
    "        X_transformed.columns = [\n",
    "            \"{}_{}\".format(c, v.replace(\" \", \"_\"))\n",
    "            for v in ohe[c].categories_[0]\n",
    "        ]\n",
    "    \n",
    "    else:\n",
    "        _X_buff = pd.DataFrame(feat_arr)\n",
    "        _X_buff.columns = [\n",
    "            f\"{c}_{v.replace(' ', '_')}\"\n",
    "            for v in ohe[c].categories_[0]\n",
    "        ]\n",
    "        X_transformed = pd.concat([X_transformed, _X_buff], axis=1)\n",
    "\n",
    "print(X_transformed.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   sex_0  sex_1  cp_0  cp_1  cp_2  cp_3  fbs_0  fbs_1  restecg_0  restecg_1  \\\n",
       "0    1.0    0.0   0.0   0.0   1.0   0.0    0.0    1.0        0.0        1.0   \n",
       "1    1.0    0.0   0.0   1.0   0.0   0.0    1.0    0.0        0.0        1.0   \n",
       "2    0.0    1.0   1.0   0.0   0.0   0.0    1.0    0.0        1.0        0.0   \n",
       "3    1.0    0.0   1.0   0.0   0.0   0.0    1.0    0.0        0.0        1.0   \n",
       "4    0.0    1.0   0.0   0.0   1.0   0.0    1.0    0.0        0.0        1.0   \n",
       "\n",
       "   ...  slope_2  ca_0  ca_1  ca_2  ca_3  ca_4  thal_0  thal_1  thal_2  thal_3  \n",
       "0  ...      1.0   1.0   0.0   0.0   0.0   0.0     0.0     0.0     1.0     0.0  \n",
       "1  ...      1.0   0.0   0.0   1.0   0.0   0.0     0.0     0.0     1.0     0.0  \n",
       "2  ...      0.0   0.0   0.0   1.0   0.0   0.0     0.0     0.0     0.0     1.0  \n",
       "3  ...      1.0   1.0   0.0   0.0   0.0   0.0     0.0     0.0     1.0     0.0  \n",
       "4  ...      1.0   0.0   0.0   0.0   0.0   1.0     0.0     0.0     1.0     0.0  \n",
       "\n",
       "[5 rows x 25 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>sex_0</th>\n      <th>sex_1</th>\n      <th>cp_0</th>\n      <th>cp_1</th>\n      <th>cp_2</th>\n      <th>cp_3</th>\n      <th>fbs_0</th>\n      <th>fbs_1</th>\n      <th>restecg_0</th>\n      <th>restecg_1</th>\n      <th>...</th>\n      <th>slope_2</th>\n      <th>ca_0</th>\n      <th>ca_1</th>\n      <th>ca_2</th>\n      <th>ca_3</th>\n      <th>ca_4</th>\n      <th>thal_0</th>\n      <th>thal_1</th>\n      <th>thal_2</th>\n      <th>thal_3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 25 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 140
    }
   ],
   "source": [
    "X_transformed.head()"
   ]
  },
  {
   "source": [
    "## Concat numeric with one hot variables"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "After concat:\t(212, 30)\n"
     ]
    }
   ],
   "source": [
    "# drop categorical columns to concat one hot\n",
    "X_train = X_train.drop(columns=cat_col)\n",
    "\n",
    "X_train = pd.concat([\n",
    "    X_train, \n",
    "    X_transformed\n",
    "], axis=1)\n",
    "\n",
    "print(f\"After concat:\\t{X_train.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   age  trestbps  chol  thalach  oldpeak  sex_0  sex_1  cp_0  cp_1  cp_2  ...  \\\n",
       "0   60       120   178       96      0.0    1.0    0.0   0.0   0.0   1.0  ...   \n",
       "1   63       140   195      179      0.0    1.0    0.0   0.0   1.0   0.0  ...   \n",
       "2   60       130   206      132      2.4    0.0    1.0   1.0   0.0   0.0  ...   \n",
       "3   64       180   325      154      0.0    1.0    0.0   1.0   0.0   0.0  ...   \n",
       "4   38       138   175      173      0.0    0.0    1.0   0.0   0.0   1.0  ...   \n",
       "\n",
       "   slope_2  ca_0  ca_1  ca_2  ca_3  ca_4  thal_0  thal_1  thal_2  thal_3  \n",
       "0      1.0   1.0   0.0   0.0   0.0   0.0     0.0     0.0     1.0     0.0  \n",
       "1      1.0   0.0   0.0   1.0   0.0   0.0     0.0     0.0     1.0     0.0  \n",
       "2      0.0   0.0   0.0   1.0   0.0   0.0     0.0     0.0     0.0     1.0  \n",
       "3      1.0   1.0   0.0   0.0   0.0   0.0     0.0     0.0     1.0     0.0  \n",
       "4      1.0   0.0   0.0   0.0   0.0   1.0     0.0     0.0     1.0     0.0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>age</th>\n      <th>trestbps</th>\n      <th>chol</th>\n      <th>thalach</th>\n      <th>oldpeak</th>\n      <th>sex_0</th>\n      <th>sex_1</th>\n      <th>cp_0</th>\n      <th>cp_1</th>\n      <th>cp_2</th>\n      <th>...</th>\n      <th>slope_2</th>\n      <th>ca_0</th>\n      <th>ca_1</th>\n      <th>ca_2</th>\n      <th>ca_3</th>\n      <th>ca_4</th>\n      <th>thal_0</th>\n      <th>thal_1</th>\n      <th>thal_2</th>\n      <th>thal_3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>60</td>\n      <td>120</td>\n      <td>178</td>\n      <td>96</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>63</td>\n      <td>140</td>\n      <td>195</td>\n      <td>179</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>60</td>\n      <td>130</td>\n      <td>206</td>\n      <td>132</td>\n      <td>2.4</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>64</td>\n      <td>180</td>\n      <td>325</td>\n      <td>154</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>38</td>\n      <td>138</td>\n      <td>175</td>\n      <td>173</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>...</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 30 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 144
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "source": [
    "## Apply one hot to test set"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_transformed = None\n",
    "for c in cat_col:\n",
    "    feat_arr = ohe[c].transform(X_test[c].values.reshape(-1, 1))\n",
    "\n",
    "    if X_transformed is None:\n",
    "        X_transformed = pd.DataFrame(feat_arr)\n",
    "        X_transformed.columns = [\n",
    "            \"{}_{}\".format(c, v.replace(\" \", \"_\"))\n",
    "            for v in ohe[c].categories_[0]\n",
    "        ]\n",
    "    \n",
    "    else:\n",
    "        _X_buff = pd.DataFrame(feat_arr)\n",
    "        _X_buff.columns = [\n",
    "            f\"{c}_{v.replace(' ', '_')}\"\n",
    "            for v in ohe[c].categories_[0]\n",
    "        ]\n",
    "        X_transformed = pd.concat([X_transformed, _X_buff], axis=1)"
   ]
  },
  {
   "source": [
    "## Concat One Hot with Numeric variables on test set"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "After concat:\t(91, 30)\n"
     ]
    }
   ],
   "source": [
    "# drop categorical columns to concat one hot\n",
    "X_test = X_test.drop(columns=cat_col)\n",
    "\n",
    "X_test = pd.concat([\n",
    "    X_test, \n",
    "    X_transformed\n",
    "], axis=1)\n",
    "\n",
    "print(f\"After concat:\\t{X_test.shape}\")"
   ]
  },
  {
   "source": [
    "## Build Model"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelXGBM = xgb.XGBClassifier(colsample_bytree= 1, learning_rate=0.1, max_depth=4, \n",
    "                              min_child_weight=1e-05, n_estimators=200, \n",
    "                              objective='binary:logistic', subsample=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(212, 30)\n(212, 1)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[21:08:52] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "/home/paulo/Documents/workspace/heart-disease-classifier/venv/lib/python3.8/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/home/paulo/Documents/workspace/heart-disease-classifier/venv/lib/python3.8/site-packages/sklearn/utils/validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "modelXGBM.fit(X_train, y_train)\n",
    "model_prediction = modelXGBM.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.7582417582417582\n0.7536585365853659\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(accuracy_score(y_test, model_prediction))\n",
    "print(roc_auc_score(y_test, model_prediction))"
   ]
  },
  {
   "source": [
    "## Persist Model and data used"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['../data/modelXGBM.pkl']"
      ]
     },
     "metadata": {},
     "execution_count": 154
    }
   ],
   "source": [
    "joblib.dump(modelXGBM, \"../data/modelXGBM.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_res.to_csv(\"../data/X_train.csv\")\n",
    "X_test.to_csv(\"../data/X_test.csv\")\n",
    "y_train_res.to_csv(\"../data/y_train.csv\")\n",
    "y_test.to_csv(\"../data/y_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data to test inference\n",
    "df_data.drop(columns=[\"target\"]).to_parquet(\"../data/dataset_to_infer.parquet\")\n",
    "df_data.drop(columns=[\"target\"]).to_csv(\"../data/dataset_to_infer.csv\", sep=\",\", header=True, index=False)"
   ]
  }
 ]
}