4 Gradient Boosted Regression Tree_05.2021.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "id": "r_nFH9Fd6y_G",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Metamodeling - GBRT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M-U3uPvMwoiL"
   },
   "source": [
    "Objective: Develop and Optimize GBRT to predict oil spill\n",
    "\n",
    "Outline of this script:\n",
    "\n",
    "Data Preparation\n",
    "Develop SVR model\n",
    "Fit and measure performance\n",
    "Apply Grid Search, Random Search, HyperOpt and TPOT\n",
    "Compare their performance\n",
    "Input for this script: A dataset which include\n",
    "\n",
    "Output of this script: A few dataframe that contains performance measures (e.g. R^2 and RMSE) and design of the algorithm\n",
    "\n",
    "Date: Feb 20, 2021\n",
    "\n",
    "Written by: Tanmoy Das"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8QZDZk92woiL"
   },
   "source": [
    "## Import libraries & dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "x3Fgi6d7woiM"
   },
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ML modeling\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "p7OtG8uG0Ta6"
   },
   "outputs": [],
   "source": [
    "# Import Training dataset  \n",
    "X_train_dev = pd.read_csv('Dataset for metamodeling/X_train_dev.csv', header=0).copy()\n",
    "X_val = pd.read_csv('Dataset for metamodeling/X_val.csv', header=0).copy()\n",
    "y_train_dev = pd.read_csv('Dataset for metamodeling/y_train_dev.csv', header=0).copy()\n",
    "y_val = pd.read_csv('Dataset for metamodeling/y_val.csv', header=0).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 513,
     "status": "ok",
     "timestamp": 1606631065785,
     "user": {
      "displayName": "Tanmoy Das",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgMmxHcU_jnpEQnP9s8nrNXQJMgMWwxHelgPXC5=s64",
      "userId": "02309773046279713936"
     },
     "user_tz": 240
    },
    "id": "RaSpIgawrG2x",
    "outputId": "499a54fb-2edf-42bb-925c-ee098477a7a2"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>$type_A$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.490008</td>\n",
       "      <td>0.334118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$type_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.491090</td>\n",
       "      <td>0.331719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$type_{oil}$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.492446</td>\n",
       "      <td>0.376486</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$velocity_A$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.498944</td>\n",
       "      <td>0.288640</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.247730</td>\n",
       "      <td>0.498735</td>\n",
       "      <td>0.749022</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$velocity_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.499480</td>\n",
       "      <td>0.288556</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.249848</td>\n",
       "      <td>0.499230</td>\n",
       "      <td>0.750018</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$angle_{collision}$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.500327</td>\n",
       "      <td>0.288654</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.250539</td>\n",
       "      <td>0.500202</td>\n",
       "      <td>0.749741</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$location_{impact.B}$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.398842</td>\n",
       "      <td>0.242892</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.197668</td>\n",
       "      <td>0.361345</td>\n",
       "      <td>0.575490</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$displacement_A$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.459782</td>\n",
       "      <td>0.325586</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.122403</td>\n",
       "      <td>0.457617</td>\n",
       "      <td>0.673070</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$displacement_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.405861</td>\n",
       "      <td>0.288774</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.219923</td>\n",
       "      <td>0.362168</td>\n",
       "      <td>0.543024</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$length_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.639653</td>\n",
       "      <td>0.318023</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.442238</td>\n",
       "      <td>0.748682</td>\n",
       "      <td>0.937961</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$width_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.589180</td>\n",
       "      <td>0.318618</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.592339</td>\n",
       "      <td>0.894425</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         count      mean       std  min       25%       50%  \\\n",
       "$type_A$               74880.0  0.490008  0.334118  0.0  0.166667  0.500000   \n",
       "$type_B$               74880.0  0.491090  0.331719  0.0  0.230769  0.538462   \n",
       "$type_{oil}$           74880.0  0.492446  0.376486  0.0  0.000000  0.333333   \n",
       "$velocity_A$           74880.0  0.498944  0.288640  0.0  0.247730  0.498735   \n",
       "$velocity_B$           74880.0  0.499480  0.288556  0.0  0.249848  0.499230   \n",
       "$angle_{collision}$    74880.0  0.500327  0.288654  0.0  0.250539  0.500202   \n",
       "$location_{impact.B}$  74880.0  0.398842  0.242892  0.0  0.197668  0.361345   \n",
       "$displacement_A$       74880.0  0.459782  0.325586  0.0  0.122403  0.457617   \n",
       "$displacement_B$       74880.0  0.405861  0.288774  0.0  0.219923  0.362168   \n",
       "$length_B$             74880.0  0.639653  0.318023  0.0  0.442238  0.748682   \n",
       "$width_B$              74880.0  0.589180  0.318618  0.0  0.421053  0.592339   \n",
       "\n",
       "                            75%  max  \n",
       "$type_A$               0.833333  1.0  \n",
       "$type_B$               0.769231  1.0  \n",
       "$type_{oil}$           1.000000  1.0  \n",
       "$velocity_A$           0.749022  1.0  \n",
       "$velocity_B$           0.750018  1.0  \n",
       "$angle_{collision}$    0.749741  1.0  \n",
       "$location_{impact.B}$  0.575490  1.0  \n",
       "$displacement_A$       0.673070  1.0  \n",
       "$displacement_B$       0.543024  1.0  \n",
       "$length_B$             0.937961  1.0  \n",
       "$width_B$              0.894425  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(X_train_dev.describe().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "RnFESHwJwoiO",
    "outputId": "46db1c63-b8d5-4eed-d40c-0fe0616fae4b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alpha': 0.9,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'criterion': 'friedman_mse',\n",
       " 'init': None,\n",
       " 'learning_rate': 0.1,\n",
       " 'loss': 'ls',\n",
       " 'max_depth': 3,\n",
       " 'max_features': None,\n",
       " 'max_leaf_nodes': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_iter_no_change': None,\n",
       " 'presort': 'deprecated',\n",
       " 'random_state': None,\n",
       " 'subsample': 1.0,\n",
       " 'tol': 0.0001,\n",
       " 'validation_fraction': 0.1,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gbrt = GradientBoostingRegressor()\n",
    "model_gbrt.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M_KPqi5awoiP"
   },
   "source": [
    "## Initial Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 294659,
     "status": "ok",
     "timestamp": 1606631588306,
     "user": {
      "displayName": "Tanmoy Das",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgMmxHcU_jnpEQnP9s8nrNXQJMgMWwxHelgPXC5=s64",
      "userId": "02309773046279713936"
     },
     "user_tz": 240
    },
    "id": "TmPEIDFxrtHO",
    "outputId": "c89e27c5-9c42-422a-949b-bf63266b97b5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7937048131362022\n"
     ]
    }
   ],
   "source": [
    "model = GradientBoostingRegressor(max_depth = 4, learning_rate=1, n_estimators=10, random_state=0)\n",
    "model_gbrt = MultiOutputRegressor(model, n_jobs=30) #\n",
    "\n",
    "model_gbrt.fit(X_train_dev, y_train_dev)\n",
    "print(model_gbrt.score(X_val, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>r2_gbrt</th>\n",
       "      <th>neg_mean_squared_error_gbrt</th>\n",
       "      <th>neg_median_absolute_error_gbrt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.448889</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.908323</td>\n",
       "      <td>-0.003641</td>\n",
       "      <td>-0.015363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.448889</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.872836</td>\n",
       "      <td>-0.005085</td>\n",
       "      <td>-0.017101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.868438</td>\n",
       "      <td>-0.005151</td>\n",
       "      <td>-0.002933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.868254</td>\n",
       "      <td>-0.005163</td>\n",
       "      <td>-0.003951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.361111</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.861596</td>\n",
       "      <td>-0.005677</td>\n",
       "      <td>-0.030874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.185556</td>\n",
       "      <td>5.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.782763</td>\n",
       "      <td>-0.009656</td>\n",
       "      <td>-0.065057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.536667</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.779106</td>\n",
       "      <td>-0.009906</td>\n",
       "      <td>-0.042250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.273333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.745657</td>\n",
       "      <td>-0.011359</td>\n",
       "      <td>-0.070110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.097778</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.421054</td>\n",
       "      <td>-0.027082</td>\n",
       "      <td>-0.114142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.097778</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.147944</td>\n",
       "      <td>-0.040693</td>\n",
       "      <td>-0.153109</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   learning_rate  n_estimators  max_depth   r2_gbrt  \\\n",
       "7       0.448889           5.0       10.0  0.908323   \n",
       "1       0.448889           4.0       20.0  0.872836   \n",
       "6       0.800000           5.0       20.0  0.868438   \n",
       "5       0.800000           3.0       20.0  0.868254   \n",
       "0       0.361111           4.0       20.0  0.861596   \n",
       "8       0.185556           5.0       20.0  0.782763   \n",
       "4       0.536667           4.0        5.0  0.779106   \n",
       "2       0.273333           3.0       10.0  0.745657   \n",
       "3       0.097778           5.0        5.0  0.421054   \n",
       "9       0.097778           2.0        3.0  0.147944   \n",
       "\n",
       "   neg_mean_squared_error_gbrt  neg_median_absolute_error_gbrt  \n",
       "7                    -0.003641                       -0.015363  \n",
       "1                    -0.005085                       -0.017101  \n",
       "6                    -0.005151                       -0.002933  \n",
       "5                    -0.005163                       -0.003951  \n",
       "0                    -0.005677                       -0.030874  \n",
       "8                    -0.009656                       -0.065057  \n",
       "4                    -0.009906                       -0.042250  \n",
       "2                    -0.011359                       -0.070110  \n",
       "3                    -0.027082                       -0.114142  \n",
       "9                    -0.040693                       -0.153109  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 04.04.2021 Tanmoy at 10.45am\n",
    "model = MultiOutputRegressor(GradientBoostingRegressor())\n",
    "\n",
    "hyperparameters = dict(estimator__learning_rate= np.linspace(0.01,.8,10),#[0.001, .05, 0.01, 0.1, 0.2, 0.5, 0.9], \n",
    "                    estimator__n_estimators=[2,3,4,5,10,20],\n",
    "                 #    estimator__criterion=['friedman_mse', 'mse'], estimator__min_samples_split=[2, 4, 7, 10],\n",
    "                     estimator__max_depth=[3, 5, 10, 15, 20] ) #, estimator__min_samples_leaf=[1, 2, 3, 5, 8, 10],\n",
    "                #     estimator__min_impurity_decrease=[0, 0.2, 0.4, 0.6, 0.8],\n",
    "                 #    estimator__max_leaf_nodes=[5, 10, 20, 30, 50, 100, 300])\n",
    "number_models = 10\n",
    "\n",
    "randomized_search_model_gbrt = RandomizedSearchCV(model, hyperparameters, #n_iter=number_models,  #random_state=0, \n",
    "                                                  scoring=['r2', 'neg_mean_squared_error', 'neg_median_absolute_error'],\n",
    "                                                   n_jobs=50, refit=False, cv=2, verbose=0, #n_jobs=1 to get Verbose output (runtime of each model) for each iteration\n",
    "                                                   pre_dispatch='2*n_jobs', error_score='raise', return_train_score=True)\n",
    "\n",
    "randomized_search_model_gbrt = randomized_search_model_gbrt.fit(X_train_dev, y_train_dev)\n",
    "#print('Best Parameters = {}'.format(randomized_search_model_gbrt.best_params_))\n",
    "# tuned_model = randomized_search_model_gbrt.best_estimator_\n",
    "\n",
    "# https://stackoverflow.com/questions/43532811/gridsearch-over-multioutputregressor/52562463#52562463\n",
    "\n",
    "\n",
    "# Calculate scores on validation training set +++ \n",
    "learning_rate = [item['estimator__learning_rate'] for item in randomized_search_model_gbrt.cv_results_['params']]\n",
    "n_estimators = [item['estimator__n_estimators'] for item in randomized_search_model_gbrt.cv_results_['params']]\n",
    "max_depth = [item['estimator__max_depth'] for item in randomized_search_model_gbrt.cv_results_['params']]\n",
    "\n",
    "r2_gbrt = list(randomized_search_model_gbrt.cv_results_['mean_test_r2'])\n",
    "neg_mean_squared_error_gbrt = list(randomized_search_model_gbrt.cv_results_['mean_test_neg_mean_squared_error'])\n",
    "neg_median_absolute_error_gbrt = list(randomized_search_model_gbrt.cv_results_['mean_test_neg_median_absolute_error'])\n",
    "\n",
    "\n",
    "df_rands_gbrt_perfm = pd.DataFrame([learning_rate, n_estimators, max_depth, \n",
    "                                    r2_gbrt, neg_mean_squared_error_gbrt, neg_median_absolute_error_gbrt]).T\n",
    "df_rands_gbrt_perfm.columns = ['learning_rate','n_estimators', 'max_depth', \n",
    "                               'r2_gbrt', 'neg_mean_squared_error_gbrt','neg_median_absolute_error_gbrt']\n",
    "# df_rands_dnn_perfm.groupby(['Learning Rate']).mean()\n",
    "#display(df_rands_gbrt_perfm)\n",
    "display(df_rands_gbrt_perfm.sort_values(by=['r2_gbrt'],ascending=False)) #ascending=False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TPOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TPOT\n",
    "from tpot import TPOTRegressor\n",
    "\n",
    "config_dict = { 'sklearn.ensemble.GradientBoostingRegressor':  { 'learning_rate' : np.linspace(0.01,.9,100),\n",
    "                    'max_depth' : list(range(2, 11, 4)) }}\n",
    "tpot_gbrt_model = TPOTRegressor(generations=2, population_size=5,\n",
    "                    verbosity=2, offspring_size=2,\n",
    "                    scoring='r2', cv=2, config_dict = config_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.8660274667468869\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.8912501809554101\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(GradientBoostingRegressor(input_matrix, learning_rate=0.12686868686868688, max_depth=10), learning_rate=0.882020202020202, max_depth=2)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9043795125872096\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9043795125872096\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(GradientBoostingRegressor(input_matrix, learning_rate=0.15383838383838386, max_depth=10), learning_rate=0.27070707070707073, max_depth=10)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9025658158837089\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9025658158837089\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(CombineDFs(input_matrix, input_matrix), learning_rate=0.2976767676767677, max_depth=10)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9541953258740021\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9541953258740021\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(input_matrix, learning_rate=0.17181818181818184, max_depth=10)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9668641287693318\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9668641287693318\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(input_matrix, learning_rate=0.2167676767676768, max_depth=10)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9631859774944118\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9631859774944118\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(GradientBoostingRegressor(input_matrix, learning_rate=0.3156565656565657, max_depth=10), learning_rate=0.7741414141414142, max_depth=10)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9651365741933935\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9651365741933935\n",
      "\n",
      "Best pipeline: GradientBoostingRegressor(GradientBoostingRegressor(input_matrix, learning_rate=0.5224242424242425, max_depth=10), learning_rate=0.4325252525252526, max_depth=6)\n",
      "TPOT EA on GBRT: \n",
      "0.9446493010904197\n"
     ]
    }
   ],
   "source": [
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "TPOT_multi = MultiOutputRegressor(tpot_gbrt_model)\n",
    "TPOT_multi.fit(X_train_dev, y_train_dev)\n",
    "\n",
    "print('TPOT EA on GBRT: ')\n",
    "print(TPOT_multi.score(X_val, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2fNDB_n8tK4C"
   },
   "source": [
    "# Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "2q_SlkbjwoiR",
    "outputId": "ec20ccd6-7483-40dd-cba5-0c3c958cd556",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>r2_gbrt</th>\n",
       "      <th>neg_mean_squared_error_gbrt</th>\n",
       "      <th>neg_median_absolute_error_gbrt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.500</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.919382</td>\n",
       "      <td>-0.003071</td>\n",
       "      <td>-0.013188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>0.500</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.910391</td>\n",
       "      <td>-0.003535</td>\n",
       "      <td>-0.013929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.200</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.901756</td>\n",
       "      <td>-0.003964</td>\n",
       "      <td>-0.023332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>0.500</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.900875</td>\n",
       "      <td>-0.003964</td>\n",
       "      <td>-0.017037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>0.500</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.877629</td>\n",
       "      <td>-0.004985</td>\n",
       "      <td>-0.026289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.001</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.003463</td>\n",
       "      <td>-0.048097</td>\n",
       "      <td>-0.176225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.001</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.002596</td>\n",
       "      <td>-0.048141</td>\n",
       "      <td>-0.176329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.001</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.001689</td>\n",
       "      <td>-0.048190</td>\n",
       "      <td>-0.176490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.001</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.001247</td>\n",
       "      <td>-0.048211</td>\n",
       "      <td>-0.176516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.001</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.000857</td>\n",
       "      <td>-0.048230</td>\n",
       "      <td>-0.176562</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    learning_rate  n_estimators  max_depth   r2_gbrt  \\\n",
       "59          0.500          10.0       10.0  0.919382   \n",
       "58          0.500           5.0       10.0  0.910391   \n",
       "44          0.200          10.0       10.0  0.901756   \n",
       "57          0.500           4.0       10.0  0.900875   \n",
       "56          0.500           3.0       10.0  0.877629   \n",
       "..            ...           ...        ...       ...   \n",
       "2           0.001           4.0        3.0  0.003463   \n",
       "1           0.001           3.0        3.0  0.002596   \n",
       "10          0.001           1.0       10.0  0.001689   \n",
       "5           0.001           1.0        5.0  0.001247   \n",
       "0           0.001           1.0        3.0  0.000857   \n",
       "\n",
       "    neg_mean_squared_error_gbrt  neg_median_absolute_error_gbrt  \n",
       "59                    -0.003071                       -0.013188  \n",
       "58                    -0.003535                       -0.013929  \n",
       "44                    -0.003964                       -0.023332  \n",
       "57                    -0.003964                       -0.017037  \n",
       "56                    -0.004985                       -0.026289  \n",
       "..                          ...                             ...  \n",
       "2                     -0.048097                       -0.176225  \n",
       "1                     -0.048141                       -0.176329  \n",
       "10                    -0.048190                       -0.176490  \n",
       "5                     -0.048211                       -0.176516  \n",
       "0                     -0.048230                       -0.176562  \n",
       "\n",
       "[75 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 05.01.2021 Tanmoy at 10.45am\n",
    "model = MultiOutputRegressor(GradientBoostingRegressor())\n",
    "\n",
    "hyperparameters = dict(estimator__learning_rate=[0.001, 0.1, 0.2, 0.5, 0.9], \n",
    "                    estimator__n_estimators=[1,3,4,5,10],\n",
    "                    estimator__max_depth=[3, 5, 10] ) \n",
    "\n",
    "\n",
    "grid_search_model_gbrt = GridSearchCV(model, hyperparameters, #n_iter=number_models,  #random_state=0, \n",
    "                                                  scoring=['r2', 'neg_mean_squared_error', 'neg_median_absolute_error'],\n",
    "                                                   n_jobs=50, refit=False, cv=2, verbose=0, #n_jobs=1 to get Verbose output (runtime of each model) for each iteration\n",
    "                                                   pre_dispatch='2*n_jobs', error_score='raise', return_train_score=True)\n",
    "\n",
    "grid_search_model_gbrt = grid_search_model_gbrt.fit(X_train_dev, y_train_dev)\n",
    "\n",
    "learning_rate = [item['estimator__learning_rate'] for item in grid_search_model_gbrt.cv_results_['params']]\n",
    "n_estimators = [item['estimator__n_estimators'] for item in grid_search_model_gbrt.cv_results_['params']]\n",
    "max_depth = [item['estimator__max_depth'] for item in grid_search_model_gbrt.cv_results_['params']]\n",
    "\n",
    "r2_gbrt = list(grid_search_model_gbrt.cv_results_['mean_test_r2'])\n",
    "neg_mean_squared_error_gbrt = list(grid_search_model_gbrt.cv_results_['mean_test_neg_mean_squared_error'])\n",
    "neg_median_absolute_error_gbrt = list(grid_search_model_gbrt.cv_results_['mean_test_neg_median_absolute_error'])\n",
    "\n",
    "\n",
    "df_grids_gbrt_perfm = pd.DataFrame([learning_rate, n_estimators, max_depth, \n",
    "                                    r2_gbrt, neg_mean_squared_error_gbrt, neg_median_absolute_error_gbrt]).T\n",
    "df_grids_gbrt_perfm.columns = ['learning_rate','n_estimators', 'max_depth', \n",
    "                               'r2_gbrt', 'neg_mean_squared_error_gbrt','neg_median_absolute_error_gbrt']\n",
    "# df_rands_dnn_perfm.groupby(['Learning Rate']).mean()\n",
    "display(df_grids_gbrt_perfm.sort_values(by=['r2_gbrt'],ascending=False)) #ascending=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "jIy7ps8GwoiS",
    "outputId": "deb1880d-0283-4d25-8f60-9086c5fc0b3b"
   },
   "outputs": [],
   "source": [
    "df_rands_gbrt_perfm.to_csv('df_rands_gbrt_perfm.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RuRMySBh6y_e"
   },
   "source": [
    "# Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gbrt = GradientBoostingRegressor()\n",
    "model_gbrt.get_params()\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "7T5H9s116y_f",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from hyperopt import hp\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.metrics import r2_score\n",
    "# Setting the space\n",
    "space = {\n",
    "    'max_depth': hp.uniform('max_depth', 4, 20),\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.001, .1),\n",
    "   'n_estimators': hp.uniform('n_estimators', 1,20)\n",
    "}# r2_score\n",
    "def objective_r2(params_bo):\n",
    "    params_bo = {'max_depth': int(params_bo['max_depth']),\n",
    "        'learning_rate': params_bo['learning_rate'],\n",
    "               'n_estimators': int(params_bo['n_estimators'])}\n",
    "    gbrt_bo = GradientBoostingRegressor(**params_bo)\n",
    "    #\n",
    "    gbrt_bo_multi = MultiOutputRegressor(gbrt_bo)\n",
    "    gbrt_bo_multi.fit(X_train_dev, y_train_dev)\n",
    "    #r2_bo = (-1)* cross_val_score(neural_net_bo, X_train, y_train, scoring='r2', cv=5).mean()\n",
    "    y_pred_bo = gbrt_bo_multi.predict(X_val)\n",
    "    r2_bo = r2_score(y_val, y_pred_bo) # If False, returns RMSE value\n",
    "    print('r2_bo: ', r2_bo)\n",
    "    print('max_depth, learning_rate, n_estimators', \n",
    "          params_bo['max_depth'], params_bo['learning_rate'], params_bo['n_estimators'])\n",
    "    return r2_bo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2_bo:                                                                                                                 \n",
      "0.26298845580744457                                                                                                    \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "4                                                                                                                      \n",
      "0.016667871589894637                                                                                                   \n",
      "19                                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.6823178130601081                                                                                                     \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "13                                                                                                                     \n",
      "0.09345933320699129                                                                                                    \n",
      "7                                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.3770250609511633                                                                                                     \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "7                                                                                                                      \n",
      "0.06647758681375023                                                                                                    \n",
      "5                                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.08667651798546894                                                                                                    \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "17                                                                                                                     \n",
      "0.024143034968477226                                                                                                   \n",
      "2                                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.4604320889626461                                                                                                     \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "13                                                                                                                     \n",
      "0.02057353231933003                                                                                                    \n",
      "17                                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.051810936539359566                                                                                                   \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "18                                                                                                                     \n",
      "0.0035710845790574374                                                                                                  \n",
      "8                                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.11032164511613862                                                                                                    \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "10                                                                                                                     \n",
      "0.0342431094612736                                                                                                     \n",
      "2                                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.8445128387709302                                                                                                     \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "14                                                                                                                     \n",
      "0.08298900249437612                                                                                                    \n",
      "14                                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.17050960187803194                                                                                                    \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "6                                                                                                                      \n",
      "0.06794602397622367                                                                                                    \n",
      "2                                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.3589561224467387                                                                                                     \n",
      "max_depth, learning_rate, n_estimators                                                                                 \n",
      "4                                                                                                                      \n",
      "0.05367064738490478                                                                                                    \n",
      "9                                                                                                                      \n",
      "100%|██████████████████████████████████████████████| 10/10 [04:24<00:00, 26.40s/trial, best loss: 0.051810936539359566]\n"
     ]
    }
   ],
   "source": [
    "from hyperopt import fmin, tpe\n",
    "best_result = fmin(fn=objective_r2,\n",
    "            space=space,\n",
    "            max_evals=10,\n",
    "            rstate=np.random.RandomState(22),\n",
    "            algo=tpe.suggest)\n",
    "# print the best parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "GBRT_m_04.2021.ipynb",
   "provenance": [
    {
     "file_id": "11Bmr6bkWBMY2NNXjxTsSDv7Wd6gGYnTK",
     "timestamp": 1612927343896
    },
    {
     "file_id": "1YvjXPeKCMulXpZybbAw5VybgNglnJfWk",
     "timestamp": 1606116852726
    }
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}