Mini-Project Part 3 Supervised Learning (1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0fced5da",
   "metadata": {},
   "source": [
    "## Mini-Project Part 3: Supervised Learning (1)\n",
    "### Logistic Regression, KNN, Decision Tree, and Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30a39c46",
   "metadata": {},
   "source": [
    "In this part, we will continue to use the dataset about adaptivity to online education.  Target for models is predicting adaptability levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "467e7bfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Network Type</th>\n",
       "      <th>Class Duration</th>\n",
       "      <th>Gender_Boy</th>\n",
       "      <th>Gender_Girl</th>\n",
       "      <th>Education Level_College</th>\n",
       "      <th>Education Level_School</th>\n",
       "      <th>Education Level_University</th>\n",
       "      <th>Institution Type_Government</th>\n",
       "      <th>Institution Type_Non Government</th>\n",
       "      <th>...</th>\n",
       "      <th>Financial Condition_Mid</th>\n",
       "      <th>Financial Condition_Poor</th>\n",
       "      <th>Financial Condition_Rich</th>\n",
       "      <th>Internet Type_Mobile Data</th>\n",
       "      <th>Internet Type_Wifi</th>\n",
       "      <th>Self Lms_No</th>\n",
       "      <th>Self Lms_Yes</th>\n",
       "      <th>Device_Computer</th>\n",
       "      <th>Device_Mobile</th>\n",
       "      <th>Device_Tab</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age  Network Type  Class Duration  Gender_Boy  Gender_Girl  \\\n",
       "0    5             4               2           1            0   \n",
       "1    5             4               1           0            1   \n",
       "2    4             4               1           0            1   \n",
       "\n",
       "   Education Level_College  Education Level_School  \\\n",
       "0                        0                       0   \n",
       "1                        0                       0   \n",
       "2                        1                       0   \n",
       "\n",
       "   Education Level_University  Institution Type_Government  \\\n",
       "0                           1                            0   \n",
       "1                           1                            0   \n",
       "2                           0                            1   \n",
       "\n",
       "   Institution Type_Non Government  ...  Financial Condition_Mid  \\\n",
       "0                                1  ...                        1   \n",
       "1                                1  ...                        1   \n",
       "2                                0  ...                        1   \n",
       "\n",
       "   Financial Condition_Poor  Financial Condition_Rich  \\\n",
       "0                         0                         0   \n",
       "1                         0                         0   \n",
       "2                         0                         0   \n",
       "\n",
       "   Internet Type_Mobile Data  Internet Type_Wifi  Self Lms_No  Self Lms_Yes  \\\n",
       "0                          0                   1            1             0   \n",
       "1                          1                   0            0             1   \n",
       "2                          0                   1            1             0   \n",
       "\n",
       "   Device_Computer  Device_Mobile  Device_Tab  \n",
       "0                0              0           1  \n",
       "1                0              1           0  \n",
       "2                0              1           0  \n",
       "\n",
       "[3 rows x 26 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Encode variables using the part one code\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "data = pd.read_csv(r\"C:\\Users\\roryq\\Downloads\\online_adapt.csv\")\n",
    "\n",
    "# Encode `Age` to integers, 1, 2, 3, 4, 5, 6.\n",
    "\n",
    "age_mapper = {'26-30':6, '21-25':5, '16-20':4, '11-15':3, '6-10':2, '1-5':1}\n",
    "age_t = data['Age'].replace(age_mapper)\n",
    "\n",
    "# Encode `Network Type` to integers, 2, 3, 4.\n",
    "\n",
    "net_mapper = {'2G':2, '3G':3, '4G':4}\n",
    "net_t = data['Network Type'].replace(net_mapper)\n",
    "\n",
    "# Encode `Class Duration` to integers, 0, 1, 2.\n",
    "\n",
    "class_mapper = {'0':0, '1-3':1, '3-6':2}\n",
    "class_t = data['Class Duration'].replace(class_mapper)\n",
    "\n",
    "# Replace `Age`, `Network Type`, `Class Duration` by their corresponding numeric versions.\n",
    "\n",
    "data['Age'] = age_t\n",
    "data['Network Type'] = net_t\n",
    "data['Class Duration'] = class_t\n",
    "\n",
    "# One-hot encode the rest of the variables except for the response variable, `Adaptivity Level`.\n",
    "y = data['Adaptivity Level']\n",
    "data1 = pd.get_dummies(data.drop('Adaptivity Level', axis=1),dtype=int)\n",
    "\n",
    "\n",
    "# Check data\n",
    "data1.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "0b927dec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1205, 26)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14d302de",
   "metadata": {},
   "source": [
    "## Regularized Logistic Regression\n",
    "Now, fit a regularized logistic regression model on `data1`. \n",
    "\n",
    "\n",
    "1. Standardize all features of `data1`. \n",
    "2. Split data (80% training; 20% test) (set `random_state=100`)\n",
    "3. Use `L2` penalty.\n",
    "4. Use 5-fold cross-validation to find the optimal `C`.\n",
    "5. Consider 20 values of the hyperparameter `C`.\n",
    "6. Calculate the confusion matrix. \n",
    "7. Calculate the accuracy of the test set using the final model.\n",
    "8. Calculate the accuracy for each class of `y`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "57570dd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7095435684647303"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import datasets\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "\n",
    "# Create features and target to specifcy for the rest of models in the part\n",
    "# Target is adaptability level\n",
    "# Features are all other variables in the data set specified in read me file\n",
    "features = data1\n",
    "target = y\n",
    "\n",
    "# Standardize all features of the data with standard scaler\n",
    "scaler = StandardScaler()\n",
    "features_standardized = scaler.fit_transform(features)\n",
    "\n",
    "# define your test split on data with 80% training and 20% test\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    features_standardized, target, random_state=100, test_size= .2)\n",
    "\n",
    "# Create logistics regression specifications\n",
    "logistic_regression = LogisticRegressionCV(cv=5, # Use 5 fold cross validation\n",
    "    solver='lbfgs', \n",
    "    multi_class='multinomial', \n",
    "    penalty='l2',   # Use L2 penalty\n",
    "    Cs=20,          # Check 20 values of our hyperparameter\n",
    "    random_state=100, n_jobs=-1)\n",
    "\n",
    "# Create model with out logistics regression specifications\n",
    "model_cv = logistic_regression.fit(X_train, y_train)\n",
    "\n",
    "# Predictions of our model for out test data\n",
    "y_pred_cv = model_cv.predict(X_test)\n",
    "\n",
    "# accuracy for final test\n",
    "np.mean(y_pred_cv==y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2f75912f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Moderate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>High</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Low</th>\n",
       "      <td>7</td>\n",
       "      <td>64</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Moderate</th>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          High  Low  Moderate\n",
       "High         9    1         4\n",
       "Low          7   64        24\n",
       "Moderate     7   27        98"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# Get lables to use for confusion matrix\n",
    "labels = np.unique(y_test)\n",
    "\n",
    "# Create confusion matrix from predicted and test data\n",
    "matrix= confusion_matrix(y_pred_cv, y_test, labels= labels)\n",
    "\n",
    "# Print matrix with lables\n",
    "pd.DataFrame(matrix, index=labels, columns=labels)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "45149d05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy [0.64285714 0.67368421 0.74242424]\n"
     ]
    }
   ],
   "source": [
    "# Print accuracy for each level of adaptability\n",
    "print(\"Accuracy\", matrix.diagonal()/matrix.sum(axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3203dcdb",
   "metadata": {},
   "source": [
    "#### Accuracy for each level of adaptability\n",
    "+ Accruacy for High is .6428\n",
    "+ Accuracy for Low is .6736\n",
    "+ Accuracy for Moderate is .7424\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e34d107f",
   "metadata": {},
   "source": [
    "## K Nearest Neighbors and Radius Nearest Neighbors\n",
    " \n",
    "\n",
    "1. Split data (80% training; 20% test) (set `random_state=100`)\n",
    "2. Use the min-max scaler to preprocess features.\n",
    "3. Use 5-fold cross-validation in the grid-search.\n",
    "4. Consider 10 different values of `K`, i.e. 1, 2, 3, ..., 10.\n",
    "5. After you find the optimal value of `K`, use the entire training set to fit the KNN model and calculate the accuracy for the test set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "82d00c5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load libraries\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# set x and y for model\n",
    "X = features\n",
    "y = target\n",
    "\n",
    "\n",
    "\n",
    "# Split data to training and testing sets and select size\n",
    "X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2,\n",
    "                                                    random_state=100, stratify=target)\n",
    "# select standardizer\n",
    "standardizer = MinMaxScaler()\n",
    "\n",
    "# Set KNN with classifier\n",
    "knn = KNeighborsClassifier(n_jobs=-1) \n",
    "\n",
    "\n",
    "pipe = Pipeline([(\"standardizer\", standardizer), (\"knn\", knn)])\n",
    "\n",
    "# consider 10 different values of K\n",
    "search_space = [{\"knn__n_neighbors\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}]\n",
    "\n",
    "# Create search with parameters and set 5 fold cross validation\n",
    "classifier = GridSearchCV(\n",
    "    pipe, search_space, cv=5, verbose=0).fit(X_train, y_train)\n",
    "\n",
    "# Calculate best estimator\n",
    "classifier.best_estimator_.get_params()[\"knn__n_neighbors\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4aa8102",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 0.89\n"
     ]
    }
   ],
   "source": [
    "# Create knn model\n",
    "knn_1 = KNeighborsClassifier(n_neighbors=1, n_jobs=-1)\n",
    "\n",
    "# Standarize data to train and test\n",
    "X_train_sc = standardizer.fit_transform(X_train)\n",
    "X_test_sc = standardizer.fit_transform(X_test)\n",
    "\n",
    "# Train and test model\n",
    "knn_1.fit(X_train_sc,y_train)\n",
    "\n",
    "# Print accuracy of model\n",
    "print(\"Test set accuracy: {:.2f}\".format(knn_1.score(X_test_sc,y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d0e0b26",
   "metadata": {},
   "source": [
    "#### Radius Nearest Neighbors \n",
    "\n",
    "* first split the data (80% training; 20% test). Set `random_state=100`.\n",
    "* normalize the features.\n",
    "* use 10-fold cross-validation in the grid-search.\n",
    "* consider about 200 different values of radius between 0.01 and 2.\n",
    "* Set `outlier_label='most_frequent'` for `RadiusNeighborsClassifier`. \n",
    "* After finding the optimal value of the radius, use the entire training set to fit the RNN model and calculate the accuracy for the test set.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bbc8e3d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load libraries\n",
    "from sklearn.neighbors import RadiusNeighborsClassifier\n",
    "from sklearn.preprocessing import Normalizer\n",
    "\n",
    "\n",
    "# Set normalizer\n",
    "# Set features and target\n",
    "# Create normalized data\n",
    "normalizer = Normalizer()\n",
    "features = data1\n",
    "target = y\n",
    "X_norm = normalizer.fit_transform(features)\n",
    "\n",
    "\n",
    "\n",
    "# Set train and test data, specify split, and random state\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_norm, target, test_size=0.2,\n",
    "                                                    random_state=100,stratify=target)\n",
    "\n",
    "\n",
    "# Create rnn using radius neighbor classifier\n",
    "rnn = RadiusNeighborsClassifier(outlier_label='most_frequent') \n",
    "\n",
    "pipe = Pipeline([(\"normalizer\", normalizer), (\"rnn\", rnn)])\n",
    "\n",
    "# Specify search space\n",
    "SearchSpace=[{'rnn__radius': np.arange(0.01, 200, 2) }]\n",
    "\n",
    "# Create classifier\n",
    "    # Use 10 fold cross validation\n",
    "    # fit training data\n",
    "    # Use search space\n",
    "classifier = GridSearchCV(\n",
    "    pipe, SearchSpace, cv=10, verbose=0).fit(X_train, y_train)\n",
    "\n",
    "# Get best estimator from our search\n",
    "classifier.best_estimator_.get_params()[\"rnn__radius\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c553f691",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.950207468879668"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Run model\n",
    "\n",
    "# scale train and test data\n",
    "X_train_sc1 = standardizer.fit_transform(X_train)\n",
    "X_test_sc1 = standardizer.fit_transform(X_test)\n",
    "\n",
    "# fit model with test data, and best radius parameter\n",
    "rnn_final = RadiusNeighborsClassifier(radius=0.01,outlier_label='most_frequent').fit(X_train_sc1,y_train)\n",
    "\n",
    "# Asses model accuracy with test data\n",
    "rnn_final.score(X_test_sc1,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71a24a9e",
   "metadata": {},
   "source": [
    "## Decision Trees\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae46af11",
   "metadata": {},
   "source": [
    "####  Split the data (80% training; 20% test). Set `random_state=100`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3826053b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "X = features\n",
    "\n",
    "# Specify train and test split and x and y for model\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, random_state=100, test_size=.2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "435441ce",
   "metadata": {},
   "source": [
    "####  Train a decision tree (`random_state=100`) without constraints. Calculate the accuracies of the training and the test sets. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a5bbcaed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with trained data is: 0.9273858921161826\n",
      "Accuracy with test data is: 0.9336099585062241\n"
     ]
    }
   ],
   "source": [
    "# Create tree params\n",
    "tree_clf5 = DecisionTreeClassifier(random_state=100)\n",
    "\n",
    "# Train model\n",
    "tree_clf5.fit(X_train,y_train)\n",
    "\n",
    "# Show accuracy of trained data and test data\n",
    "print(\"Accuracy with trained data is:\",tree_clf5.score(X_train, y_train))\n",
    "print(\"Accuracy with test data is:\",tree_clf5.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a4979de",
   "metadata": {},
   "source": [
    "####  Train a decision tree with `random_state=100` and  `max_features=3`. Calculate the accuracies of the training and the test set.\n",
    "may increase the bias and reduce the variance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ed1ee47f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with trained data is: 0.9273858921161826\n",
      "Accuracy with test data is: 0.9377593360995851\n"
     ]
    }
   ],
   "source": [
    "# Specify training, test data, split and x and y for model\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, random_state=100,test_size=.2)\n",
    "\n",
    "# set params for tree with classifier (random state and max features)\n",
    "tree_clf4 = DecisionTreeClassifier(random_state=100, max_features=3)\n",
    "\n",
    "# Train tree\n",
    "tree_clf4.fit(X_train,y_train)\n",
    "\n",
    "# Print tree accuracy\n",
    "print(\"Accuracy with trained data is:\",tree_clf4.score(X_train, y_train))\n",
    "print(\"Accuracy with test data is:\",tree_clf4.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb551800",
   "metadata": {},
   "source": [
    "#### Train a decision tree with `random_state=100` and `min_samples_leaf=5`. Calculate the accuracies of the training and the test set.\n",
    " may increase the bias and reduce the variance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f47fcba6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with trained data is: 0.83298755186722\n",
      "Accuracy with test data is: 0.8049792531120332\n"
     ]
    }
   ],
   "source": [
    "# Specify training, test data, split and x and y for model\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, random_state=100,test_size=.2)\n",
    "\n",
    "# set params for tree with classifier (random state and min samples leaf)\n",
    "tree_clf8 = DecisionTreeClassifier(random_state=100,min_samples_leaf=5 )\n",
    "\n",
    "# Train tree\n",
    "tree_clf8.fit(X_train,y_train)\n",
    "\n",
    "# Print tree accuracy\n",
    "print(\"Accuracy with trained data is:\",tree_clf8.score(X_train, y_train))\n",
    "print(\"Accuracy with test data is:\",tree_clf8.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ac4df71",
   "metadata": {},
   "source": [
    "## Random Forests \n",
    "\n",
    "\n",
    "\n",
    "* first split the data (80% training; 20% test). Set `random_state=100`.\n",
    "* In `RandomForestClassifier`, set `random_state=100` and `n_jobs=-1`.\n",
    "* Calculate the accuracies for the training and the test sets."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87429c0c",
   "metadata": {},
   "source": [
    "#### Train a random forest with 10 trees.\n",
    "Note that the default value of `n_estimators` changed from 10 to 100 in Version 0.22 of `sklearn`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fe7c920f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on training set: 0.925\n",
      "Accuracy on test set: 0.929\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Specify training, test data, split and x and y for model\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    features, target, random_state=100, test_size=.2)\n",
    "\n",
    "# Create forest params with classifier and set random state and estimators\n",
    "forest = RandomForestClassifier(n_estimators=10,\n",
    "                                random_state=100, n_jobs=-1)\n",
    "\n",
    "# Create and Train model using forest params specified \n",
    "model_forest = forest.fit(X_train, y_train)\n",
    "\n",
    "# Print accuracy for model\n",
    "print(\"Accuracy on training set: {:.3f}\".format(model_forest.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(model_forest.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95ab0bdd",
   "metadata": {},
   "source": [
    "####  Train a random forest with 100 trees. Any improvement compared with ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "60852e15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on training set: 0.927\n",
      "Accuracy on test set: 0.938\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Specify training, test data, split and x and y for model\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    features, target, random_state=100, test_size=.2)\n",
    "\n",
    "# Create forest params with classifier and set random state and estimators\n",
    "forest1 = RandomForestClassifier(n_estimators=100,\n",
    "                                random_state=100, n_jobs=-1)\n",
    "\n",
    "# Create and Train model using forest params specified \n",
    "model_forest1 = forest1.fit(X_train, y_train)\n",
    "\n",
    "# Print accuracy for model\n",
    "print(\"Accuracy on training set: {:.3f}\".format(model_forest1.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(model_forest1.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29355545",
   "metadata": {},
   "source": [
    "+ Test accuracy increases by .009 & training set stays approximately the same"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "573b2960",
   "metadata": {},
   "source": [
    "####  Train a random forest with 1000 trees. Any improvement compared with Above?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3da7ac6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on training set: 0.927\n",
      "Accuracy on test set: 0.934\n"
     ]
    }
   ],
   "source": [
    "# Specify training, test data, split and x and y for model\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    features, target, random_state=100,test_size=.2)\n",
    "\n",
    "# Create forest params with classifier and set random state and estimators\n",
    "forest2 = RandomForestClassifier(n_estimators=1000,\n",
    "                                random_state=100, n_jobs=-1)\n",
    "\n",
    "# Create and Train model using forest params specified \n",
    "model_forest2 = forest2.fit(X_train, y_train)\n",
    "\n",
    "# Print accuracy for model\n",
    "print(\"Accuracy on training set: {:.3f}\".format(model_forest2.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(model_forest2.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d18cd397",
   "metadata": {},
   "source": [
    "+ Training and Test accuracy stays approximately the same\n",
    "+ Test decreases by .004"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddb4d0ed",
   "metadata": {},
   "source": [
    "#### Train a random forest with 100 trees with `max_features=10`. Any difference from Part 4.2?\n",
    "The default value for `max_features` is `sqrt(n_features)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b2a6b173",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on training set: 0.927\n",
      "Accuracy on test set: 0.934\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    features, target, random_state=100,test_size=.2)\n",
    "\n",
    "\n",
    "forest3 = RandomForestClassifier(n_estimators=100,\n",
    "                         random_state=100, n_jobs=-1, max_features=10)\n",
    "\n",
    "\n",
    "model_forest3 = forest3.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(model_forest3.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(model_forest3.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e56917e6",
   "metadata": {},
   "source": [
    "+ Test decreases slightly from 4.2\n",
    "+ Train accuracy stays the same as 4.2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9118782a",
   "metadata": {},
   "source": [
    "####  Show the features importance in the model. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f95d5d5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Age 0.08583812991070759\n",
      "Network Type 0.07565276574449656\n",
      "Class Duration 0.16590760615206018\n",
      "Gender_Boy 0.037789738996994046\n",
      "Gender_Girl 0.037781803465573756\n",
      "Education Level_College 0.030388563751933356\n",
      "Education Level_School 0.02271238603454941\n",
      "Education Level_University 0.03459409680515524\n",
      "Institution Type_Government 0.024974667159102366\n",
      "Institution Type_Non Government 0.048354303113187745\n",
      "IT Student_No 0.023221504438925924\n",
      "IT Student_Yes 0.02743510384161384\n",
      "Location_No 0.029671200182698254\n",
      "Location_Yes 0.0233220814202405\n",
      "Load-shedding_High 0.033404993405830676\n",
      "Load-shedding_Low 0.02204304041777359\n",
      "Financial Condition_Mid 0.04054708948528579\n",
      "Financial Condition_Poor 0.04066353310245535\n",
      "Financial Condition_Rich 0.041994417281285785\n",
      "Internet Type_Mobile Data 0.018826427987668388\n",
      "Internet Type_Wifi 0.03970590648400056\n",
      "Self Lms_No 0.018839622864837945\n",
      "Self Lms_Yes 0.01970665294402183\n",
      "Device_Computer 0.03009359668253998\n",
      "Device_Mobile 0.02070333987627108\n",
      "Device_Tab 0.005827428450790262\n"
     ]
    }
   ],
   "source": [
    "for name, score in zip(data1.columns, model_forest.feature_importances_):\n",
    "    print(name, score)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}