SVMnotSVC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.dpi'] = 70 #display 70 dpi in Jupyter Notebook, may consider100 dpi \n",
    "plt.rcParams['savefig.dpi'] = 300 #define 300 dpi for saving figures\n",
    "\n",
    "import seaborn as sns\n",
    "## here are some settings \n",
    "sns.set_style('whitegrid')\n",
    "sns.set(rc={\"figure.dpi\":70, 'savefig.dpi':300}) #defining dpi setting\n",
    "sns.set_context('notebook')\n",
    "sns.set_style(\"ticks\")\n",
    "from IPython.display import set_matplotlib_formats\n",
    "from sklearn.model_selection import LeaveOneGroupOut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World\n",
      "---Run time is 9.889999637380242e-05 seconds ---\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Tells matplotlib to display images inline instead of a new window\n",
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import LeaveOneOut\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint\n",
    "from tqdm import tqdm\n",
    "import wfdb\n",
    "\n",
    "from time import time\n",
    "import timeit #imports timeit module\n",
    "start_time = timeit.default_timer() #defines start time so computational time can be calculated\n",
    "print(\"Hello World\")\n",
    "elapsed = timeit.default_timer() - start_time #gives total computation time\n",
    "print(\"---Run time is %s seconds ---\" % elapsed) #prints computation time\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "import sklearn.linear_model as skl_lm\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis \n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "\n",
    "from sklearn.tree import DecisionTreeRegressor \n",
    "from sklearn.tree import DecisionTreeClassifier \n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import LeaveOneOut\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/25 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sampto must be greater than sampfrom\n",
      "sampto must be greater than sampfrom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25/25 [00:17<00:00,  1.43it/s]\n"
     ]
    }
   ],
   "source": [
    "#data = load_breast_cancer(as_frame = True); \n",
    "#df = data.frame; \n",
    "#print(df.shape)\n",
    "# In this block use the code we looked at on 6/30 as a reference to save our rr-intervals as a .\n",
    "rlist = []\n",
    "records = 'mit-bih-raw\\RECORDS' # Replace the _______ with the name of the records file in your MIT-BIH data folder\n",
    "with open(records) as rfile: #Then we open the file \n",
    "                             #The 'with' command only opens the file while we are in it. Automatically closes the file when we're not\n",
    "    for record in rfile:  # Then we iterate through the lines in the file\n",
    "        record = record[0:len(record)-1] # Remove any erronious new line characters at the end ('\\n')\n",
    "        rlist.append(record) # Then build an array with it\n",
    "        \n",
    "\n",
    "###### Step 1: Initialize all Arrays\n",
    "             # Below, replace all of the ___ with the command that declares an array/list\n",
    "             # hint: https://stackoverflow.com/questions/1514553/how-to-declare-an-array-in-python\n",
    "samples = [] # will house the samples of all subjects\n",
    "good_list = [] # will list the names of the subjects we successfully extracted\n",
    "bad_list = [] # will house the names of the subjects we failed to extract\n",
    "qrs = [] # will house the indices of R-Peaks for all subjects\n",
    "atr_label = [] # will house the labels for each rhythm annotation for all subjects\n",
    "atr_locs = [] # will house the locations corresponding to the rhythm annotation labels\n",
    "\n",
    "\n",
    "###### Step 2: Extract Information\n",
    "for x in tqdm(rlist): #this will iterate through te records that we found above\n",
    "  \n",
    "    try: # A try statement will run the except statement if for some reason the try commands fail\n",
    "         # In this case I use the try statement because one of the subjects has no signal data causing failure\n",
    "         # I then use bad_list and good_list so that all of the indices in rlist match with the arrays we initialized in Step 1, above\n",
    "        ######################################################\n",
    "            # Below find the wfdb function that will return the information that is described below \n",
    "        # Then replace _____ with the correct function call\n",
    "        samp = wfdb.rdsamp('mit-bih-raw/'+x) # wfdb._____(file_location) will read the signal & header data and return a 2 value array\n",
    "            # samp[0] - the signal data is the raw reading from the ecg. Each value is a sample taken.\n",
    "            # samp[1] - the header data includes things about the signal data such as:\n",
    "              # samples per section, denoted 'fs'\n",
    "              # number of signals, denoted 'n_sig'\n",
    "            \n",
    "        ######################################################\n",
    "        samples.append(samp) #add it to our array for all subject\n",
    "            #What is our file extension that has the annotation we want? Find it here and replace _____ with it \n",
    "            #hint: READ THE VARIABLE NAMES!!!!\n",
    "        qrs_tmp = wfdb.rdann('mit-bih-raw/'+x, extension='qrs') #extract the QRS Info\n",
    "        qrs_locs = np.array(qrs_tmp.sample, dtype='int') #Get just the loccation of R-Peaks from the QRS Info\n",
    "        qrs.append(qrs_locs) # Add to our array for all subjects\n",
    "        \n",
    "            #Do the same thing here\n",
    "        atr = wfdb.rdann('mit-bih-raw/'+x,extension='atr') #extract the atr info which stores the rhythm type(s) over the whole signal\n",
    "        atr_label.append(atr.aux_note) # aux_note stores the type of rhythm - main two are '(N' for normal and '(AFIB' for AFIB\n",
    "        atr_locs.append(np.append(atr.sample, len(samp[0]))) #I add the length of the whole sample to the end for better visualization later\n",
    "        \n",
    "        good_list.append(x) # when all extraction is successful append the record name to good_list\n",
    "    except Exception as exep:\n",
    "        print(exep) # Alert the user of an exception\n",
    "        bad_list.append(x) # add to the bad list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Unnamed: 0', 'StoS', 'StoR', 'StoL', 'RtoS', 'RtoR', 'RtoL', 'LtoS',\n",
      "       'LtoR', 'LtoL', 'STD', 'CoefVar', 'Range', 'Volatility', 'ShEn',\n",
      "       'AppEn', 'SampEn', 'Rhythm', 'Subject'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "features_df_list = []\n",
    "for subject in good_list:\n",
    "    data = pd.read_csv(f'subject-features-dataframes/subject{subject}.csv')\n",
    "    data = data.assign(Subject = subject)\n",
    "    #print(data)\n",
    "    features_df_list.append(data)\n",
    "#features_maindf = pd.concat(features_df_list, ignore_index = True)\n",
    "features_maindf = pd.concat(features_df_list, ignore_index = True)\n",
    "print(features_maindf.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time\n",
    "import timeit #imports timeit module\n",
    "start_time = timeit.default_timer() #defines start time so computational time can be calculated\n",
    "\n",
    "acc_score = [];\n",
    "Truth = [];\n",
    "Output = [];\n",
    "n_estimators_list = []\n",
    "max_depth_list = []\n",
    "\n",
    "df = features_maindf\n",
    "x_values = ['StoS', 'StoR', 'StoL', 'RtoS', 'RtoR', 'RtoL', 'LtoS',\n",
    "       'LtoR', 'LtoL', 'STD', 'CoefVar', 'Range', 'Volatility', 'ShEn',\n",
    "       'AppEn', 'SampEn']\n",
    "y_values = ['Rhythm']\n",
    "\n",
    "#logo = LeaveOneGroupOut()\n",
    "#splits = list(logo.split(x_values, y_values, groups=df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Classifier \n",
    "\n",
    "The ${\\tt SVC()}$ function can be used to fit a\n",
    "support vector classifier when the argument ${\\tt kernel=\"linear\"}$ is used. This\n",
    "function uses a slightly different formulation of the equations we saw in lecture to build the\n",
    "support vector classifier. The ${\\tt c}$ argument allows us to specify the cost of\n",
    "a violation to the margin. When the ${\\tt c}$ argument is **small**, then the margins\n",
    "will be wide and many support vectors will be on the margin or will\n",
    "violate the margin. When the ${\\tt c}$ argument is large, then the margins will\n",
    "be narrow and there will be few support vectors on the margin or violating\n",
    "the margin.\n",
    "\n",
    "We can use the ${\\tt SVC()}$ function to fit the support vector classifier for a\n",
    "given value of the ${\\tt cost}$ parameter. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #for df in features_df_list:\n",
    "# #model = DecisionTreeClassifier(max_depth = 6) #criterion='entropy'\n",
    "# df = features_maindf\n",
    "# acc_score = [];\n",
    "# Truth = [];\n",
    "# Output = [];\n",
    "# max_depth_list = []\n",
    "# for idx, subject in tqdm(enumerate(good_list)):\n",
    "    \n",
    "#     train_index = ~df['Subject'].str.contains(subject)\n",
    "#     test_index = df['Subject'].str.contains(subject)\n",
    "#     X_train , X_test = df.loc[train_index,x_values], df.loc[test_index,x_values]\n",
    "#     y_train , y_test = df.loc[train_index,y_values].values.ravel(), df.loc[test_index,y_values]\n",
    "\n",
    "# #     temp_model = DecisionTreeClassifier(max_depth = 6)    \n",
    "# #     param_distributions = {'max_depth': randint(7,9)}\n",
    "\n",
    "# #     search = RandomizedSearchCV(estimator=temp_model, param_distributions=param_distributions, random_state=0)\n",
    "# #     search.fit(X_train, y_train)\n",
    "\n",
    "# #     max_depth_list.append(search.best_params_['max_depth'])\n",
    "    \n",
    "#     model = SVC(C=1, kernel='linear', random_state=2, probability = True) #criterion='entropy'\n",
    "#     model.fit(X_train,y_train)\n",
    "\n",
    "#     pred_values = model.predict(X_test)\n",
    "#     pred_prob = model.predict_proba(X_test)[:, 1]\n",
    "     \n",
    "#     acc = accuracy_score(y_test, pred_values)\n",
    "#     acc_score.append(acc)\n",
    "    \n",
    "#     Truth.extend(y_test.values.reshape(y_test.shape[0])); ## it is a list\n",
    "#     Output.extend(pred_values); ## it is a list \n",
    "# elapsed = timeit.default_timer() - start_time #gives total computation time\n",
    "# print(\"---Run time is %s seconds ---\" % elapsed) #prints computation time\n",
    "# print()\n",
    "\n",
    "# print('Accuracy of each fold: \\n {}'.format(acc_score))\n",
    "# print()\n",
    "# print('Avg accuracy : \\n{}'.format(np.mean(acc_score))); \n",
    "# print()\n",
    "# print('Std of accuracy : \\n{}'.format(np.std(acc_score)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ${\\tt sklearn.grid\\_search}$ module\n",
    "\n",
    "The ${\\tt sklearn.grid\\_search}$ module includes a a function ${\\tt GridSearchCV()}$ to perform cross-validation. In order to use this function, we pass in relevant information about the set of models that are under consideration. The\n",
    "following command indicates that we want perform 10-fold cross-validation to compare SVMs with a linear\n",
    "kernel, using a range of values of the cost parameter:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# # Select the optimal C parameter by cross-validation\n",
    "# #tuned_parameters = [{'C': [0.001, 0.01, 0.1, 1, 5, 10, 100]}]\n",
    "# tuned_parameters = [{'C': [0.001, 0.01, 1, 10]}]\n",
    "\n",
    "# from time import time\n",
    "# import timeit #imports timeit module\n",
    "# start_time = timeit.default_timer() #defines start time so computational time can be calculated\n",
    "\n",
    "# clf = GridSearchCV(SVC(kernel='linear'), tuned_parameters, cv=10, scoring='accuracy')\n",
    "# clf.fit(X_train, y_train)\n",
    "\n",
    "# elapsed = timeit.default_timer() - start_time #gives total computation time\n",
    "# print(\"---Run time is %s seconds ---\" % elapsed) #prints computation time\n",
    "# print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(clf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = features_maindf\n",
    "# acc_score = [];\n",
    "# Truth = [];\n",
    "# Output = [];\n",
    "# max_depth_list = []\n",
    "# for idx, subject in tqdm(enumerate(good_list)):\n",
    "    \n",
    "#     train_index = ~df['Subject'].str.contains(subject)\n",
    "#     test_index = df['Subject'].str.contains(subject)\n",
    "#     X_train , X_test = df.loc[train_index,x_values], df.loc[test_index,x_values]\n",
    "#     y_train , y_test = df.loc[train_index,y_values].values.ravel(), df.loc[test_index,y_values]\n",
    "\n",
    "# #     temp_model = DecisionTreeClassifier(max_depth = 6)    \n",
    "# #     param_distributions = {'max_depth': randint(7,9)}\n",
    "\n",
    "# #     search = RandomizedSearchCV(estimator=temp_model, param_distributions=param_distributions, random_state=0)\n",
    "# #     search.fit(X_train, y_train)\n",
    "\n",
    "# #     max_depth_list.append(search.best_params_['max_depth'])\n",
    "    \n",
    "#     model = SVC(C=0.1, kernel='linear', random_state=2, probability = True) #criterion='entropy'\n",
    "#     model.fit(X_train,y_train)\n",
    "\n",
    "#     pred_values = model.predict(X_test)\n",
    "#     pred_prob = model.predict_proba(X_test)[:, 1]\n",
    "     \n",
    "#     acc = accuracy_score(y_test, pred_values)\n",
    "#     acc_score.append(acc)\n",
    "    \n",
    "#     Truth.extend(y_test.values.reshape(y_test.shape[0])); ## it is a list\n",
    "#     Output.extend(pred_values); ## it is a list \n",
    "# elapsed = timeit.default_timer() - start_time #gives total computation time\n",
    "# print(\"---Run time is %s seconds ---\" % elapsed) #prints computation time\n",
    "# print()\n",
    "\n",
    "# print('Accuracy of each fold: \\n {}'.format(acc_score))\n",
    "# print()\n",
    "# print('Avg accuracy : \\n{}'.format(np.mean(acc_score))); \n",
    "# print()\n",
    "# print('Std of accuracy : \\n{}'.format(np.std(acc_score)))\n",
    "\n",
    "# # crossvalidation = KFold(n_splits=5, random_state=2, shuffle=True)\n",
    "# # model = SVC(C=0.01, kernel='linear')\n",
    "# # scores = cross_val_score(model, X, y, cv=crossvalidation)\n",
    "# # print('Accuracy of each fold: \\n {}'.format(scores))\n",
    "# # print()\n",
    "# # print(\"Avg accuracy: {}\".format(scores.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "In order to fit an SVM using a **non-linear kernel**, we once again use the ${\\tt SVC()}$\n",
    "function. However, now we use a different value of the parameter kernel.\n",
    "To fit an SVM with a polynomial kernel we use ${\\tt kernel=\"poly\"}$, and\n",
    "to fit an SVM with a radial kernel we use ${\\tt kernel=\"rbf\"}$. In the former\n",
    "case we also use the ${\\tt degree}$ argument to specify a degree for the polynomial\n",
    "kernel, and in the latter case we use ${\\tt gamma}$ to specify a\n",
    "value of $\\gamma$ for the radial basis kernel.\n",
    "\n",
    "Let's generate some data with a non-linear class boundary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23it [39:44, 103.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Run time is 2384.363411099999 seconds ---\n",
      "\n",
      "Accuracy of each fold: \n",
      " [0.9084668192219679, 0.9521298174442191, 0.9955919395465995, 0.8591127098321343, 0.9973876698014629, 0.9310344827586207, 0.8925735637552545, 0.9959155888359428, 0.945281124497992, 0.9283351708930541, 0.9841197822141561, 0.9885139985642498, 0.7401360544217687, 0.9980891719745223, 0.42173189009159034, 0.973859105006646, 0.9897189856065799, 0.9924985574148875, 0.5967878275570583, 0.9593183067619572, 0.9931565440547476, 0.9874450721908349, 0.9906976744186047]\n",
      "\n",
      "Avg accuracy : \n",
      "0.9139957329071674\n",
      "\n",
      "Std of accuracy : \n",
      "0.14027089495672998\n",
      "[0.9084668192219679, 0.9521298174442191, 0.9955919395465995, 0.8591127098321343, 0.9973876698014629, 0.9310344827586207, 0.8925735637552545, 0.9959155888359428, 0.945281124497992, 0.9283351708930541, 0.9841197822141561, 0.9885139985642498, 0.7401360544217687, 0.9980891719745223, 0.42173189009159034, 0.973859105006646, 0.9897189856065799, 0.9924985574148875, 0.5967878275570583, 0.9593183067619572, 0.9931565440547476, 0.9874450721908349, 0.9906976744186047]\n",
      "\n",
      "[[22277  2466]\n",
      " [ 1893 18332]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91     24743\n",
      "           1       0.88      0.91      0.89     20225\n",
      "\n",
      "    accuracy                           0.90     44968\n",
      "   macro avg       0.90      0.90      0.90     44968\n",
      "weighted avg       0.90      0.90      0.90     44968\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df = features_maindf\n",
    "acc_score = [];\n",
    "Truth = [];\n",
    "Output = [];\n",
    "max_depth_list = []\n",
    "for idx, subject in tqdm(enumerate(good_list)):\n",
    "    \n",
    "    train_index = ~df['Subject'].str.contains(subject)\n",
    "    test_index = df['Subject'].str.contains(subject)\n",
    "    X_train , X_test = df.loc[train_index,x_values], df.loc[test_index,x_values]\n",
    "    y_train , y_test = df.loc[train_index,y_values].values.ravel(), df.loc[test_index,y_values]\n",
    "\n",
    "#     temp_model = DecisionTreeClassifier(max_depth = 6)    \n",
    "#     param_distributions = {'max_depth': randint(7,9)}\n",
    "\n",
    "#     search = RandomizedSearchCV(estimator=temp_model, param_distributions=param_distributions, random_state=0)\n",
    "#     search.fit(X_train, y_train)\n",
    "\n",
    "#     max_depth_list.append(search.best_params_['max_depth'])\n",
    "    \n",
    "    model = SVC(C=0.01, gamma=1, kernel='rbf', probability=True)\n",
    "    model.fit(X_train,y_train)\n",
    "\n",
    "    pred_values = model.predict(X_test)\n",
    "    pred_prob = model.predict_proba(X_test)[:, 1]\n",
    "     \n",
    "    acc = accuracy_score(y_test, pred_values)\n",
    "    acc_score.append(acc)\n",
    "    \n",
    "    Truth.extend(y_test.values.reshape(y_test.shape[0])); ## it is a list\n",
    "    Output.extend(pred_values); ## it is a list \n",
    "elapsed = timeit.default_timer() - start_time #gives total computation time\n",
    "print(\"---Run time is %s seconds ---\" % elapsed) #prints computation time\n",
    "print()\n",
    "\n",
    "print('Accuracy of each fold: \\n {}'.format(acc_score))\n",
    "print()\n",
    "print('Avg accuracy : \\n{}'.format(np.mean(acc_score))); \n",
    "print()\n",
    "print('Std of accuracy : \\n{}'.format(np.std(acc_score)))\n",
    "print(acc_score)\n",
    "\n",
    "print()\n",
    "print(confusion_matrix(Truth, Output))\n",
    "print(classification_report(Truth, Output))\n",
    "print()\n",
    "\n",
    "# crossvalidation = KFold(n_splits=5, random_state=2, shuffle=True)\n",
    "# model = SVC(C=0.01, gamma=1, kernel='rbf')\n",
    "# scores = cross_val_score(model, X, y, cv=crossvalidation)\n",
    "# print('Accuracy of each fold: \\n {}'.format(scores))\n",
    "# print()\n",
    "# print(\"Avg accuracy: {}\".format(scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 100, 'gamma': 1}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuned_parameters = [{'C': [0.01, 0.1, 1, 10, 100],\n",
    "                     'gamma': [0.01, 0.1, 0.5, 1, 5, 10]}]\n",
    "\n",
    "clf = GridSearchCV(SVC(kernel='rbf'), tuned_parameters, cv=10, scoring='accuracy')\n",
    "clf.fit(X_train, y_train)\n",
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23it [1:31:10, 237.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Run time is 16213.0924696 seconds ---\n",
      "\n",
      "Accuracy of each fold: \n",
      " [0.9010297482837528, 0.9399594320486815, 0.9949622166246851, 0.8231414868105515, 0.9869383490073145, 0.9249492900608519, 0.9028491359177955, 0.988427501701838, 0.9257028112449799, 0.8291069459757442, 0.9827586206896551, 0.9691313711414213, 0.853514739229025, 1.0, 0.38467943380516234, 0.8998670801949491, 0.9725839616175462, 0.9913444893248702, 0.6863905325443787, 0.9334799340296867, 0.9726261762189906, 0.9196484620213434, 0.9543340380549683]\n",
      "\n",
      "Avg accuracy : \n",
      "0.9016272068064431\n",
      "\n",
      "Std of accuracy : \n",
      "0.13129389576403158\n",
      "[0.9010297482837528, 0.9399594320486815, 0.9949622166246851, 0.8231414868105515, 0.9869383490073145, 0.9249492900608519, 0.9028491359177955, 0.988427501701838, 0.9257028112449799, 0.8291069459757442, 0.9827586206896551, 0.9691313711414213, 0.853514739229025, 1.0, 0.38467943380516234, 0.8998670801949491, 0.9725839616175462, 0.9913444893248702, 0.6863905325443787, 0.9334799340296867, 0.9726261762189906, 0.9196484620213434, 0.9543340380549683]\n",
      "\n",
      "[[22321  2422]\n",
      " [ 2421 17804]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.90      0.90     24743\n",
      "           1       0.88      0.88      0.88     20225\n",
      "\n",
      "    accuracy                           0.89     44968\n",
      "   macro avg       0.89      0.89      0.89     44968\n",
      "weighted avg       0.89      0.89      0.89     44968\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df = features_maindf\n",
    "acc_score = [];\n",
    "Truth = [];\n",
    "Output = [];\n",
    "max_depth_list = []\n",
    "for idx, subject in tqdm(enumerate(good_list)):\n",
    "    \n",
    "    train_index = ~df['Subject'].str.contains(subject)\n",
    "    test_index = df['Subject'].str.contains(subject)\n",
    "    X_train , X_test = df.loc[train_index,x_values], df.loc[test_index,x_values]\n",
    "    y_train , y_test = df.loc[train_index,y_values].values.ravel(), df.loc[test_index,y_values]\n",
    "\n",
    "#     temp_model = DecisionTreeClassifier(max_depth = 6)    \n",
    "#     param_distributions = {'max_depth': randint(7,9)}\n",
    "\n",
    "#     search = RandomizedSearchCV(estimator=temp_model, param_distributions=param_distributions, random_state=0)\n",
    "#     search.fit(X_train, y_train)\n",
    "\n",
    "#     max_depth_list.append(search.best_params_['max_depth'])\n",
    "    \n",
    "    model = SVC(C=0.01, gamma=0.01, kernel='rbf', probability=True)\n",
    "    model.fit(X_train,y_train)\n",
    "\n",
    "    pred_values = model.predict(X_test)\n",
    "    pred_prob = model.predict_proba(X_test)[:, 1]\n",
    "     \n",
    "    acc = accuracy_score(y_test, pred_values)\n",
    "    acc_score.append(acc)\n",
    "    \n",
    "    Truth.extend(y_test.values.reshape(y_test.shape[0])); ## it is a list\n",
    "    Output.extend(pred_values); ## it is a list \n",
    "elapsed = timeit.default_timer() - start_time #gives total computation time\n",
    "print(\"---Run time is %s seconds ---\" % elapsed) #prints computation time\n",
    "print()\n",
    "\n",
    "print('Accuracy of each fold: \\n {}'.format(acc_score))\n",
    "print()\n",
    "print('Avg accuracy : \\n{}'.format(np.mean(acc_score))); \n",
    "print()\n",
    "print('Std of accuracy : \\n{}'.format(np.std(acc_score)))\n",
    "print(acc_score)\n",
    "print()\n",
    "print(confusion_matrix(Truth, Output))\n",
    "print(classification_report(Truth, Output))\n",
    "print()\n",
    "\n",
    "# crossvalidation = KFold(n_splits=5, random_state=2, shuffle=True)\n",
    "# model = SVC(C=0.01, gamma=0.01, kernel='rbf')\n",
    "# scores = cross_val_score(model, X, y, cv=crossvalidation)\n",
    "# print('Accuracy of each fold: \\n {}'.format(scores))\n",
    "# print()\n",
    "# print(\"Avg accuracy: {}\".format(scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}