regression_model_testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa11c6bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.svm import LinearSVR\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import ElasticNet\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "from sklearn.linear_model import MultiTaskLasso\n",
    "from sklearn.linear_model import MultiTaskElasticNet\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SequentialFeatureSelector\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67bce870",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"parser_output/extracted_features_split_4_18_upsample.csv\")\n",
    "data = data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28d520c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5728312",
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOOK INTO https://scikit-learn.org/stable/modules/classes.html#module-sklearn.feature_selection\n",
    "#multi output regressor? - https://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputRegressor.html#sklearn.multioutput.MultiOutputRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99fc5e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = data[['danceability', 'energy', 'loudness', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo']]\n",
    "x = data[data.columns[1:len(data.columns)-12]]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.7)\n",
    "X_train = X_train.iloc[: , 1:]\n",
    "X_test = X_test.iloc[: , 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db596768",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b7a89bb",
   "metadata": {},
   "source": [
    "## REGRESSION MODELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4391a94",
   "metadata": {},
   "outputs": [],
   "source": [
    "#lINEAR REGRESSION - \n",
    "reg_model = LinearRegression().fit(X_train, y_train)\n",
    "ypred_train = reg_model.predict(X_train)\n",
    "ypred_test = reg_model.predict(X_test)\n",
    "print(\"LINEAR REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b29eeb3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "223440e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#K NEIGHBORS REGRESSOR - https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsRegressor.html#sklearn.neighbors.KNeighborsRegressor\n",
    "knn_model = KNeighborsRegressor().fit(X_train, y_train)\n",
    "ypred_train = knn_model.predict(X_train)\n",
    "ypred_test = knn_model.predict(X_test)\n",
    "print(\"KNN REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de1367d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#LINEAR SVR - https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVR.html#sklearn.svm.LinearSVR\n",
    "svr_model = MultiOutputRegressor(LinearSVR()).fit(X_train, y_train)\n",
    "ypred_train = svr_model.predict(X_train)\n",
    "ypred_test = svr_model.predict(X_test)\n",
    "print(\"SVR REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ec8a94f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tree - https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeRegressor.html#sklearn.tree.DecisionTreeRegressor\n",
    "tree_model = DecisionTreeRegressor().fit(X_train, y_train)\n",
    "ypred_train = tree_model.predict(X_train)\n",
    "ypred_test = tree_model.predict(X_test)\n",
    "print(\"TREE REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e776f020",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest - https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestRegressor.html#sklearn.ensemble.RandomForestRegressor\n",
    "rf_model = RandomForestRegressor().fit(X_train, y_train)\n",
    "ypred_train = rf_model.predict(X_train)\n",
    "ypred_test = rf_model.predict(X_test)\n",
    "print(\"RANDOM FOREST REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "891cb79f",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "454df710",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "selected_RF = RFECV(rf_model, step=25, cv=5, n_jobs = -1)\n",
    "selected_RF = selected_RF.fit(X_train, y_train)\n",
    "ypred_train = selected_RF.predict(X_train)\n",
    "ypred_test = selected_RF.predict(X_test)\n",
    "print(\"RANDOM FOREST REGRESSION - SELECTED FEATURES\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5009f004",
   "metadata": {},
   "outputs": [],
   "source": [
    "names = pd.DataFrame(selected_RF.feature_names_in_)\n",
    "best_features = [names.iloc[selected_RF.ranking_==1]]\n",
    "best_features\n",
    "#pd.DataFrame(best_features).to_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29f25037",
   "metadata": {},
   "outputs": [],
   "source": [
    "featureSelectedX_train = X_train[names.iloc[selected_RF.ranking_==1][0].to_numpy()]\n",
    "featureSelectedX_test = X_test[names.iloc[selected_RF.ranking_==1][0].to_numpy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75c0a845",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cac509ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "names.iloc[selected_RF.ranking_==1][0].to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "078db5d0",
   "metadata": {},
   "source": [
    "# Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d267f456",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "import numpy as np\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "\n",
    "# Use the random grid to search for best hyperparameters\n",
    "# First create the base model to tune\n",
    "rf = RandomForestRegressor()\n",
    "# Random search of parameters, using 3 fold cross validation, \n",
    "# search across 100 different combinations, and use all available cores\n",
    "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "# Fit the random search model\n",
    "rf_random.fit(featureSelectedX_train, y_train)\n",
    "\n",
    "ypred_train = rf_random.predict(featureSelectedX_train)\n",
    "ypred_test = rf_random.predict(featureSelectedX_test)\n",
    "print(\"RANDOM FOREST REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "179e98c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08212881",
   "metadata": {},
   "source": [
    "# Model Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5158fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'model_4_20.sav'\n",
    "pickle.dump(rf_random, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b27d8d08",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'model_4_20.sav'\n",
    "model = pickle.load(open(filename, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "484a62e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b72cba7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred_train = model.predict(X_train)\n",
    "ypred_test = model.predict(X_test)\n",
    "print(\"RANDOM FOREST REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83b565c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_final = RandomForestRegressor(n_estimators=400, min_samples_split=2,min_samples_leaf=1, max_features='sqrt', max_depth=None, bootstrap=False).fit(featureSelectedX_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d87da26",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred_train = rf_final.predict(featureSelectedX_train)\n",
    "ypred_test = rf_final.predict(featureSelectedX_test)\n",
    "print(\"RANDOM FOREST REGRESSION\")\n",
    "print(\"Train Score: \")\n",
    "print(r2_score(y_train, ypred_train))\n",
    "print(\"Test Score: \")\n",
    "print(r2_score(y_test, ypred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d25a131",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'model_final.sav'\n",
    "pickle.dump(rf_final, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee4fdfa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(names.iloc[selected_RF.ranking_==1][0].to_numpy()).to_csv(\"Selected_Feature_List.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}