climateHack-Lisa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from geopy.geocoders import Nominatim\n",
    "import datetime\n",
    "\n",
    "import generate_vectors as gv\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/HFER_e_lat_lon.csv', encoding = \"utf-8\",index_col = 'Election Date')\n",
    "df = df[pd.to_datetime(df.index) > pd.to_datetime('1994-01-01')]\n",
    "df= df[df['Province']=='Quebec']\n",
    "#df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# give number for each major party or other\n",
    "def partyLabel (row):\n",
    "    if row['Party']=='Liberal':\n",
    "        return 0\n",
    "    elif (row['Party']=='Conservative' or row['Party']=='Progressive Conservative'or\n",
    "            row['Party']== 'Reform Party of Canada' or row['Party']== 'Reform'or\n",
    "         row['Party']== 'Canadian Alliance'or row['Party']=='Conservative Party of Canada'):\n",
    "        return 1\n",
    "    elif row['Party']=='New Democratic Party' or row['Party']== 'Co-operative Commonwealth Federation':\n",
    "        return 2\n",
    "    elif row['Party']=='Green Party of Canada':\n",
    "        return 3\n",
    "    elif row['Party']=='Bloc Québécois':\n",
    "        return 4\n",
    "    else:\n",
    "        return 5\n",
    "df['partyLabel']= df.apply (lambda row: partyLabel(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataframe with one row per riding per date\n",
    "parties=['Liberal','Conservative','NDP','Green','Bloc','Other']\n",
    "dfUnique=pd.DataFrame(columns=['Election Date', 'Province', 'Riding','lat','lon',\n",
    "                                'Results','Liberal','Conservative','NDP','Green','Bloc','Other'])\n",
    "prevRiding=''\n",
    "i=0\n",
    "for date, row in df.iterrows():\n",
    "    #print(date)\n",
    "    percentVotes=row['Votes (%)']/100\n",
    "    if prevRiding != row['Riding']:\n",
    "        if i>2 and 0.98> sum(dfUnique['Results'][i]) >1.02:\n",
    "            print(\"!!! Sum of results is not 1!!!\", sum(dfUnique['Results'][i]),dfUnique['Election Date'][i],dfUnique['Riding'][i] )    \n",
    "        i+=1\n",
    "        Results=np.zeros(6)\n",
    "        Results[row['partyLabel']]+= percentVotes\n",
    "        dfUnique.loc[i]=[date,row['Province'],row['Riding'],row['lat'],row['lon'],Results,0,0,0,0,0,0]\n",
    "        dfUnique[parties[row['partyLabel']]][i]+=percentVotes\n",
    "    elif prevRiding == row['Riding']:\n",
    "        dfUnique[parties[row['partyLabel']]][i]+=percentVotes\n",
    "        dfUnique['Results'][i][row['partyLabel']]+=percentVotes\n",
    "    prevRiding=row['Riding']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Election Date</th>\n",
       "      <th>Province</th>\n",
       "      <th>Riding</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>Results</th>\n",
       "      <th>Liberal</th>\n",
       "      <th>Conservative</th>\n",
       "      <th>NDP</th>\n",
       "      <th>Green</th>\n",
       "      <th>Bloc</th>\n",
       "      <th>Other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1995-02-13</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>BROME--MISSISQUOI</td>\n",
       "      <td>45.189207</td>\n",
       "      <td>-72.562857</td>\n",
       "      <td>[0.5102, 0.0468, 0.009899999999999999, 0.0027,...</td>\n",
       "      <td>0.5102</td>\n",
       "      <td>0.0468</td>\n",
       "      <td>0.0099</td>\n",
       "      <td>0.0027</td>\n",
       "      <td>0.4216</td>\n",
       "      <td>0.0088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1995-02-13</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>SAINT-HENRI--WESTMOUNT</td>\n",
       "      <td>45.478460</td>\n",
       "      <td>-73.584618</td>\n",
       "      <td>[0.7591, 0.060599999999999994, 0.0177, 0.0128,...</td>\n",
       "      <td>0.7591</td>\n",
       "      <td>0.0606</td>\n",
       "      <td>0.0177</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.1412</td>\n",
       "      <td>0.0085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1996-03-25</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>LAC-SAINT-JEAN</td>\n",
       "      <td>48.583336</td>\n",
       "      <td>-72.004880</td>\n",
       "      <td>[0.21539999999999998, 0.014, 0.005, 0.0, 0.765...</td>\n",
       "      <td>0.2154</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0</td>\n",
       "      <td>0.7656</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1996-03-25</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>PAPINEAU--SAINT-MICHEL</td>\n",
       "      <td>45.852553</td>\n",
       "      <td>-75.141481</td>\n",
       "      <td>[0.5920000000000001, 0.034, 0.0347, 0.0, 0.339...</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.0347</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3393</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1996-03-25</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>SAINT-LAURENT--CARTIERVILLE</td>\n",
       "      <td>45.503270</td>\n",
       "      <td>-73.725448</td>\n",
       "      <td>[0.7927, 0.0424, 0.0079, 0.0, 0.14859999999999...</td>\n",
       "      <td>0.7927</td>\n",
       "      <td>0.0424</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1486</td>\n",
       "      <td>0.0085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>391</td>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>VAUDREUIL--SOULANGES</td>\n",
       "      <td>45.399423</td>\n",
       "      <td>-74.051462</td>\n",
       "      <td>[0.2133, 0.2369, 0.09630000000000001, 0.0401, ...</td>\n",
       "      <td>0.2133</td>\n",
       "      <td>0.2369</td>\n",
       "      <td>0.0963</td>\n",
       "      <td>0.0401</td>\n",
       "      <td>0.4134</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>392</td>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>VERCHÈRES--LES PATRIOTES</td>\n",
       "      <td>45.772785</td>\n",
       "      <td>-73.355442</td>\n",
       "      <td>[0.1634, 0.1426, 0.1545, 0.030899999999999997,...</td>\n",
       "      <td>0.1634</td>\n",
       "      <td>0.1426</td>\n",
       "      <td>0.1545</td>\n",
       "      <td>0.0309</td>\n",
       "      <td>0.5085</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>393</td>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>WESTMOUNT--VILLE-MARIE</td>\n",
       "      <td>45.485733</td>\n",
       "      <td>-73.596395</td>\n",
       "      <td>[0.4647, 0.15810000000000002, 0.2293, 0.0704, ...</td>\n",
       "      <td>0.4647</td>\n",
       "      <td>0.1581</td>\n",
       "      <td>0.2293</td>\n",
       "      <td>0.0704</td>\n",
       "      <td>0.0726</td>\n",
       "      <td>0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>394</td>\n",
       "      <td>2009-11-09</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>HOCHELAGA</td>\n",
       "      <td>45.540386</td>\n",
       "      <td>-73.541541</td>\n",
       "      <td>[0.1434, 0.10060000000000001, 0.196, 0.0326, 0...</td>\n",
       "      <td>0.1434</td>\n",
       "      <td>0.1006</td>\n",
       "      <td>0.196</td>\n",
       "      <td>0.0326</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.0157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>395</td>\n",
       "      <td>2009-11-09</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>MONTMAGNY--L’ISLET--KAMOURASKA--RIVIÈRE-DU-LOUP</td>\n",
       "      <td>46.982633</td>\n",
       "      <td>-70.555916</td>\n",
       "      <td>[0.1322, 0.4267, 0.0478, 0.0166, 0.37670000000...</td>\n",
       "      <td>0.1322</td>\n",
       "      <td>0.4267</td>\n",
       "      <td>0.0478</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.3767</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>395 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Election Date Province                                           Riding  \\\n",
       "1      1995-02-13   Quebec                                BROME--MISSISQUOI   \n",
       "2      1995-02-13   Quebec                           SAINT-HENRI--WESTMOUNT   \n",
       "3      1996-03-25   Quebec                                   LAC-SAINT-JEAN   \n",
       "4      1996-03-25   Quebec                           PAPINEAU--SAINT-MICHEL   \n",
       "5      1996-03-25   Quebec                      SAINT-LAURENT--CARTIERVILLE   \n",
       "..            ...      ...                                              ...   \n",
       "391    2008-10-14   Quebec                             VAUDREUIL--SOULANGES   \n",
       "392    2008-10-14   Quebec                         VERCHÈRES--LES PATRIOTES   \n",
       "393    2008-10-14   Quebec                           WESTMOUNT--VILLE-MARIE   \n",
       "394    2009-11-09   Quebec                                        HOCHELAGA   \n",
       "395    2009-11-09   Quebec  MONTMAGNY--L’ISLET--KAMOURASKA--RIVIÈRE-DU-LOUP   \n",
       "\n",
       "           lat        lon                                            Results  \\\n",
       "1    45.189207 -72.562857  [0.5102, 0.0468, 0.009899999999999999, 0.0027,...   \n",
       "2    45.478460 -73.584618  [0.7591, 0.060599999999999994, 0.0177, 0.0128,...   \n",
       "3    48.583336 -72.004880  [0.21539999999999998, 0.014, 0.005, 0.0, 0.765...   \n",
       "4    45.852553 -75.141481  [0.5920000000000001, 0.034, 0.0347, 0.0, 0.339...   \n",
       "5    45.503270 -73.725448  [0.7927, 0.0424, 0.0079, 0.0, 0.14859999999999...   \n",
       "..         ...        ...                                                ...   \n",
       "391  45.399423 -74.051462  [0.2133, 0.2369, 0.09630000000000001, 0.0401, ...   \n",
       "392  45.772785 -73.355442  [0.1634, 0.1426, 0.1545, 0.030899999999999997,...   \n",
       "393  45.485733 -73.596395  [0.4647, 0.15810000000000002, 0.2293, 0.0704, ...   \n",
       "394  45.540386 -73.541541  [0.1434, 0.10060000000000001, 0.196, 0.0326, 0...   \n",
       "395  46.982633 -70.555916  [0.1322, 0.4267, 0.0478, 0.0166, 0.37670000000...   \n",
       "\n",
       "    Liberal Conservative     NDP   Green    Bloc   Other  \n",
       "1    0.5102       0.0468  0.0099  0.0027  0.4216  0.0088  \n",
       "2    0.7591       0.0606  0.0177  0.0128  0.1412  0.0085  \n",
       "3    0.2154        0.014   0.005       0  0.7656       0  \n",
       "4     0.592        0.034  0.0347       0  0.3393       0  \n",
       "5    0.7927       0.0424  0.0079       0  0.1486  0.0085  \n",
       "..      ...          ...     ...     ...     ...     ...  \n",
       "391  0.2133       0.2369  0.0963  0.0401  0.4134       0  \n",
       "392  0.1634       0.1426  0.1545  0.0309  0.5085       0  \n",
       "393  0.4647       0.1581  0.2293  0.0704  0.0726   0.005  \n",
       "394  0.1434       0.1006   0.196  0.0326  0.5116  0.0157  \n",
       "395  0.1322       0.4267  0.0478  0.0166  0.3767       0  \n",
       "\n",
       "[395 rows x 12 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfUnique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nb of election results: 395\n",
      "shape of x (395, 68)\n",
      "shape of y (395, 6)\n"
     ]
    }
   ],
   "source": [
    "# length of election results\n",
    "print('Nb of election results:', len(dfUnique))\n",
    "\n",
    "# getting output for training neural network\n",
    "x, y = gv.get_input_output()\n",
    "\n",
    "# a, b shape\n",
    "print('shape of x', x.shape)\n",
    "print('shape of y', y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training a neural network!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import neural_network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01813272321102029\n"
     ]
    }
   ],
   "source": [
    "# normalizing?\n",
    "for i in range(x.shape[1]):\n",
    "    x[:,i] = (x[:,i] - np.min(x[:,i])) / (np.max(x[:,i]) - np.min(x[:,i]))\n",
    "\n",
    "# train mlp\n",
    "alpha = 0.1\n",
    "mlp   = neural_network.MLPRegressor(activation = 'tanh', hidden_layer_sizes=(50,25), max_iter = 3000, solver='adam', random_state=99, learning_rate = 'adaptive',alpha = alpha)\n",
    "mlp.fit(x,y)\n",
    "score = mlp.score(x,y);print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "fashion_mnist = keras.datasets.fashion_mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'tensorflow._api.v1.keras.datasets.fashion_mnist' has no attribute 'data'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-72e10d9b439d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfashion_mnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: module 'tensorflow._api.v1.keras.datasets.fashion_mnist' has no attribute 'data'"
     ]
    }
   ],
   "source": [
    "fashion_mnist.data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (climate_hack)",
   "language": "python",
   "name": "climate_hack"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}