GamePredictionWeather.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import statsapi\n",
    "import requests as re\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import more_itertools as mit\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from pyowm.utils import timestamps\n",
    "from pyowm.owm import OWM\n",
    "from selenium import webdriver\n",
    "import sqlalchemy\n",
    "driver = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Connect to a local postgres database for weather data\n",
    "engine = sqlalchemy.create_engine('postgresql://postgres:password@localhost:5432/Weather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\leowe\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "# Get game info for today's game\n",
    "f = {'game_id':[],'away_id':[],'home_id':[],'apitcher':[],'hpitcher':[],'time':[]}\n",
    "for game in statsapi.schedule(date=str(datetime.date.today())):\n",
    "    f['game_id'].append(game['game_id'])\n",
    "    f['away_id'].append(game['away_id'])\n",
    "    f['home_id'].append(game['home_id'])\n",
    "    f['apitcher'].append(game['away_probable_pitcher'])\n",
    "    f['hpitcher'].append(game['home_probable_pitcher'])\n",
    "    f['time'].append(game['game_datetime'][11:16])\n",
    "f = pd.DataFrame(f)\n",
    "f = f[f['apitcher'] != ''][f['hpitcher'] != ''].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 14/14 [01:22<00:00,  5.89s/it]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 14/14 [01:14<00:00,  5.31s/it]\n"
     ]
    }
   ],
   "source": [
    "# Web-scrape starting pitcher data\n",
    "pa = defaultdict(list)\n",
    "ph = defaultdict(list)\n",
    "\n",
    "# away pitchers\n",
    "for p in f['apitcher']:\n",
    "    ps = statsapi.lookup_player(p)\n",
    "    if len(ps) == 1:\n",
    "        pid = ps[0]['id']\n",
    "    else:\n",
    "        for player in ps:\n",
    "            if player['primaryPosition']['abbreviation'] == 'P':\n",
    "                pid = player['id']\n",
    "    for key, value in statsapi.player_stat_data(pid, group='pitching',type='career')['stats'][0]['stats'].items():\n",
    "        pa[key+'_away'].append(value)\n",
    "\n",
    "# home pitchers\n",
    "for p in f['hpitcher']:\n",
    "    ps = statsapi.lookup_player(p)\n",
    "    if len(ps) == 1:\n",
    "        pid = ps[0]['id']\n",
    "    else:\n",
    "        for player in ps:\n",
    "            if player['primaryPosition']['abbreviation'] == 'P':\n",
    "                pid = player['id']\n",
    "    for key, value in statsapi.player_stat_data(pid, group='pitching',type='career')['stats'][0]['stats'].items():\n",
    "        ph[key+'_home'].append(value)\n",
    "\n",
    "# delete empty keys of the dictionary if there are any\n",
    "todelete = []\n",
    "for k in ph.keys():\n",
    "    if len(ph[k]) < f.shape[0]:\n",
    "        todelete.append(k)\n",
    "        \n",
    "for k in todelete:\n",
    "    del ph[k]\n",
    "    \n",
    "todelete = [] \n",
    "for k in pa.keys():\n",
    "    if len(pa[k]) < f.shape[0]:\n",
    "        todelete.append(k)\n",
    "\n",
    "for k in todelete:\n",
    "    del pa[k]\n",
    "\n",
    "\n",
    "# convert and append to a dataframe\n",
    "ph = pd.DataFrame(ph)\n",
    "pa = pd.DataFrame(pa)\n",
    "f = pd.concat([f,pa,ph],axis=1)\n",
    "\n",
    "# read in the team abbreviations file\n",
    "abbre = pd.read_csv('Data/teamabbre.csv')\n",
    "\n",
    "# Merge\n",
    "f = f.merge(abbre,left_on=['away_id'],right_on=['teamid']).drop(columns=['teamid']).merge(abbre,left_on=['home_id'],right_on=['teamid'],suffixes=['_away','_home']).drop(columns=['teamid'])\n",
    "\n",
    "# web-scrape away team batting splits\n",
    "a = defaultdict(list)\n",
    "for team in tqdm(f['teamabb_away']):\n",
    "    if team == 'WAS':\n",
    "        teamm = 'WSN'\n",
    "    elif team == 'CWS':\n",
    "        teamm = 'CHW'\n",
    "    elif team == 'ANA':\n",
    "        teamm = 'LAA'\n",
    "    elif team == 'SF':\n",
    "        teamm = 'SFG'\n",
    "    elif team == 'KC':\n",
    "        teamm = 'KCR'\n",
    "    elif team == 'LA':\n",
    "        teamm = 'LAD'\n",
    "    elif team == 'TB':\n",
    "        teamm = 'TBR'\n",
    "    elif team == 'SD':\n",
    "        teamm = 'SDP'\n",
    "    else:\n",
    "        teamm = team\n",
    " \n",
    "    driver.get(\"https://www.baseball-reference.com/teams/split.cgi?t=b&team=\"+teamm+\"&year=2021\")\n",
    "    t = driver.find_element_by_id(\"hmvis\").text\n",
    "\n",
    "    away = t.split('\\n')[2].split(' ')\n",
    "    a['BA_away'].append(away[15])\n",
    "    a['OBP_away'].append(away[16])\n",
    "    a['SLG_away'].append(away[17])\n",
    "    a['OPS_away'].append(away[18])\n",
    "    a['BAbip_away'].append(away[26])\n",
    "\n",
    "    time.sleep(2)\n",
    "a = pd.DataFrame(a)\n",
    "\n",
    "# webscrape for home team batting splits\n",
    "h = defaultdict(list)\n",
    "for team in tqdm(f['teamabb_home']):\n",
    "    if team == 'WAS':\n",
    "        teamm = 'WSN'\n",
    "    elif team == 'CWS':\n",
    "        teamm = 'CHW'\n",
    "    elif team == 'ANA':\n",
    "        teamm = 'LAA'\n",
    "    elif team == 'SF':\n",
    "        teamm = 'SFG'\n",
    "    elif team == 'KC':\n",
    "        teamm = 'KCR'\n",
    "    elif team == 'LA':\n",
    "        teamm = 'LAD'\n",
    "    elif team == 'TB':\n",
    "        teamm = 'TBR'\n",
    "    elif team == 'SD':\n",
    "        teamm = 'SDP'\n",
    "    else:\n",
    "        teamm = team\n",
    "\n",
    "    driver.get(\"https://www.baseball-reference.com/teams/split.cgi?t=b&team=\"+teamm+\"&year=2021\")\n",
    "    t = driver.find_element_by_id(\"hmvis\").text\n",
    "\n",
    "    home = t.split('\\n')[1].split(' ')\n",
    "    h['BA_home'].append(home[15])\n",
    "    h['OBP_home'].append(home[16])\n",
    "    h['SLG_home'].append(home[17])\n",
    "    h['OPS_home'].append(home[18])\n",
    "    h['BAbip_home'].append(home[26])\n",
    "\n",
    "    time.sleep(2)\n",
    "h = pd.DataFrame(h)\n",
    "\n",
    "# combine the columns\n",
    "f = pd.concat([f,a,h],axis=1)\n",
    "\n",
    "# drop columns we dont use for training\n",
    "pre = f.drop(columns=['away_id','home_id','apitcher','hpitcher','teamabb_away'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# geolocation of each home venue\n",
    "venues = [{\"team\":\"ANA\",\"address\":\"2000 Gene Autry Way, Anaheim, CA. 92806\",\"lat\":33.799572,\"lng\":-117.889031},\n",
    "      {\"team\":\"ARI\",\"address\":\"P.O. Box 2095, Phoenix, AZ. 85001\",\"lat\":33.452922,\"lng\":-112.038669},\n",
    "      {\"team\":\"ATL\",\"address\":\"P.O. Box 4064, Atlanta, GA. 30302\",\"lat\":33.74691,\"lng\":-84.391239},\n",
    "      {\"team\":\"BAL\",\"address\":\"333 W. Camden Street, Baltimore, MD. 21201\",\"lat\":39.285243,\"lng\":-76.620103},\n",
    "      {\"team\":\"BOS\",\"address\":\"4 Yawkey Way, Boston, MA 02215\",\"lat\":42.346613,\"lng\":-71.098817},\n",
    "      {\"team\":\"CHC\",\"address\":\"1060 Addison Street, Chicago, IL 60616\",\"lat\":41.947201,\"lng\":-87.656413},\n",
    "      {\"team\":\"CWS\",\"address\":\"333 W. 35th Street, Chicago, IL 60616\",\"lat\":41.830883,\"lng\":-87.635083},\n",
    "      {\"team\":\"CIN\",\"address\":\"100 Cinergy Field, Cincinnati, OH 45202\",\"lat\":39.107183,\"lng\":-84.507713},\n",
    "      {\"team\":\"CLE\",\"address\":\"2401 Ontario Street, Cleveland, OH 44115\",\"lat\":41.495149,\"lng\":-81.68709},\n",
    "      {\"team\":\"COL\",\"address\":\"Coors Field, 2001 Blake Street, Denver, CO 80205-2000\",\"lat\":39.75698,\"lng\":-104.965329},\n",
    "      {\"team\":\"DET\",\"address\":\"Comerica Park, 2100 Woodward Ave., Detroit, MI 48201\",\"lat\":42.346354,\"lng\":-83.059619},\n",
    "      {\"team\":\"MIA\",\"address\":\"2269 NW 199th Street, Miami, FL 33056\",\"lat\":25.954428,\"lng\":-80.238164},\n",
    "      {\"team\":\"HOU\",\"address\":\"P.O. Box 288, Houston, TX 77001-0288\",\"lat\":29.76045,\"lng\":-95.369784},\n",
    "      {\"team\":\"KC\",\"address\":\"P.O. Boz 419969, Kansas City, MO 64141\",\"lat\":39.10222,\"lng\":-94.583559},\n",
    "      {\"team\":\"LA\",\"address\":\"1000 Elysian Park Ave., Los Angeles, CA 90012\",\"lat\":34.072437,\"lng\":-118.246879},\n",
    "      {\"team\":\"MIL\",\"address\":\"P.O. Box 3099, Milwaukee, WI 53201-3099\",\"lat\":43.04205,\"lng\":-87.905599},\n",
    "      {\"team\":\"MIN\",\"address\":\"501 Chicago Ave. S., Minneapolis, MN 55415\",\"lat\":44.974346,\"lng\":-93.259616},\n",
    "      {\"team\":\"WAS\",\"address\":\"1500 South Capitol Street SE, Washington, DC\",\"lat\":38.87,\"lng\":-77.01},\n",
    "      {\"team\":\"NYM\",\"address\":\"Roosevelt Ave & 126th Street, New York, NY 11368\",\"lat\":40.75535,\"lng\":-73.843219},\n",
    "      {\"team\":\"NYY\",\"address\":\"Yankee Stadium, E. 161 Street & River Ave., New York, NY 10451\",\"lat\":40.819782,\"lng\":-73.929939},\n",
    "      {\"team\":\"OAK\",\"address\":\"Oakland Coliseum, 700 Coliseum Way, Oakland, Ca 94621-1918\",\"lat\":37.74923,\"lng\":-122.196487},\n",
    "      {\"team\":\"PHI\",\"address\":\"P.O. Box 7575, Philadelphia, PA 19101\",\"lat\":39.952313,\"lng\":-75.162392},\n",
    "      {\"team\":\"PIT\",\"address\":\"600 Stadium Circle, Pittsburgh, PA 15212\",\"lat\":40.461503,\"lng\":-80.008924},\n",
    "      {\"team\":\"STL\",\"address\":\"250 Stadium Plaza, St. Louis, MO 63102\",\"lat\":38.629683,\"lng\":-90.188247},\n",
    "      {\"team\":\"SD\",\"address\":\"P.O. Box 2000, San Diego, CA 92112-2000\",\"lat\":32.752148,\"lng\":-117.143635},\n",
    "      {\"team\":\"SF\",\"address\":\"Pacific Bell Park, 24 Willie Mays Plaza, San Francisco, CA 94107\",\"lat\":37.77987,\"lng\":-122.389754},\n",
    "      {\"team\":\"SEA\",\"address\":\"P.O. Box 41000, 411 First Ave. S., Seattle, WA 98104\",\"lat\":47.60174,\"lng\":-122.330829},\n",
    "      {\"team\":\"TB\",\"address\":\"1 Tropicana Drive, St. Petersburg, FL 33705\",\"lat\":27.768487,\"lng\":-82.648191},\n",
    "      {\"team\":\"TEX\",\"address\":\"1000 Ballpark Way, Arlington, TX 76011\",\"lat\":32.750156,\"lng\":-97.081117},\n",
    "      {\"team\":\"TOR\",\"address\":\"1 James D. Griffin Plaza, Buffalo, NY \",\"lat\":42.8804,\"lng\":-78.8738}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OpenWeatherMap API\n",
    "owm = OWM('xxxxx')\n",
    "mgr = owm.weather_manager()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the weather forcasts based on game location\n",
    "wdf = {'game_id':[],'pressure':[],'temperature':[],'temperature_feelslike':[],'humidity':[],'rainprob':[]}\n",
    "for i in range (pre.shape[0]):\n",
    "    for stadium in venues:\n",
    "        if stadium['team'] == pre.iloc[i]['teamabb_home']:\n",
    "            forcasts = mgr.one_call(stadium['lat'],stadium['lng']).forecast_hourly\n",
    "            for w in forcasts:\n",
    "                if w.reference_time('iso')[11:13] == pre.iloc[i]['time'][:2]:\n",
    "                    wdf['game_id'].append(pre.iloc[i]['game_id'])\n",
    "                    wdf['pressure'].append(w.pressure['press'])\n",
    "                    wdf['temperature'].append(w.temperature('fahrenheit')['temp'])\n",
    "                    wdf['temperature_feelslike'].append(w.temperature('fahrenheit')['feels_like'])\n",
    "                    wdf['humidity'].append(w.humidity)\n",
    "                    wdf['rainprob'].append(w.precipitation_probability)\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stores the weather data in a local database for future use\n",
    "pd.DataFrame(wdf).to_sql('mlbweather',con=engine,if_exists='append',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take in temperature\n",
    "pre['temp'] = pd.DataFrame(wdf)['temperature']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reformat the time\n",
    "pre['time'] = pre['time'].apply(lambda x: float(x[:2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop columns we dont use for training\n",
    "pre = pre.drop(columns=['game_id','teamabb_home'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace null averages with 0\n",
    "pre = pre.replace('.---',0).replace('-.--',0).astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select predictors for training\n",
    "pca = ['avg_away','obp_away','slg_away','ops_away','stolenBasePercentage_away','era_away','whip_away','strikePercentage_away','groundOutsToAirouts_away','pitchesPerInning_away','strikeoutWalkRatio_away','strikeoutsPer9Inn_away','walksPer9Inn_away','hitsPer9Inn_away','runsScoredPer9_away','homeRunsPer9_away']\n",
    "pch = ['avg_home','obp_home','slg_home','ops_home','stolenBasePercentage_home','era_home','whip_home','strikePercentage_home','groundOutsToAirouts_home','pitchesPerInning_home','strikeoutWalkRatio_home','strikeoutsPer9Inn_home','walksPer9Inn_home','hitsPer9Inn_home','runsScoredPer9_home','homeRunsPer9_home']\n",
    "bca = ['BA_away','OBP_away','SLG_away','OPS_away','BAbip_away']\n",
    "bch = ['BA_home','OBP_home','SLG_home','OPS_home','BAbip_home']\n",
    "temp = ['temp','time']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model and make predictions on over/under\n",
    "pre = pre[pca+pch+bca+bch+temp]\n",
    "model = pickle.load(open('total.sav', 'rb'))\n",
    "f['predictions'] = np.exp(model.predict(pre))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format the predictions in a string to email to myself\n",
    "s = ''\n",
    "for i in range (f.shape[0]):\n",
    "    s += f.iloc[i]['apitcher'] + '\\t'+ str(\"{:.2f}\".format(f.iloc[i]['predictions'])) + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# send the email\n",
    "import smtplib, ssl\n",
    "\n",
    "port = 465  # For SSL\n",
    "smtp_server = \"smtp.gmail.com\"\n",
    "sender_email = \"leowei08@gmail.com\"\n",
    "receiver_email2 = \"leowei08@gmail.com\"\n",
    "password = 'password'\n",
    "message = \"\"\"\\\n",
    "Subject: Predictions Today {today}\n",
    "\n",
    "\n",
    "{content}.\"\"\"\n",
    "\n",
    "context = ssl.create_default_context()\n",
    "with smtplib.SMTP_SSL(smtp_server, port, context=context) as server:\n",
    "    server.login(sender_email, password)\n",
    "    server.sendmail(sender_email, receiver_email2, message.format(today=str(datetime.date.today()).replace('-',''), content=s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model and make predictions for score difference\n",
    "model = pickle.load(open('diff.sav', 'rb'))\n",
    "f['predictions'] = model.predict(pre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format the predictions in a string to email to myself\n",
    "s = ''\n",
    "for i in range (f.shape[0]):\n",
    "    s += f.iloc[i]['apitcher'] + '\\t'+ str(\"{:.2f}\".format(f.iloc[i]['predictions'])) + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# send the email\n",
    "import smtplib, ssl\n",
    "\n",
    "port = 465  # For SSL\n",
    "smtp_server = \"smtp.gmail.com\"\n",
    "sender_email = \"leowei08@gmail.com\"\n",
    "receiver_email2 = \"leowei08@gmail.com\"\n",
    "password = 'password'\n",
    "message = \"\"\"\\\n",
    "Subject: Predictions Today {today}\n",
    "\n",
    "\n",
    "{content}.\"\"\"\n",
    "\n",
    "context = ssl.create_default_context()\n",
    "with smtplib.SMTP_SSL(smtp_server, port, context=context) as server:\n",
    "    server.login(sender_email, password)\n",
    "    server.sendmail(sender_email, receiver_email2, message.format(today=str(datetime.date.today()).replace('-',''), content=s))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}