Reinforcement Learning- Introduction to Monte Carlo Learning using the OpenAI Gym Toolkit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Monte Carlo Implementation in Python "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frozen Lake Environment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import operator\n",
    "from IPython.display import clear_output\n",
    "from time import sleep\n",
    "import random\n",
    "import itertools\n",
    "import tqdm\n",
    "\n",
    "tqdm.monitor_interval = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_random_policy(env):\n",
    "     policy = {}\n",
    "     for key in range(0, env.observation_space.n):\n",
    "          current_end = 0\n",
    "          p = {}\n",
    "          for action in range(0, env.action_space.n):\n",
    "               p[action] = 1 / env.action_space.n\n",
    "          policy[key] = p\n",
    "     return policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_state_action_dictionary(env, policy):\n",
    "    Q = {}\n",
    "    for key in policy.keys():\n",
    "         Q[key] = {a: 0.0 for a in range(0, env.action_space.n)}\n",
    "    return Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_game(env, policy, display=True):\n",
    "     env.reset()\n",
    "     episode = []\n",
    "     finished = False\n",
    "\n",
    "     while not finished:\n",
    "          s = env.env.s\n",
    "          if display:\n",
    "               clear_output(True)\n",
    "               env.render()\n",
    "               sleep(1)\n",
    "\n",
    "          timestep = []\n",
    "          timestep.append(s)\n",
    "           n = random.uniform(0, sum(policy[s].values()))\n",
    "           top_range = 0\n",
    "           for prob in policy[s].items():\n",
    "                 top_range += prob[1]\n",
    "                 if n < top_range:\n",
    "                       action = prob[0]\n",
    "                       break \n",
    "           state, reward, finished, info = env.step(action)\n",
    "           timestep.append(action)\n",
    "           timestep.append(reward)\n",
    "\n",
    "           episode.append(timestep)\n",
    "\n",
    "     if display:\n",
    "          clear_output(True)\n",
    "          env.render()\n",
    "          sleep(1)\n",
    "     return episode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_policy(policy, env):\n",
    "      wins = 0\n",
    "      r = 100\n",
    "      for i in range(r):\n",
    "            w = run_game(env, policy, display=False)[-1][-1]\n",
    "            if w == 1:\n",
    "                  wins += 1\n",
    "      return wins / r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_e_soft(env, episodes=100, policy=None, epsilon=0.01):\n",
    "    if not policy:\n",
    "        policy = create_random_policy(env)  # Create an empty dictionary to store state action values    \n",
    "    Q = create_state_action_dictionary(env, policy) # Empty dictionary for storing rewards for each state-action pair\n",
    "    returns = {} # 3.\n",
    "    \n",
    "    for _ in range(episodes): # Looping through episodes\n",
    "        G = 0 # Store cumulative reward in G (initialized at 0)\n",
    "        episode = run_game(env=env, policy=policy, display=False) # Store state, action and value respectively \n",
    "        \n",
    "        # for loop through reversed indices of episode array. \n",
    "        # The logic behind it being reversed is that the eventual reward would be at the end. \n",
    "        # So we have to go back from the last timestep to the first one propagating result from the future.\n",
    "        \n",
    "        for i in reversed(range(0, len(episode))):   \n",
    "            s_t, a_t, r_t = episode[i] \n",
    "            state_action = (s_t, a_t)\n",
    "            G += r_t # Increment total reward by reward on current timestep\n",
    "            \n",
    "            if not state_action in [(x[0], x[1]) for x in episode[0:i]]: # \n",
    "                if returns.get(state_action):\n",
    "                    returns[state_action].append(G)\n",
    "                else:\n",
    "                    returns[state_action] = [G]   \n",
    "                    \n",
    "                Q[s_t][a_t] = sum(returns[state_action]) / len(returns[state_action]) # Average reward across episodes\n",
    "                \n",
    "                Q_list = list(map(lambda x: x[1], Q[s_t].items())) # Finding the action with maximum value\n",
    "                indices = [i for i, x in enumerate(Q_list) if x == max(Q_list)]\n",
    "                max_Q = random.choice(indices)\n",
    "                \n",
    "                A_star = max_Q # 14.\n",
    "                \n",
    "                for a in policy[s_t].items(): # Update action probability for s_t in policy\n",
    "                    if a[0] == A_star:\n",
    "                        policy[s_t][a[0]] = 1 - epsilon + (epsilon / abs(sum(policy[s_t].values())))\n",
    "                    else:\n",
    "                        policy[s_t][a[0]] = (epsilon / abs(sum(policy[s_t].values())))\n",
    "\n",
    "    return policy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}