Policy-Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import numpy as np\n",
    "try:\n",
    "    import cPickle as pickle\n",
    "except:\n",
    "    import pickle\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "try:\n",
    "    xrange = xrange\n",
    "except:\n",
    "    xrange = range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-06-02 09:45:49,252] Making new env: CartPole-v0\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "env = gym.make('CartPole-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reward for this episode was:', 28.0)\n",
      "('Reward for this episode was:', 21.0)\n",
      "('Reward for this episode was:', 18.0)\n",
      "('Reward for this episode was:', 11.0)\n",
      "('Reward for this episode was:', 16.0)\n",
      "('Reward for this episode was:', 26.0)\n",
      "('Reward for this episode was:', 54.0)\n",
      "('Reward for this episode was:', 11.0)\n",
      "('Reward for this episode was:', 13.0)\n",
      "('Reward for this episode was:', 17.0)\n"
     ]
    }
   ],
   "source": [
    "env.reset()\n",
    "random_episodes = 0\n",
    "reward_sum = 0\n",
    "while random_episodes < 10:\n",
    "    env.render()\n",
    "    observation, reward, done, _ = env.step(np.random.randint(0,2))\n",
    "    reward_sum += reward\n",
    "    if done:\n",
    "        random_episodes += 1\n",
    "        print(\"Reward for this episode was:\",reward_sum)\n",
    "        reward_sum = 0\n",
    "        env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "H = 10 # number of hidden layer neurons\n",
    "batch_size = 5 # every how many episodes to do a param update?\n",
    "learning_rate = 1e-2 # feel free to play with this to train faster or more stably.\n",
    "gamma = 0.99 # discount factor for reward\n",
    "\n",
    "D = 4 # input dimensionality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "#This defines the network as it goes from taking an observation of the environment to \n",
    "#giving a probability of chosing to the action of moving left or right.\n",
    "observations = tf.placeholder(tf.float32, [None,D] , name=\"input_x\")\n",
    "W1 = tf.get_variable(\"W1\", shape=[D, H],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "layer1 = tf.nn.relu(tf.matmul(observations,W1))\n",
    "W2 = tf.get_variable(\"W2\", shape=[H, 1],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "score = tf.matmul(layer1,W2)\n",
    "probability = tf.nn.sigmoid(score)\n",
    "\n",
    "#From here we define the parts of the network needed for learning a good policy.\n",
    "tvars = tf.trainable_variables()\n",
    "input_y = tf.placeholder(tf.float32,[None,1], name=\"input_y\")\n",
    "advantages = tf.placeholder(tf.float32,name=\"reward_signal\")\n",
    "\n",
    "# The loss function. This sends the weights in the direction of making actions \n",
    "# that gave good advantage (reward over time) more likely, and actions that didn't less likely.\n",
    "loglik = tf.log(input_y*(input_y - probability) + (1 - input_y)*(input_y + probability))\n",
    "loss = -tf.reduce_mean(loglik * advantages) \n",
    "newGrads = tf.gradients(loss,tvars)\n",
    "\n",
    "# Once we have collected a series of gradients from multiple episodes, we apply them.\n",
    "# We don't just apply gradeients after every episode in order to account for noise in the reward signal.\n",
    "adam = tf.train.AdamOptimizer(learning_rate=learning_rate) # Our optimizer\n",
    "W1Grad = tf.placeholder(tf.float32,name=\"batch_grad1\") # Placeholders to send the final gradients through when we update.\n",
    "W2Grad = tf.placeholder(tf.float32,name=\"batch_grad2\")\n",
    "batchGrad = [W1Grad,W2Grad]\n",
    "updateGrads = adam.apply_gradients(zip(batchGrad,tvars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def discount_rewards(r):\n",
    "    \"\"\" take 1D float array of rewards and compute discounted reward \"\"\"\n",
    "    discounted_r = np.zeros_like(r)\n",
    "    running_add = 0\n",
    "    for t in reversed(xrange(0, r.size)):\n",
    "        running_add = running_add * gamma + r[t]\n",
    "        discounted_r[t] = running_add\n",
    "    return discounted_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 27.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 21.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 23.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 21.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 17.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 25.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 24.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 23.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 27.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 17.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 23.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 24.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 30.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 33.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 27.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 42.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 19.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 20.000000.\n",
      "Average reward for episode 28.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 19.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 42.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 41.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 44.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 24.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 44.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 21.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 30.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 19.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 20.000000.  Total average reward 21.000000.\n",
      "Average reward for episode 35.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 43.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 38.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 21.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 39.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 49.000000.  Total average reward 22.000000.\n",
      "Average reward for episode 40.000000.  Total average reward 23.000000.\n",
      "Average reward for episode 40.000000.  Total average reward 23.000000.\n",
      "Average reward for episode 35.000000.  Total average reward 23.000000.\n",
      "Average reward for episode 36.000000.  Total average reward 23.000000.\n",
      "Average reward for episode 37.000000.  Total average reward 23.000000.\n",
      "Average reward for episode 38.000000.  Total average reward 23.000000.\n",
      "Average reward for episode 51.000000.  Total average reward 24.000000.\n",
      "Average reward for episode 61.000000.  Total average reward 24.000000.\n",
      "Average reward for episode 49.000000.  Total average reward 24.000000.\n",
      "Average reward for episode 56.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 36.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 50.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 30.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 26.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 36.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 30.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 54.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 22.000000.  Total average reward 25.000000.\n",
      "Average reward for episode 50.000000.  Total average reward 26.000000.\n",
      "Average reward for episode 43.000000.  Total average reward 26.000000.\n",
      "Average reward for episode 39.000000.  Total average reward 26.000000.\n",
      "Average reward for episode 39.000000.  Total average reward 26.000000.\n",
      "Average reward for episode 52.000000.  Total average reward 26.000000.\n",
      "Average reward for episode 38.000000.  Total average reward 26.000000.\n",
      "Average reward for episode 51.000000.  Total average reward 27.000000.\n",
      "Average reward for episode 57.000000.  Total average reward 27.000000.\n",
      "Average reward for episode 56.000000.  Total average reward 27.000000.\n",
      "Average reward for episode 74.000000.  Total average reward 28.000000.\n",
      "Average reward for episode 35.000000.  Total average reward 28.000000.\n",
      "Average reward for episode 50.000000.  Total average reward 28.000000.\n",
      "Average reward for episode 45.000000.  Total average reward 28.000000.\n",
      "Average reward for episode 48.000000.  Total average reward 28.000000.\n",
      "Average reward for episode 59.000000.  Total average reward 29.000000.\n",
      "Average reward for episode 64.000000.  Total average reward 29.000000.\n",
      "Average reward for episode 48.000000.  Total average reward 29.000000.\n",
      "Average reward for episode 59.000000.  Total average reward 30.000000.\n",
      "Average reward for episode 45.000000.  Total average reward 30.000000.\n",
      "Average reward for episode 47.000000.  Total average reward 30.000000.\n",
      "Average reward for episode 46.000000.  Total average reward 30.000000.\n",
      "Average reward for episode 74.000000.  Total average reward 30.000000.\n",
      "Average reward for episode 42.000000.  Total average reward 31.000000.\n",
      "Average reward for episode 51.000000.  Total average reward 31.000000.\n",
      "Average reward for episode 61.000000.  Total average reward 31.000000.\n",
      "Average reward for episode 51.000000.  Total average reward 31.000000.\n",
      "Average reward for episode 64.000000.  Total average reward 32.000000.\n",
      "Average reward for episode 52.000000.  Total average reward 32.000000.\n",
      "Average reward for episode 58.000000.  Total average reward 32.000000.\n",
      "Average reward for episode 53.000000.  Total average reward 32.000000.\n",
      "Average reward for episode 84.000000.  Total average reward 33.000000.\n",
      "Average reward for episode 106.000000.  Total average reward 33.000000.\n",
      "Average reward for episode 60.000000.  Total average reward 34.000000.\n",
      "Average reward for episode 45.000000.  Total average reward 34.000000.\n",
      "Average reward for episode 73.000000.  Total average reward 34.000000.\n",
      "Average reward for episode 70.000000.  Total average reward 35.000000.\n",
      "Average reward for episode 85.000000.  Total average reward 35.000000.\n",
      "Average reward for episode 94.000000.  Total average reward 36.000000.\n",
      "Average reward for episode 32.000000.  Total average reward 36.000000.\n",
      "Average reward for episode 65.000000.  Total average reward 36.000000.\n",
      "Average reward for episode 86.000000.  Total average reward 36.000000.\n",
      "Average reward for episode 65.000000.  Total average reward 37.000000.\n",
      "Average reward for episode 73.000000.  Total average reward 37.000000.\n",
      "Average reward for episode 68.000000.  Total average reward 37.000000.\n",
      "Average reward for episode 58.000000.  Total average reward 38.000000.\n",
      "Average reward for episode 84.000000.  Total average reward 38.000000.\n",
      "Average reward for episode 56.000000.  Total average reward 38.000000.\n",
      "Average reward for episode 106.000000.  Total average reward 39.000000.\n",
      "Average reward for episode 114.000000.  Total average reward 40.000000.\n",
      "Average reward for episode 71.000000.  Total average reward 40.000000.\n",
      "Average reward for episode 86.000000.  Total average reward 40.000000.\n",
      "Average reward for episode 75.000000.  Total average reward 41.000000.\n",
      "Average reward for episode 113.000000.  Total average reward 42.000000.\n",
      "Average reward for episode 86.000000.  Total average reward 42.000000.\n",
      "Average reward for episode 77.000000.  Total average reward 42.000000.\n",
      "Average reward for episode 70.000000.  Total average reward 43.000000.\n",
      "Average reward for episode 108.000000.  Total average reward 43.000000.\n",
      "Average reward for episode 127.000000.  Total average reward 44.000000.\n",
      "Average reward for episode 109.000000.  Total average reward 45.000000.\n",
      "Average reward for episode 140.000000.  Total average reward 46.000000.\n",
      "Average reward for episode 103.000000.  Total average reward 46.000000.\n",
      "Average reward for episode 102.000000.  Total average reward 47.000000.\n",
      "Average reward for episode 95.000000.  Total average reward 47.000000.\n",
      "Average reward for episode 139.000000.  Total average reward 48.000000.\n",
      "Average reward for episode 143.000000.  Total average reward 49.000000.\n",
      "Average reward for episode 84.000000.  Total average reward 49.000000.\n",
      "Average reward for episode 120.000000.  Total average reward 50.000000.\n",
      "Average reward for episode 110.000000.  Total average reward 51.000000.\n",
      "Average reward for episode 176.000000.  Total average reward 52.000000.\n",
      "Average reward for episode 130.000000.  Total average reward 53.000000.\n",
      "Average reward for episode 158.000000.  Total average reward 54.000000.\n",
      "Average reward for episode 157.000000.  Total average reward 55.000000.\n",
      "Average reward for episode 171.000000.  Total average reward 56.000000.\n",
      "Average reward for episode 118.000000.  Total average reward 57.000000.\n",
      "Average reward for episode 162.000000.  Total average reward 58.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 59.000000.\n",
      "Average reward for episode 156.000000.  Total average reward 60.000000.\n",
      "Average reward for episode 154.000000.  Total average reward 61.000000.\n",
      "Average reward for episode 141.000000.  Total average reward 62.000000.\n",
      "Average reward for episode 189.000000.  Total average reward 63.000000.\n",
      "Average reward for episode 170.000000.  Total average reward 64.000000.\n",
      "Average reward for episode 187.000000.  Total average reward 65.000000.\n",
      "Average reward for episode 163.000000.  Total average reward 66.000000.\n",
      "Average reward for episode 170.000000.  Total average reward 67.000000.\n",
      "Average reward for episode 165.000000.  Total average reward 68.000000.\n",
      "Average reward for episode 172.000000.  Total average reward 69.000000.\n",
      "Average reward for episode 176.000000.  Total average reward 71.000000.\n",
      "Average reward for episode 168.000000.  Total average reward 71.000000.\n",
      "Average reward for episode 177.000000.  Total average reward 73.000000.\n",
      "Average reward for episode 166.000000.  Total average reward 73.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 75.000000.\n",
      "Average reward for episode 152.000000.  Total average reward 75.000000.\n",
      "Average reward for episode 166.000000.  Total average reward 76.000000.\n",
      "Average reward for episode 190.000000.  Total average reward 77.000000.\n",
      "Average reward for episode 197.000000.  Total average reward 79.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 80.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 81.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 82.000000.\n",
      "Average reward for episode 161.000000.  Total average reward 83.000000.\n",
      "Average reward for episode 177.000000.  Total average reward 84.000000.\n",
      "Average reward for episode 194.000000.  Total average reward 85.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 86.000000.\n",
      "Average reward for episode 194.000000.  Total average reward 87.000000.\n",
      "Average reward for episode 167.000000.  Total average reward 88.000000.\n",
      "Average reward for episode 186.000000.  Total average reward 89.000000.\n",
      "Average reward for episode 181.000000.  Total average reward 90.000000.\n",
      "Average reward for episode 174.000000.  Total average reward 91.000000.\n",
      "Average reward for episode 191.000000.  Total average reward 92.000000.\n",
      "Average reward for episode 154.000000.  Total average reward 92.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 93.000000.\n",
      "Average reward for episode 188.000000.  Total average reward 94.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 95.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 96.000000.\n",
      "Average reward for episode 174.000000.  Total average reward 97.000000.\n",
      "Average reward for episode 147.000000.  Total average reward 97.000000.\n",
      "Average reward for episode 187.000000.  Total average reward 98.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 99.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 100.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 101.000000.\n",
      "Average reward for episode 169.000000.  Total average reward 102.000000.\n",
      "Average reward for episode 182.000000.  Total average reward 103.000000.\n",
      "Average reward for episode 196.000000.  Total average reward 104.000000.\n",
      "Average reward for episode 182.000000.  Total average reward 104.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 105.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 106.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 107.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 108.000000.\n",
      "Average reward for episode 166.000000.  Total average reward 109.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 109.000000.\n",
      "Average reward for episode 191.000000.  Total average reward 110.000000.\n",
      "Average reward for episode 193.000000.  Total average reward 111.000000.\n",
      "Average reward for episode 196.000000.  Total average reward 112.000000.\n",
      "Average reward for episode 168.000000.  Total average reward 112.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 113.000000.\n",
      "Average reward for episode 195.000000.  Total average reward 114.000000.\n",
      "Average reward for episode 174.000000.  Total average reward 114.000000.\n",
      "Average reward for episode 182.000000.  Total average reward 115.000000.\n",
      "Average reward for episode 196.000000.  Total average reward 116.000000.\n",
      "Average reward for episode 184.000000.  Total average reward 117.000000.\n",
      "Average reward for episode 189.000000.  Total average reward 117.000000.\n",
      "Average reward for episode 174.000000.  Total average reward 118.000000.\n",
      "Average reward for episode 195.000000.  Total average reward 119.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 120.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 120.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 121.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 122.000000.\n",
      "Average reward for episode 198.000000.  Total average reward 123.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 123.000000.\n",
      "Average reward for episode 197.000000.  Total average reward 124.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 125.000000.\n",
      "Average reward for episode 178.000000.  Total average reward 125.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 126.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 127.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 128.000000.\n",
      "Average reward for episode 184.000000.  Total average reward 128.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 129.000000.\n",
      "Average reward for episode 198.000000.  Total average reward 130.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 130.000000.\n",
      "Average reward for episode 178.000000.  Total average reward 131.000000.\n",
      "Average reward for episode 187.000000.  Total average reward 131.000000.\n",
      "Average reward for episode 186.000000.  Total average reward 132.000000.\n",
      "Average reward for episode 195.000000.  Total average reward 132.000000.\n",
      "Average reward for episode 188.000000.  Total average reward 133.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 134.000000.\n",
      "Average reward for episode 171.000000.  Total average reward 134.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 135.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 135.000000.\n",
      "Average reward for episode 190.000000.  Total average reward 136.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 136.000000.\n",
      "Average reward for episode 189.000000.  Total average reward 137.000000.\n",
      "Average reward for episode 177.000000.  Total average reward 137.000000.\n",
      "Average reward for episode 184.000000.  Total average reward 138.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 138.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 139.000000.\n",
      "Average reward for episode 178.000000.  Total average reward 139.000000.\n",
      "Average reward for episode 182.000000.  Total average reward 139.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 140.000000.\n",
      "Average reward for episode 173.000000.  Total average reward 140.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 141.000000.\n",
      "Average reward for episode 191.000000.  Total average reward 142.000000.\n",
      "Average reward for episode 190.000000.  Total average reward 142.000000.\n",
      "Average reward for episode 175.000000.  Total average reward 142.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 143.000000.\n",
      "Average reward for episode 180.000000.  Total average reward 143.000000.\n",
      "Average reward for episode 197.000000.  Total average reward 144.000000.\n",
      "Average reward for episode 197.000000.  Total average reward 144.000000.\n",
      "Average reward for episode 167.000000.  Total average reward 144.000000.\n",
      "Average reward for episode 156.000000.  Total average reward 145.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 145.000000.\n",
      "Average reward for episode 195.000000.  Total average reward 146.000000.\n",
      "Average reward for episode 186.000000.  Total average reward 146.000000.\n",
      "Average reward for episode 188.000000.  Total average reward 146.000000.\n",
      "Average reward for episode 167.000000.  Total average reward 147.000000.\n",
      "Average reward for episode 186.000000.  Total average reward 147.000000.\n",
      "Average reward for episode 184.000000.  Total average reward 147.000000.\n",
      "Average reward for episode 198.000000.  Total average reward 148.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 148.000000.\n",
      "Average reward for episode 197.000000.  Total average reward 149.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 149.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 150.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 150.000000.\n",
      "Average reward for episode 189.000000.  Total average reward 151.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 151.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 152.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 152.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 153.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 153.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 153.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 154.000000.\n",
      "Average reward for episode 191.000000.  Total average reward 154.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 155.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 155.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 156.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 156.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 156.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 157.000000.\n",
      "Average reward for episode 180.000000.  Total average reward 157.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 158.000000.\n",
      "Average reward for episode 171.000000.  Total average reward 158.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 158.000000.\n",
      "Average reward for episode 193.000000.  Total average reward 158.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 159.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 159.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 160.000000.\n",
      "Average reward for episode 179.000000.  Total average reward 160.000000.\n",
      "Average reward for episode 177.000000.  Total average reward 160.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 160.000000.\n",
      "Average reward for episode 184.000000.  Total average reward 161.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 161.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 161.000000.\n",
      "Average reward for episode 189.000000.  Total average reward 162.000000.\n",
      "Average reward for episode 186.000000.  Total average reward 162.000000.\n",
      "Average reward for episode 179.000000.  Total average reward 162.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 162.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 163.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 163.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 163.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 164.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 164.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 165.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 165.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 165.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 166.000000.\n",
      "Average reward for episode 194.000000.  Total average reward 166.000000.\n",
      "Average reward for episode 190.000000.  Total average reward 166.000000.\n",
      "Average reward for episode 166.000000.  Total average reward 166.000000.\n",
      "Average reward for episode 193.000000.  Total average reward 166.000000.\n",
      "Average reward for episode 182.000000.  Total average reward 167.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 167.000000.\n",
      "Average reward for episode 198.000000.  Total average reward 167.000000.\n",
      "Average reward for episode 186.000000.  Total average reward 167.000000.\n",
      "Average reward for episode 176.000000.  Total average reward 167.000000.\n",
      "Average reward for episode 187.000000.  Total average reward 168.000000.\n",
      "Average reward for episode 165.000000.  Total average reward 168.000000.\n",
      "Average reward for episode 197.000000.  Total average reward 168.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 168.000000.\n",
      "Average reward for episode 180.000000.  Total average reward 168.000000.\n",
      "Average reward for episode 191.000000.  Total average reward 169.000000.\n",
      "Average reward for episode 188.000000.  Total average reward 169.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 169.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 169.000000.\n",
      "Average reward for episode 174.000000.  Total average reward 169.000000.\n",
      "Average reward for episode 184.000000.  Total average reward 170.000000.\n",
      "Average reward for episode 198.000000.  Total average reward 170.000000.\n",
      "Average reward for episode 170.000000.  Total average reward 170.000000.\n",
      "Average reward for episode 194.000000.  Total average reward 170.000000.\n",
      "Average reward for episode 180.000000.  Total average reward 170.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 170.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 171.000000.\n",
      "Average reward for episode 188.000000.  Total average reward 171.000000.\n",
      "Average reward for episode 196.000000.  Total average reward 171.000000.\n",
      "Average reward for episode 187.000000.  Total average reward 171.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 171.000000.\n",
      "Average reward for episode 192.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 178.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 188.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 179.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 194.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 193.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 162.000000.  Total average reward 172.000000.\n",
      "Average reward for episode 191.000000.  Total average reward 173.000000.\n",
      "Average reward for episode 196.000000.  Total average reward 173.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 173.000000.\n",
      "Average reward for episode 183.000000.  Total average reward 173.000000.\n",
      "Average reward for episode 196.000000.  Total average reward 173.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 174.000000.\n",
      "Average reward for episode 185.000000.  Total average reward 174.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 174.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 174.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 174.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 175.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 175.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 175.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 175.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 176.000000.\n",
      "Average reward for episode 199.000000.  Total average reward 176.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 176.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 176.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 177.000000.\n",
      "Average reward for episode 200.000000.  Total average reward 177.000000.\n"
     ]
    }
   ],
   "source": [
    "xs,hs,dlogps,drs,ys,tfps = [],[],[],[],[],[]\n",
    "running_reward = None\n",
    "reward_sum = 0\n",
    "episode_number = 1\n",
    "total_episodes = 10000\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    rendering = False\n",
    "    sess.run(init)\n",
    "    observation = env.reset() # Obtain an initial observation of the environment\n",
    "\n",
    "    # Reset the gradient placeholder. We will collect gradients in \n",
    "    # gradBuffer until we are ready to update our policy network. \n",
    "    gradBuffer = sess.run(tvars)\n",
    "    for ix,grad in enumerate(gradBuffer):\n",
    "        gradBuffer[ix] = grad * 0\n",
    "    \n",
    "    while episode_number <= total_episodes:\n",
    "        \n",
    "        # Rendering the environment slows things down, \n",
    "        # so let's only look at it once our agent is doing a good job.\n",
    "        if reward_sum/batch_size > 100 or rendering == True : \n",
    "            env.render()\n",
    "            rendering = True\n",
    "            \n",
    "        # Make sure the observation is in a shape the network can handle.\n",
    "        x = np.reshape(observation,[1,D])\n",
    "        \n",
    "        # Run the policy network and get an action to take. \n",
    "        tfprob = sess.run(probability,feed_dict={observations: x})\n",
    "        action = 1 if np.random.uniform() < tfprob else 0\n",
    "        \n",
    "        xs.append(x) # observation\n",
    "        y = 1 if action == 0 else 0 # a \"fake label\"\n",
    "        ys.append(y)\n",
    "\n",
    "        # step the environment and get new measurements\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        drs.append(reward) # record reward (has to be done after we call step() to get reward for previous action)\n",
    "\n",
    "        if done: \n",
    "            episode_number += 1\n",
    "            # stack together all inputs, hidden states, action gradients, and rewards for this episode\n",
    "            epx = np.vstack(xs)\n",
    "            epy = np.vstack(ys)\n",
    "            epr = np.vstack(drs)\n",
    "            tfp = tfps\n",
    "            xs,hs,dlogps,drs,ys,tfps = [],[],[],[],[],[] # reset array memory\n",
    "\n",
    "            # compute the discounted reward backwards through time\n",
    "            discounted_epr = discount_rewards(epr)\n",
    "            # size the rewards to be unit normal (helps control the gradient estimator variance)\n",
    "            discounted_epr -= np.mean(discounted_epr)\n",
    "            discounted_epr //= np.std(discounted_epr)\n",
    "            \n",
    "            # Get the gradient for this episode, and save it in the gradBuffer\n",
    "            tGrad = sess.run(newGrads,feed_dict={observations: epx, input_y: epy, advantages: discounted_epr})\n",
    "            for ix,grad in enumerate(tGrad):\n",
    "                gradBuffer[ix] += grad\n",
    "                \n",
    "            # If we have completed enough episodes, then update the policy network with our gradients.\n",
    "            if episode_number % batch_size == 0: \n",
    "                sess.run(updateGrads,feed_dict={W1Grad: gradBuffer[0],W2Grad:gradBuffer[1]})\n",
    "                for ix,grad in enumerate(gradBuffer):\n",
    "                    gradBuffer[ix] = grad * 0\n",
    "                \n",
    "                # Give a summary of how well our network is doing for each batch of episodes.\n",
    "                running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01\n",
    "                print('Average reward for episode %f.  Total average reward %f.' % (reward_sum//batch_size, running_reward//batch_size))\n",
    "                \n",
    "                if reward_sum//batch_size > 200: \n",
    "                    print(\"Task solved in\",episode_number,'episodes!')\n",
    "                    break\n",
    "                    \n",
    "                reward_sum = 0\n",
    "            \n",
    "            observation = env.reset()\n",
    "        \n",
    "print(episode_number,'Episodes completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}