Deep-Q-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import itertools\n",
    "import random\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-06-25 21:59:04,563] Making new env: CartPole-v1\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DeepQ:\n",
    "    def __init__(self,na,ns):\n",
    "        self.ns = ns\n",
    "        self.na = na\n",
    "        self.store = []\n",
    "        self.episode = 1000\n",
    "        self.discount = 1\n",
    "        self.epsilon = 1\n",
    "        self.learning_rate = 0.001\n",
    "        self.model = self._model()\n",
    "        self.epsilon_min = 0.01\n",
    "        self.epsilon_decay = 0.995\n",
    "    \n",
    "    def _model(self):\n",
    "        model = Sequential()\n",
    "        model.add(Dense(10,input_shape=(self.ns,),activation='relu'))\n",
    "        model.add(Dense(16,activation='relu'))\n",
    "        model.add(Dense(self.na,activation='softmax'))\n",
    "        model.compile(optimizer=Adam(lr=self.learning_rate),loss='mean_squared_error')\n",
    "#         model = Sequential()\n",
    "#         model.add(Dense(24, input_shape=(self.ns,), activation='relu'))\n",
    "#         model.add(Dense(24, activation='relu'))\n",
    "#         model.add(Dense(self.na, activation='linear'))\n",
    "#         model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))\n",
    "        return model\n",
    "    \n",
    "    def memory(self,step):\n",
    "        self.store.append((step[0],step[1],step[2],step[3],step[4])) #state,action,next_state,reward,done\n",
    "    \n",
    "#     def soft_greedy(self):\n",
    "#         def policy_fn(observation):\n",
    "#             A = np.ones(self.na, dtype=float) * self.epsilon / self.na\n",
    "# #             best_action = np.argmax(Q[observation])\n",
    "#             best_action = np.argmax(self.model.predict(observation)[0])\n",
    "#             A[best_action] += (1.0 - self.epsilon)\n",
    "#             return A\n",
    "#         return policy_fn\n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return random.randrange(self.na)\n",
    "        act_values = self.model.predict(state)\n",
    "        return np.argmax(act_values[0])\n",
    "\n",
    "    def greedy(self, nA):\n",
    "        def policy_fn(observation):\n",
    "            A = np.zeros(nA, dtype=float)\n",
    "            best_action = np.argmax(self.model.predict(observation)[0])\n",
    "            A[best_action] = 1\n",
    "            return A\n",
    "        return policy_fn\n",
    "    \n",
    "    def expirence_replay(self,batch_size):\n",
    "#         pass\n",
    "        batch = random.sample(self.store,batch_size)\n",
    "        for s,a,s_,R,done in batch:\n",
    "            temp = R + self.discount * np.amax(self.model.predict(s_)[0]) if not done else R\n",
    "            target = self.model.predict(s)\n",
    "            target[0][a] = temp\n",
    "            self.model.fit(s,target,verbose=0,epochs=1)\n",
    "        if self.epsilon > self.epsilon_min:\n",
    "            self.epsilon *= self.epsilon_decay\n",
    "    \n",
    "    def load(self, name):\n",
    "        self.model.load_weights(name)\n",
    "\n",
    "    def save(self, name):\n",
    "        self.model.save_weights(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n"
     ]
    }
   ],
   "source": [
    "deepQ = DeepQ(env.action_space.n,env.observation_space.shape[0])\n",
    "# policy = deepQ.soft_greedy(env.action_space.n)\n",
    "batch_size = 40\n",
    "\n",
    "for i_episode in xrange(deepQ.episode):\n",
    "    s = env.reset()\n",
    "    s = np.reshape(s, [1, deepQ.ns])\n",
    "    for i in itertools.count():\n",
    "#         probs = policy(s)\n",
    "#         a = np.random.choice(np.arange(len(probs)),p=probs)\n",
    "#         s_,R,done,_ = env.step(a)\n",
    "        a = deepQ.act(s)\n",
    "        s_,R,done,_ = env.step(a)\n",
    "        s_ = np.reshape(s_, [1, deepQ.ns])\n",
    "        R = R if not done else -5\n",
    "        deepQ.memory((s,a,s_,R,done))\n",
    "        if done:\n",
    "            break\n",
    "        s = s_\n",
    "        \n",
    "    if batch_size < len(deepQ.store):\n",
    "        deepQ.expirence_replay(batch_size)\n",
    "    \n",
    "    \n",
    "    if (i_episode+1)%100 == 0:\n",
    "        print i_episode+1\n",
    "    \n",
    "# deepQ.save('cartpole.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8.0, 8.0, 10.0, 10.0, 9.0, 8.0, 11.0, 10.0, 10.0, 10.0]\n"
     ]
    }
   ],
   "source": [
    "# polcy = soft_deepQ.greedy()\n",
    "reward_list = []\n",
    "for j_episode in xrange(10):\n",
    "    s = env.reset()\n",
    "    s = np.reshape(s,[1,deepQ.ns])\n",
    "    r = 0\n",
    "    for j in itertools.count():\n",
    "#         probs = polcy(s)\n",
    "#         a = np.random.choice(np.arange(len(probs)),p=probs)\n",
    "        s_,R,done,_ = env.step(deepQ.act(s))\n",
    "#         s_,R,done,_ = env.step(a)\n",
    "#         s_,R,done,_ = env.step(0)\n",
    "        s_ = np.reshape(s_,[1,deepQ.ns])\n",
    "        r += R\n",
    "        if done:\n",
    "            break\n",
    "        s = s_\n",
    "    reward_list.append(r)\n",
    "\n",
    "print reward_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env.observation_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.reshape(env.observation_space.sample(), [1, env.observation_space.shape[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "abcd = [[1,1],[2,2],[3,3],[4,4],[5,5],[6,6],[7,7],[8,8]]\n",
    "random.sample(abcd,k=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l = []\n",
    "for i in xrange(10):\n",
    "    l.append(env.observation_space.sample())\n",
    "for i in l:\n",
    "    i = np.reshape(i,[1,deepQ.ns])\n",
    "    print i\n",
    "    print deepQ.act(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}