Taxi_Mellow_vs_Boltzmann.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Taxi_Mellow-vs-Boltzmann.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayam47/Mellowmax-RL/blob/master/Taxi_Mellow_vs_Boltzmann.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1Vg5tOMy15V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import deque\n",
        "from collections import defaultdict\n",
        "import gym\n",
        "import sys\n",
        "import math\n",
        "import numpy as np\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "no8VlX2yz8VF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "averages_history=[]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiLxQ1iYz8qu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def interact(env, agent, num_episodes=10000, window=100):\n",
        "    avg_rewards = deque(maxlen=num_episodes)\n",
        "    best_avg_reward = -math.inf\n",
        "    samp_rewards = deque(maxlen=window)\n",
        "    for i_episode in range(1, num_episodes+1):\n",
        "        state = env.reset()\n",
        "        samp_reward = 0\n",
        "        while True:\n",
        "            action = agent.select_action(state)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            agent.step(state, action, reward, next_state, done)\n",
        "            samp_reward += reward\n",
        "            state = next_state\n",
        "            if done:\n",
        "                samp_rewards.append(samp_reward)\n",
        "                break\n",
        "        avg_reward=0\n",
        "        if (i_episode >= 100):\n",
        "            avg_reward = np.mean(samp_rewards)\n",
        "            avg_rewards.append(avg_reward)\n",
        "            if avg_reward > best_avg_reward:\n",
        "                best_avg_reward = avg_reward\n",
        "        averages_history.append(avg_reward)\n",
        "        if i_episode%10==0:\n",
        "            print(\"\\rEpisode {}/{} || Average reward {} || Best average reward {}\".format(i_episode, num_episodes,avg_reward , best_avg_reward), end=\"\")\n",
        "            sys.stdout.flush()\n",
        "        if best_avg_reward >= 9.7:\n",
        "            print('\\nEnvironment solved in {} episodes.'.format(i_episode), end=\"\")\n",
        "#             return i_episode\n",
        "            break\n",
        "        if i_episode == num_episodes: print('\\n')\n",
        "#     return 100000\n",
        "    return avg_rewards, best_avg_reward"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6cf_13rfz_kP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from scipy.optimize import brentq\n",
        "\n",
        "class Agent:\n",
        "\n",
        "    def __init__(self, nA=6,omega=25,beta=1,mellow=0):\n",
        "        self.mellow=mellow\n",
        "        self.beta=beta\n",
        "        self.nA = nA\n",
        "        self.omega=omega\n",
        "        self.Q = defaultdict(lambda: np.zeros(self.nA))\n",
        "        self.gamma = 0.9\n",
        "        self.alpha = 0.1\n",
        "        \n",
        "    def mellowmax(self,beta,state):\n",
        "        c=np.max(self.Q[state])\n",
        "        mellow=c+np.log(np.sum(np.exp(self.omega*(self.Q[state]-c)))/self.nA)/self.omega\n",
        "        ans=np.sum(np.exp((self.Q[state]-mellow)*beta)*(self.Q[state]-mellow))\n",
        "        return ans\n",
        "\n",
        "    def get_action_probability(self, state):\n",
        "        if self.mellow==1:\n",
        "            beta=brentq(self.mellowmax,-500,500,args=(state,))\n",
        "        else:\n",
        "            beta=self.beta\n",
        "        policy_s = np.exp(beta*self.Q[state])/np.sum(np.exp(beta*self.Q[state]))\n",
        "        return policy_s\n",
        "\n",
        "    def select_action(self, state):\n",
        "        policy_s = self.get_action_probability(state)\n",
        "        return np.random.choice(np.arange(self.nA), p=policy_s)\n",
        "    \n",
        "    def step(self, state, action, reward, next_state, done):\n",
        "        next_action_value = self.Q[next_state][self.select_action(next_state)]\n",
        "        self.Q[state][action] += self.alpha * (reward  + self.gamma * next_action_value - self.Q[state][action])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V8YBlDOt0CQ_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "env = gym.make('Taxi-v2')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oOyxUXaO0Edc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "its=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28L0JHKP0F99",
        "colab_type": "code",
        "outputId": "d99b723d-088f-4676-b7e9-812e8d3bc46f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "source": [
        "x_best=[]\n",
        "for i in range(10,21):\n",
        "    b=0\n",
        "    for j in range(its):\n",
        "        averages_history=[]\n",
        "        agent = Agent(beta=i)\n",
        "        agent.init_for_sarsa()\n",
        "        avg_rewards,best_avg_rewards =interact(env,agent)\n",
        "        b+=best_avg_rewards\n",
        "    b/=its\n",
        "    x_best.append(b)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 10000/10000 || Average reward 8.69 || Best average reward 9.14\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.62 || Best average reward 9.1\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.09 || Best average reward 9.34\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.62 || Best average reward 9.22\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.31 || Best average reward 9.02\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.52 || Best average reward 9.17\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.42 || Best average reward 9.12\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.53 || Best average reward 9.01\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.41 || Best average reward 9.05\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.36 || Best average reward 9.26\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.6 || Best average reward 9.29\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jxtvNvp0Luy",
        "colab_type": "code",
        "outputId": "db723550-9c2c-44d0-ffd4-084cc9bcb60b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "source": [
        "y_best=[]\n",
        "for i in range(10,21):\n",
        "    b=0\n",
        "    for j in range(its):\n",
        "        averages_history=[]\n",
        "        agent = Agent(omega=i,mellow=1)\n",
        "        agent.init_for_sarsa()\n",
        "        avg_rewards,best_avg_rewards =interact(env,agent)\n",
        "        b+=best_avg_rewards\n",
        "    b/=its\n",
        "    y_best.append(b)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 10000/10000 || Average reward 7.25 || Best average reward 8.35\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.83 || Best average reward 8.66\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.14 || Best average reward 8.79\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.51 || Best average reward 8.86\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.82 || Best average reward 8.57\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.61 || Best average reward 8.8\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.69 || Best average reward 8.78\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.22 || Best average reward 8.83\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.69 || Best average reward 9.08\n",
            "\n",
            "Episode 10000/10000 || Average reward 7.63 || Best average reward 8.88\n",
            "\n",
            "Episode 10000/10000 || Average reward 8.55 || Best average reward 8.88\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-lDej_qQ0O2u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DOK-CikA0RGc",
        "colab_type": "code",
        "outputId": "23d93357-bb7a-41e5-ca9a-458da429765e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        }
      },
      "source": [
        "fig_best=plt.figure()\n",
        "ax=fig_best.add_subplot(111)\n",
        "plt.plot(np.arange(10,21),x_best,label='Boltzmann')\n",
        "plt.plot(np.arange(10,21),y_best,label='Mellowmax')\n",
        "plt.ylim(0,10)\n",
        "plt.ylabel('mean return')\n",
        "plt.xlabel('omega/beta')\n",
        "plt.legend()\n",
        "plt.savefig('taxi.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYVPWd7/H3t6r3ha27aRCIoIIL\nCKiIy7hmFo3XNXFGjTpxnOhzk3HJzKhXbhaNeXKTa3wmi7PF6IgxPCa5hjGuSdRI1IgRFAQEEQUE\nZOtFoPfq5Xv/OKe7qxsaSrqriq7zeT1PPefUqV+d8zsF/fuc8ztVv2PujoiIRFcs2xUQEZHsUhCI\niEScgkBEJOIUBCIiEacgEBGJOAWBiEjEpS0IzOy/zGynma1KWjbGzJ43s3XhdHS6ti8iIqlJ5xnB\nfOD8fsvuBF5096nAi+FzERHJIkvnD8rMbDLwtLvPCJ+vBc5x921mNh5Y5O5Hp60CIiJyQHkZ3l61\nu28L57cD1QMVNLMbgRsBSktLTzrmmGMyUD0Rkdzx5ptv1rp71YHKZToIeri7m9mApyPu/gDwAMCc\nOXN86dKlGaubiEguMLMPUymX6W8N7Qi7hAinOzO8fRER6SfTQfAk8IVw/gvArzO8fRER6SedXx99\nDFgMHG1mW8zs74HvAn9pZuuAvwifi4hIFqXtGoG7XzXAS3+erm2KiMgnp18Wi4hEnIJARCTiFAQi\nIhGnIBARiTgFgYhIxCkIREQiTkEgIhJxCoIc09nldHWlb0RZEck9WRt0TgbH3floVwvv7Whg7fbG\ncNrA+zWNdHR2Maa0kMqyAirKCqgoLQynBVSUFfZMg9cLKS2IY2bZ3iURyRIFwSHO3alpbOO97Y2s\n3dHAe9sbWLujgXU7GmhKdPaUO2xkEdPGlXPG1EoK82LUNiaoa2yjrinBii27qGtK0NDasc9tFObF\nqCzrFxZlBVR2B0hPeBQwprSAwrx4pnZfJGvcndrGBOtrGvm4uZ2Ori7aO7to73TaO7voCKe9z7tI\ndDodnWG5Lqe9o4uOru5ywXsSye/tKdO7nv7lnrnlDI6oKkvrvioIDiG7m9uDxj58rN0eTD9ubu8p\nU1lWwNSx5fz1nElMqy7n6HFlTK0uZ0RR/gHX39bRSX1TgrrGBLWNbdQ1JqhragufB/O1jQnWbm+g\ntjFBorNrn+spL8oLgiMMh4qyQipLC6gqL+RTFaVMqSjlsFFF5MXV8yiHvuZEB+trmthQ2xROG1lf\n28SGmiYa2vZ98DSQgniMvLiRH4+RH057nsdi5OcZebEYBfEYBXkxSvuUC+eTy+XFKE/hb3uwFARZ\n0JzoYN2Ovkf47+1oYMeetp4y5YV5TBtXzvkzxnN0dRnTxpUzrbqcyrLCg95uYV6c8SOLGT+y+IBl\n3Z3Gto6esAjOMHrPMrqDZENtE0s3fkx9c4Lkm93lx41Jo0uYXFnK4RUlTKks5fAcCAl3p6Gtg+27\nW9m+u5U9re0U5cUpyo9TlB/rmRbmxSkuCJfnxYbt/uaKjs4utnzcwobaJj6oaUxq9JvYvqe1T9kJ\no4o5oqqUy06cwJTKUo6oKqOitICCvFjQYMeMgrxgmp8XNvBxIx6zYdvFmtNB8KMX1/HO1t0U5MUp\niMcozA+SuDAveBSEj8K8eDCfVCZ5+f7K58cH/sdv6+hkfU1T0hF+0Je/qb65p0xRfoypY8s546iq\nnqP7o6vLGT+yKKv/qcyM8qJ8yovymVxZesDynV1ObWMbG2ub2FjXxMa65nC+mdfX19Gc1I11qIZE\nV5dT29TW08hv37PvafK+pCovZvsIiVifECnMj1Ocv/fyYJr0PHz/yOJ8xpQG3XVF+equ6+7KCRr5\nxrDRD47wN9U3097Ze6QysjifI6pKOf2oCo6sKmNKZSlTKkuZXFFKcUH0PsucDoK6xjY21jaT6Owi\n0dFFW0cnbR3d8/vu9vikzOgTHN2B4e5s/riFzvAbPHkx44iqUmZOHMlfnzSRaeOCBn/SmBLiseF5\nFJEsHjOqRxRRPaKIU46o6POau1PT0MaG2iY+rGtmQ13TJwqJyRXBYzAh0dbRyc49bWwLG/Qdu1vZ\ntruVHXta2ba7hR172tixp5WOft+4yosZY8sLGTeyiGPGlXP2tCrGjwz2c/zIYkYW59PW0Ulrexct\n7Z20ho+29i5aO7qfd9Ha3hm+3kVbe2f4WrC8OdFBfVNXz3tbO3rnU/0CWHF+nDGlBYwuzWd0SUFP\nQIwpKWB0ON+9vLtM/nA5S2lrhJp3oX49lFXTUn446xMjWV/b0qfRX1/b1Oc6WEE8xuTKEo4aW8Zf\nTR/HlMpSjqwqZUplGaNL8oft0Xs6pPXm9UMlHbeqdHfawwsybe2dSWHRf9pJoqMrLNfVJ1QSSWXb\n+pXpcueIytKwH7+cyRWlFOQNkz+8DNpXSHxY18SG2mY+rGsaMCQmV5QyubKEyRVBYCQ6uti+J2zc\nd7eyrV9jX9eU2GvbJQVxxo0oYtzIot5p8vyIIirKCrMW1N3/R3sCJdEbLi2JTna3tPNxc4K6pgQf\nNyWobwqe1zcleqYDfUEAgms9FaVhUOwVGEFYVJT1BsiIonxi+/gs3J2OLu/5m0l09DvwOsDfTvff\nVyKRoLh5M2Ma1lHR9D5jm99nXOsHVLVv3WubrZ7Ph17Nh15NbcFEWkccjlUcRem4qVRPOpIjqso5\nbFRxThxkDYaZvenucw5YLqpBIIe+TxIS/Y0pLdi7cU9+PrKI8sK8nD8qTHR0sas5QX13QDS1U9/c\nHRy9gVEfhkldU2LAs+WYweiSoBsqaMB7G/pP2oyMooFjYps5xjYFj9hmptkWSiy4TtZJjC02jg2x\nyXyYN4XNBVOoL5jI1NJmji2s5VNso6r9I0obPyT28Ubo7L2+Rl4RjJ4CY46AiiNgzJHh/JFQfhjE\nonNApiCQnNYnJOqbKcqP9zT2Y0cUqs98EFoSnQOGRX1Tgtb2rr2unfVcQ4sH1zq6u0uLYh2MatrI\nyIZ1jNi9ltJdayn+eA15TTt6tufFY/DqGVj1dKx6OlRPh6pjoKAktQp3dcGej6D+g6D7qC6c1q+H\n+g37DomKI2HMlJwPCQWBiGSOOzRsgx3vwI5VsGN1MF+7FrrC7ql4AVQdDWPDxr77UVYdXGxLh6EK\nieIx6avjgcQLDzqgUg2CnL5YLIPU2QGJxuDR1j1tgERT0nz3a02AQ/UMmHBicEQX01F5Tko0wc53\nwwb/neCx8x1o+bi3zIiJQSM/7bzeBr/iKIin/zvxfcRiMGpS8DjinL6vdXXCnq1BSCQHRN37sO75\nviGRTf+wBKqmpXUTCoJc09YIrbt6G+69GvGkhjvRkPTaPpZ1tB54ewAYFJYHf1jtTcGi/BIYPwsO\nOzEIhsNOCI6ucqVPPtEMdeugYTtYPAi9WBxiecFjn8tivfM9y+Nh2aTlFvvkn1NXV/Dv1dEK7S3B\no6MF2lsHmHaXad3/dF/LWncDYU9CQRmMPQ6OuyQ4CKieDmOPheLRQ/6RD7lYPPWQaNuTjRoGSivT\nvgl1DQ0HHW3QuDN87AgeTTW98z2v7extiPfHYsEfcEEZFJZBQWk4X95vWXk4X9Y7TZ4vLA/K5ZcE\nDVdXV/CH89FbsPWtYLp9RW+gFI0KAmHCib0BMeKw9H52g9XWGHRv1KwNvsLYPf34Q3oaw3ToCZPu\ngIj1WxaDjkRv4z6Yo9e8ouCRX9w7zS+GvGLI38drpVVB4189HUYdnnP96rlE1wgOdZ0d0FwbNuT9\nGvWmfo1+6+59r6N4dNC/WjY2mJaOhbKqoD8zlYY7I/vZDjvX9AbD1reC/mMPv/FTNi4pGE4IpiVj\nMlO3ZK27oea9sLF/t7fR3725t0wsHyqnBt1eVccE/d0jJwb9410dwT51dYSPrt557wyOMLs6kqYd\n4Ellkl/3/mU7973uvIKkxrrfNL9k7wZ8X439IPqf5dCnIMimll3BkXDD9qQGfWffI/mmWvZ5RFlQ\nHjbsY3sb+D4Nffd8VdAQDEftLbB9Zd8zh7p1va+PntL3rGH8rCDAhkJz/d5H9zVroSHpu+p5RUkN\n/tG9Df/oKRBXb6oMH7pYnCldXUFjsmUJbHkDNi8JuhKSxQt7G/RRh8PEk/s19GHDXjZ26Bq8Q1l+\nMUyaGzy6te6GrcvDYHgTNv0JVv0qeM1iQUOcfNZQPWP/QdhU23tkvzPpCL9pZ1I9SqByGhxxdlKD\nf3TY3aEL3RIdOiP4pJrrg4Zq8xtB4//Rm70XkorHBI38pJNhwkkwclLQuBeOyJ2LpJnUuLPvWcPW\nt6C5LngtXtD7DaXxs6G9ue9Rfnc5CM6ykhv6nm6dSeoWkZymrqGh0NUZ9G9vWRI8Nr/R24VhseBi\n2cS5YeM/N7e+FXMocoddm5KCYVlwFpFoCF4vGpnUf5/U6I84TP8uEknqGjoYzfW9Df6WJUFj093I\nlFQEjf7sq4KG/7ATg4uwkjlmMPrw4DH9smBZV1fw3e/CsvT+MEkkh0U3CDo7oGZNb6O/ZUnwQxII\nvqJXPR1mXREe8c/R0f6hKhaDyqOyXQuRYS06QdBU27eL56O3er9zX1oVHu1fHXTxHHZCNC7aioiQ\n60Gw4pfw/ovBt3nq1wfLLA7jjocTrg66eCaeDKMn62hfRCIrt4NgzVOw6fXgKP/ELwTT8bNTH9VQ\nRCQCcjsILvvPzP6KVkRkGMrtIFA/v4jIAenXNCIiEacgEBGJuKwEgZn9o5m9Y2arzOwxMyvKRj1E\nRCQLQWBmE4BbgDnuPgOIA1dmuh4iIhLIVtdQHlBsZnlACbD1AOVFRCRNMh4E7v4RcB+wCdgG7Hb3\n3/UvZ2Y3mtlSM1taU1OT6WqKiERGNrqGRgOXAFOAw4BSM7umfzl3f8Dd57j7nKqqqkxXU0QkMrLR\nNfQXwAZ3r3H3dmAhcHoW6iEiImQnCDYBp5pZiZkZ8OfAmizUQ0REyM41gj8BjwNvASvDOjyQ6XqI\niEggK0NMuPtdwF3Z2LaIiPSlXxaLiEScgkBEJOIUBCIiEacgEBGJOAWBiEjEKQhERCJOQSAiEnEK\nAhGRiFMQiIhEnIJARCTiFAQiIhGnIBARiTgFgYhIxCkIREQiTkEgIhJxCgIRkYhTEIiIRJyCQEQk\n4hQEIiIRpyAQEYk4BYGISMQpCEREIk5BICIScQoCEZGIUxCIiEScgkBEJOIUBCIiEacgEBGJOAWB\niEjEKQhERCIu70AFzKwQ+BwwObm8u9+TvmqJiEimHDAIgF8Du4E3gbb0VkdERDItlSCY6O7np70m\nIiKSFalcI3jNzI5Pe01ERCQrUgmCM4A3zWytma0ws5VmtmIwGzWzUWb2uJm9a2ZrzOy0waxPREQO\nXipdQ59Jw3Z/CPzG3S83swKgJA3bEBGRFOw3CMwsDvzW3Y8Zqg2a2UjgLOA6AHdPAImhWr+IiHwy\n++0acvdOYK2ZfWoItzkFqAEeNrNlZvagmZX2L2RmN5rZUjNbWlNTM4SbFxGRZKlcIxgNvGNmL5rZ\nk92PQWwzDzgR+A93PwFoAu7sX8jdH3D3Oe4+p6qqahCbExGR/UnlGsHXh3ibW4At7v6n8Pnj7CMI\nREQkMw4YBO7+h6HcoLtvN7PNZna0u68F/hxYPZTbEBGR1KUyxEQD4OHTAiAfaHL3EYPY7s3AgvAb\nQ+uBvxvEukREZBBSOSMo7543MwMuAU4dzEbdfTkwZzDrEBGRofGJRh/1wBPAeWmqj4iIZFgqXUOf\nTXoaIziSb01bjUREJKNS+dbQRUnzHcBGgu4hERHJAakEwYPu/sfkBWb2Z8DO9FRJREQyKZVrBPen\nuExERIahAc8IwhFBTweqzOyfkl4aAcTTXTEREcmM/XUNFQBlYZnypOV7gMvTWSkREcmcAYMg/EXx\nH8xsvrt/aGYl7t6cwbqJiEgGpHKN4DAzWw28C2Bms8zs39NbLRERyZRUguAHBD8gqwNw97cJ7icg\nIiI5IKVfFrv75n6LOtNQFxERyYJUfkew2cxOB9zM8oFbgTXprZaIiGRKKmcE/xP4B2AC8BEwO3wu\nIiI5IJV7Fl/r7ldnqD4iIpJhqdyz+PMZqouIiGRBKtcIXjWzfwV+QXB/YQDc/a201UpERDImlSCY\nHU7vSVrmwKeHvjoiIpJpqdyh7NxMVERERLLjE92hTEREco+CQEQk4hQEIiIRl8rFYsJfFk9OLu/u\nP01TnUREJINSuXn9o8CRwHJ6xxhyQEEgIpIDUjkjmAMc5+6e7sqIiEjmpXKNYBUwLt0VERGR7Ejl\njKASWG1mbwBt3Qvd/eK01UpERDImlSC4O92VEBGR7Enll8V/yERFREQkOw54jcDMTjWzJWbWaGYJ\nM+s0sz2ZqJyIiKRfKheL/xW4ClgHFANfBP4tnZUSEZHMSfWexe8DcXfvdPeHgfPTWy0REcmUVC4W\nN5tZAbDczO4FtqGhKUREckYqDfq1YbmbCG5MMwn4XDorJSIimZPKt4Y+NLNiYLy7fzMDdRIRkQxK\n5VtDFxGMM/Sb8PlsM3sy3RUTEZHMSKVr6G5gLrALwN2XA1MGu2Ezi5vZMjN7erDrEhGRg5dKELS7\n++5+y4ZiALpbgTVDsB4RERmEVILgHTP7PBA3s6lmdj/w2mA2amYTgf8BPDiY9YiIyOClEgQ3A9MJ\nBpx7DNgDfGWQ2/0BcAfQNVABM7vRzJaa2dKamppBbk5ERAZywCBw92Z3/6q7n+zuc8L51oPdoJld\nCOx09zcPsN0Hwu3NqaqqOtjNiYjIAaRyh7I5wP9m71tVzjzIbf4ZcLGZXQAUASPM7Gfufs1Brk9E\nRAYhlV8WLwBuB1ayn66cVLn7PGAegJmdA9ymEBARyZ5UgqDG3fW7ARGRHJVKENxlZg8CL9L3DmUL\nB7txd18ELBrsekRE5OClEgR/BxwD5NPbNeTAoINARESyL5UgONndj057TUREJCtS+R3Ba2Z2XNpr\nIiIiWZHKGcGpBPci2EBwjcAAH8TXR0VE5BCSShDobmQiIjkspfsRZKIiIiKSHbrlpIhIxCkIREQi\nTkEgIhJxCgIRkYhTEIiIRJyCQEQk4hQEIiIRpyAQEYk4BYGISMQpCEREIk5BICIScQoCEZGIUxCI\niEScgkBEJOIUBCIiEacgEBGJOAWBiEjEKQhERCJOQSAiEnEKAhGRiFMQiIhEnIJARCTiFAQiIhGn\nIBARiTgFgYhIxCkIREQiTkEgIhJxCgIRkYjLeBCY2SQze8nMVpvZO2Z2a6brICIivfKysM0O4J/d\n/S0zKwfeNLPn3X11FuoiIhJ5GT8jcPdt7v5WON8ArAEmZLoeIiISyOo1AjObDJwA/Gkfr91oZkvN\nbGlNTU2mqyYiEhlZCwIzKwN+BXzF3ff0f93dH3D3Oe4+p6qqKvMVFBGJiKwEgZnlE4TAAndfmI06\niIhIIBvfGjLgIWCNu/9LprcvIiJ9ZeOM4M+Aa4FPm9ny8HFBFuohIiJk4euj7v4qYJneroiI7Fs2\nfkcwJNrb29myZQutra3ZrkrOKyoqYuLEieTn52e7KiKSBsM2CLZs2UJ5eTmTJ08muOwg6eDu1NXV\nsWXLFqZMmZLt6ohIGgzbsYZaW1upqKhQCKSZmVFRUaEzL5EcNmyDAFAIZIg+Z5HcNqyDQEREBk9B\nMAjxeJzZs2cza9YsTjzxRF577bX9lt+4cSMzZswAYPny5Tz77LOZqKaIyH4pCAahuLiY5cuX8/bb\nb/Od73yHefPmpfxeBYGIHCqG7beGkn3zqXdYvXWv4YoG5bjDRnDXRdNTLr9nzx5Gjx4NBN+0ueOO\nO3juuecwM772ta9xxRVX9JRNJBJ84xvfoKWlhVdffZV58+bxyCOPsHXrVgA2bNjAj370I9ydJ554\ngqamJtatW8dtt91GIpHg0UcfpbCwkGeffZYxY8bwk5/8hAceeIBEIsFRRx3Fo48+SklJCddddx0j\nRoxg6dKlbN++nXvvvZfLL7+cRYsWcffdd1NZWcmqVas46aST+NnPfqZrASIRpTOCQWhpaWH27Nkc\nc8wxfPGLX+TrX/86AAsXLuw5U3jhhRe4/fbb2bZtW8/7CgoKuOeee7jiiitYvnw5V1xxBc8++yzL\nly/noYce4vDDD+fSSy8FYNWqVSxcuJAlS5bw1a9+lZKSEpYtW8Zpp53GT3/6UwA++9nPsmTJEt5+\n+22OPfZYHnrooZ5tbdu2jVdffZWnn36aO++8s2f5smXL+MEPfsDq1atZv349f/zjHzPxkYnIISgn\nzgg+yZH7UOruGgJYvHgxf/u3f8uqVat49dVXueqqq4jH41RXV3P22WezZMkSZs6cud/11dbWcu21\n1/LLX/6SkSNHAnDuuedSXl5OeXk5I0eO5KKLLgLg+OOPZ8WKFUAQFl/72tfYtWsXjY2NnHfeeT3r\nvPTSS4nFYhx33HHs2LGjZ/ncuXOZOHEiALNnz2bjxo2cccYZQ/fhiMiwkRNBcCg47bTTqK2t5WDv\nndDZ2cmVV17JN77xjZ4LygCFhYU987FYrOd5LBajo6MDgOuuu44nnniCWbNmMX/+fBYtWrTP97v7\nPpfH4/GedYlI9KhraIi8++67dHZ2UlFRwZlnnskvfvELOjs7qamp4eWXX2bu3Ll9ypeXl9PQ0NDz\n/M4772TmzJlceeWVn3jbDQ0NjB8/nvb2dhYsWDDofRGRaNEZwSB0XyOA4Gj7kUceIR6Pc9lll7F4\n8WJmzZqFmXHvvfcybtw4Nm7c2PPec889l+9+97vMnj2befPmcd999zF9+vSe9d1zzz0p1+Nb3/oW\np5xyClVVVZxyyil9AkZE5EAsubvgUDVnzhxfunRpn2Vr1qzh2GOPzVKNokeft8jwY2ZvuvucA5VT\n15CISMQpCEREIk5BICIScQoCEZGIUxCIiEScgkBEJOIUBINgZlxzzTU9zzs6OqiqquLCCy/c7/sW\nLVrUU2b+/PncdNNNaa2niMj+KAgGobS0lFWrVtHS0gLA888/z4QJE7JcKxGRTyY3fln83J2wfeXQ\nrnPc8fCZ7x6w2AUXXMAzzzzD5ZdfzmOPPcZVV13FK6+8AkBTUxM333wzq1ator29nbvvvptLLrlk\nwHVt3LiR66+/ntraWqqqqnj44YeZMGECRx11FOvXr2f37t1UVFTw0ksvcdZZZ3HWWWfx0EMPsWDB\nAjZs2MD69evZtGkT3//+93n99dd57rnnmDBhAk899RT5+fncc889PPXUU7S0tHD66afz4x//mM7O\nTk477TS+973vcc455zBv3jxisRjf/va3h+yjFJFDm84IBunKK6/k5z//Oa2traxYsYJTTjml57Vv\nf/vbfPrTn+aNN97gpZde4vbbb6epqWnAdd1888184QtfYMWKFVx99dXccsstxONxjj76aFavXs2r\nr77KiSeeyCuvvEJbWxubN29m6tSpAHzwwQf8/ve/58knn+Saa67h3HPPZeXKlRQXF/PMM88AcNNN\nN7FkyZKes5inn36avLw85s+fz5e+9CVeeOEFfvOb33DXXXel90MTkUNKbpwRpHDkni4zZ85k48aN\nPPbYY1xwwQV9Xvvd737Hk08+yX333QdAa2srmzZtGnBdixcvZuHChQBce+213HHHHQCceeaZvPzy\ny2zYsIF58+bxk5/8hLPPPpuTTz65572f+cxnyM/P5/jjj6ezs5Pzzz8fCIar7h7j6KWXXuLee++l\nubmZ+vp6pk+fzkUXXcT06dO59tprufDCC1m8eDEFBQVD9vmIyKFPZwRD4OKLL+a2227jqquu6rPc\n3fnVr37F8uXLWb58OZs2bTqo8XrOOussXnnlFd544w0uuOACdu3axaJFizjzzDN7yiQPT52fn99z\nt7Hu4apbW1v58pe/zOOPP87KlSu54YYbaG1t7Xn/ypUrGTVqFDt37jyYj0BEhjEFwRC4/vrrueuu\nuzj++OP7LD/vvPO4//77e+4DsGzZsv2u5/TTT+fnP/85AAsWLOhp6OfOnctrr71GLBajqKiI2bNn\n8+Mf/5izzjor5Tp2N/qVlZU0Njby+OOP97y2cOFC6uvrefnll7n55pvZtWtXyusVkeFPQTAEJk6c\nyC233LLX8q9//eu0t7czc+ZMpk+f3nMry4Hcf//9PPzww8ycOZNHH32UH/7wh0BwtD9p0iROPfVU\nIOgqamho2Ct49mfUqFHccMMNzJgxg/POO6+nW6m2tpY777yTBx98kGnTpnHTTTdx6623prxeERn+\nNAy1pESft8jwo2GoRUQkJQoCEZGIG9ZBMBy6tXKBPmeR3DZsg6CoqIi6ujo1Umnm7tTV1VFUVJTt\nqohImgzbH5RNnDiRLVu2UFNTk+2q5LyioiImTpyY7WqISJoM2yDIz89nypQp2a6GiMiwl5WuITM7\n38zWmtn7ZnZnNuogIiKBjAeBmcWBfwM+AxwHXGVmx2W6HiIiEsjGGcFc4H13X+/uCeDnwMBjM4uI\nSFpl4xrBBGBz0vMtwCn9C5nZjcCN4dNGM1t7kNurBGoP8r3DlfY5GrTPuW+w+3t4KoUO2YvF7v4A\n8MBg12NmS1P5iXUu0T5Hg/Y592Vqf7PRNfQRMCnp+cRwmYiIZEE2gmAJMNXMpphZAXAl8GQW6iEi\nImSha8jdO8zsJuC3QBz4L3d/J42bHHT30jCkfY4G7XPuy8j+DothqEVEJH2G7VhDIiIyNBQEIiIR\nl1NBYGb/ZWY7zWxV0rIxZva8ma0Lp6OzWcehNsA+f8/M3jWzFWb232Y2Kpt1HGr72uek1/7ZzNzM\nKrNRt3QYaH/N7Obw3/kdM7s3W/VLhwH+X882s9fNbLmZLTWzudms41Azs0lm9pKZrQ7/TW8Nl6e9\nDcupIADmA+f3W3Yn8KK7TwVeDJ/nkvnsvc/PAzPcfSbwHjAv05VKs/nsvc+Y2STgr4BNma5Qms2n\n3/6a2bkEv8if5e7TgfuyUK90ms/e/8b3At9099nAN8LnuaQD+Gd3Pw44FfiHcPidtLdhORUE7v4y\nUN9v8SXAI+H8I8ClGa1Umu1rn939d+7eET59neC3GjljgH9ngO8DdwA59Q2IAfb3S8B33b0tLLMz\n4xVLowH22YER4fxIYGtGK5UEXdkYAAAFBklEQVRm7r7N3d8K5xuANQQjMaS9DcupIBhAtbtvC+e3\nA9XZrEwWXA88l+1KpJuZXQJ85O5vZ7suGTINONPM/mRmfzCzk7NdoQz4CvA9M9tMcAaUa2e6Pcxs\nMnAC8Ccy0IZFIQh6ePBd2Zw6WtwfM/sqwenmgmzXJZ3MrAT43wTdBVGRB4wh6EK4HfilmVl2q5R2\nXwL+0d0nAf8IPJTl+qSFmZUBvwK+4u57kl9LVxsWhSDYYWbjAcJpTp1CD8TMrgMuBK723P+xyJHA\nFOBtM9tI0BX2lpmNy2qt0msLsNADbwBdBAOU5bIvAAvD+f9HMJJxTjGzfIIQWODu3fua9jYsCkHw\nJMF/IMLpr7NYl4wws/MJ+sovdvfmbNcn3dx9pbuPdffJ7j6ZoJE80d23Z7lq6fQEcC6AmU0DCsj9\nUTm3AmeH858G1mWxLkMuPKN7CFjj7v+S9FL62zB3z5kH8BiwDWgnaAz+HqgguNK+DngBGJPtemZg\nn98nGOp7efj4z2zXM9373O/1jUBltuuZ5n/jAuBnwCrgLeDT2a5nBvb5DOBN4G2CvvOTsl3PId7n\nMwi6fVYk/e1ekIk2TENMiIhEXBS6hkREZD8UBCIiEacgEBGJOAWBiEjEKQhERCLukL15vchwZmZX\nEvzQLR9odPeUBoULR4r9vLv/ezrrJ5JMZwQi6fEZ4DcH8b5RwJeHuC4i+6UgkJxiZv9kZqvCx1fM\nbHI4Zv98M3vPzBaY2V+Y2R/D8d3nhu8rDcfAf8PMloWD2GFmJWb2y3CM+P8OB3mbE772H+G4+O+Y\n2TeT6mDAbIIfegHMMrPF4fZuSCp3u5ktCe8b0f3+7wJHhmPuf8/MyszsRTN7y8xWdtdLZCipa0hy\nhpmdBPwdcApgBL8+/QNwFPDXBCOxLgE+T/ArzosJBqu7FPgq8Ht3vz7snnnDzF4gGOjsY3c/zsxm\nEPzas9tX3b3ezOLAi2Y2091XEIwa+ba7ezgO3EyCweFKgWVm9gwwA5hKMF6OAU+a2VkEY83P8GDM\nfcwsD7jM3feEN9t53cyedP0SVIaQgkByyRnAf7t7E4CZLQTOBDa4+8pw2TsEN/lwM1sJTA7f+1fA\nxWZ2W/i8CPhUuM4fArj7KjNbkbS9vzGzGwn+jsYDxxEMD3A+fYf+/rW7twAtZvYSQeN/RrjNZWGZ\nMoJg6H9THQP+TxgSXQTj01cTDEcsMiQUBBIFbUnzXUnPu+j9GzDgc+6+NvmNA43sbGZTgNuAk939\nYzObTxAeEDTwn0sq3v/o3cPtfcfdf9xvvZP7lb0aqCIYV6c9HF21CJEhpGsEkkteAS4N+/VLgcvC\nZan4LXBz95j+ZnZCuPyPwN+Ey44Djg+XjwCagN1mVk1wcRgzGwnkuXtd0rovMbMiM6sAziHonvot\ncH049jxmNsHMxgINQHnSe0cCO8MQOBc4PMX9EUmZzggkZ7j7W+GR+RvhogeBj1N8+7eAHwArzCwG\nbCC4n8O/A4+Y2WrgXeAdYLe7rzOzZeGyzQSBAfCXBCNEJlsBvERwv4BvuftWYKuZHQssDrOnEbjG\n3T8IL2SvIuhe+r/AU2E31tJweyJDSqOPiuxHeCE4391bzexIgkb+aHdPDFD+QeBBd389k/UUGQwF\ngch+mFk5wdF8PkG//v9y95y/B7REi4JARCTidLFYRCTiFAQiIhGnIBARiTgFgYhIxCkIREQi7v8D\nmZxNmZOlOwsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zwb0ncxooQBI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}