CarRacing_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CarRacing_training.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/markub3327/ImitationLearning/blob/master/CarRacing_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IsNnfkrOoag1",
        "colab_type": "text"
      },
      "source": [
        "# Prepare & install required libraries\n",
        "\n",
        "When we would like to run it in Google Colab, we need to download dependencies before running."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JJrAfFa78qfa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/markub3327/ImitationLearning.git\n",
        "%cd ImitationLearning/\n",
        "%ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Alj5vt-tAFaN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip3 install wandb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CRDLPIQGRr8H",
        "colab_type": "text"
      },
      "source": [
        "# Importing needed libaries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WMJX-a3bRssR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import wandb\n",
        "\n",
        "from nets.agent import Agent\n",
        "from wandb.keras import WandbCallback "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gy1BSoL9SrbK",
        "colab_type": "text"
      },
      "source": [
        "# Dataset\n",
        "\n",
        "This class prepare datasets for training:\n",
        "\n",
        "1.   prepare timesteps (4 frames),\n",
        "2.   shuffle data to prevent not fitting the model,\n",
        "\n",
        "This is like a pipeline for the training process, where are datasets loaded from disk and preprocessed before the training starts.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dgksts66SrDr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Dataset:\n",
        "    def __init__(self, path='data/', timesteps=4):\n",
        "        # the list of datasets\n",
        "        self.dat = []\n",
        "        self.ptr = 0\n",
        "        self.timesteps = timesteps\n",
        "\n",
        "        # scan datasets\n",
        "        with os.scandir(path) as entries:\n",
        "            for entry in entries:\n",
        "                self.dat.append(entry.path)\n",
        "    \n",
        "    def _read_dataset(self, path):        \n",
        "        with np.load(path) as data:\n",
        "            f = data['frames']\n",
        "            a = data['actions']\n",
        "            print(f.shape)\n",
        "            print(a.shape)\n",
        "\n",
        "            # make timesteps\n",
        "            f, a = self._make_timesteps(f, a)\n",
        "            print(f.shape)\n",
        "            print(a.shape)\n",
        "\n",
        "            # shuffle dataset after loading from file\n",
        "            f, a = self._shuffle_dataset(f, a)\n",
        "        \n",
        "        print(f\"Loaded dataset from: {path}\")\n",
        "\n",
        "        return f, a\n",
        "\n",
        "    def _make_timesteps(self, f_dat, a_dat):\n",
        "        # generate random indexes\n",
        "        rand_idxs = np.arange(self.timesteps + 1, f_dat.shape[0], dtype=np.int)\n",
        "        print(rand_idxs)\n",
        "        print(rand_idxs.shape)\n",
        "\n",
        "        states = np.zeros((rand_idxs.shape[0], self.timesteps) + f_dat.shape[1:], dtype=np.uint8)\n",
        "            \n",
        "        for i, idx in enumerate(rand_idxs):\n",
        "            states[i] = f_dat[idx-self.timesteps-1:idx-1]\n",
        "\n",
        "        return states, a_dat[rand_idxs]\n",
        "        \n",
        "    def _shuffle_dataset(self, f_dat, a_dat):\n",
        "        idx = np.arange(0, f_dat.shape[0], dtype=np.int)\n",
        "        print(idx)\n",
        "        np.random.shuffle(idx)\n",
        "        print(idx)\n",
        "\n",
        "        return f_dat[idx], a_dat[idx]\n",
        "\n",
        "    def __iter__(self):\n",
        "        self.ptr = 0\n",
        "        return self\n",
        "\n",
        "    def __next__(self):\n",
        "        if self.ptr == len(self.dat):\n",
        "           raise StopIteration\n",
        "        s, a = self._read_dataset(self.dat[self.ptr])\n",
        "        self.ptr = self.ptr + 1\n",
        "        return s, a"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7dDNZgSqR6fL",
        "colab_type": "text"
      },
      "source": [
        "# Training\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SGlLGhMl5hxU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "wandb.init(project=\"car_racing\")\n",
        "\n",
        "# create network\n",
        "agent = Agent()\n",
        "agent.create((num_frames, 96, 96, 3))\n",
        "\n",
        "# save model's plot\n",
        "agent.save_plot()\n",
        "    \n",
        "# load datasets from folder\n",
        "dataset = Dataset(timesteps=num_frames)\n",
        "\n",
        "# training loop\n",
        "a.top_only(True)\n",
        "for t in range(1000):\n",
        "    # take every dataset from folder\n",
        "    for s, a in dataset:\n",
        "        print('Run training...')\n",
        "        print('|-----------------------------------------------|')\n",
        "        agent.train(s, a, epochs=1, callbacks=[WandbCallback()])\n",
        "\n",
        "        print('+-----------------------------------------------+')\n",
        "\n",
        "# fine-tuning loop\n",
        "a.top_only(False)\n",
        "for t in range(1000):\n",
        "    # take every dataset from folder\n",
        "    for s, a in dataset:        \n",
        "        print('Run fine-tuning...')\n",
        "        print('|-----------------------------------------------|')\n",
        "        agent.train(s, a, epochs=1, callbacks=[WandbCallback()])\n",
        "\n",
        "        print('+-----------------------------------------------+')\n",
        "\n",
        "# save model\n",
        "agent.save('/content/drive/My Drive/model_po.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mZkueXfZSGH1",
        "colab_type": "text"
      },
      "source": [
        "# Results\n",
        "\n",
        "![chart_03.png](https://github.com/markub3327/OpenAI-CarRacing-RNN/blob/master/chart_03.png?raw=1)![chart_01.png](https://github.com/markub3327/OpenAI-CarRacing-RNN/blob/master/chart_01.png?raw=1)![chart_02.png](https://github.com/markub3327/OpenAI-CarRacing-RNN/blob/master/chart_02.png?raw=1)"
      ]
    }
  ]
}