01_Introduction_to_PyTorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "01. Introduction to PyTorch.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMo+0ltjHO6lUuVTBnG8c/T",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f826069f97ec412fa2d85071c4669667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4f0c7a702a9048e28204a9eb24219ed4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5fce3507651d431b9c29f6b1ee8a8c8a",
              "IPY_MODEL_23878f753b8f4a8d9a34e148e95b8e7e"
            ]
          }
        },
        "4f0c7a702a9048e28204a9eb24219ed4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5fce3507651d431b9c29f6b1ee8a8c8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8933ddbf6bb048f4899d70d4a60aef9a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35df55b7e64349f28614142361d94ef5"
          }
        },
        "23878f753b8f4a8d9a34e148e95b8e7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4168a3c5786b489f9ddaf0be1cf4603a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:20&lt;00:00, 81009183.51it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_64840b767f894d1c85de6674b57f2d0e"
          }
        },
        "8933ddbf6bb048f4899d70d4a60aef9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35df55b7e64349f28614142361d94ef5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4168a3c5786b489f9ddaf0be1cf4603a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "64840b767f894d1c85de6674b57f2d0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7018f942c34b4089b7fc5b7af6a77c11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dab0843350124c26850caa474b7a6dba",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f2daa3a1e5454cdb8f9cb78d16d37cda",
              "IPY_MODEL_c38cec0f0a3849209bfdb2db39d7d57b"
            ]
          }
        },
        "dab0843350124c26850caa474b7a6dba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f2daa3a1e5454cdb8f9cb78d16d37cda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_910f2a756bf64b7fab50d42bfc3a588b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 46827520,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 46827520,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b6c3bc7dd7a47449beb81015386fa15"
          }
        },
        "c38cec0f0a3849209bfdb2db39d7d57b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_868c132e2e0746bbba5db5bddf844817",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 44.7M/44.7M [00:01&lt;00:00, 45.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_46e7c8992d6e4d418a8a55800be1891f"
          }
        },
        "910f2a756bf64b7fab50d42bfc3a588b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b6c3bc7dd7a47449beb81015386fa15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "868c132e2e0746bbba5db5bddf844817": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "46e7c8992d6e4d418a8a55800be1891f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tanaymukherjee/Deep-Learning-with-PyTorch/blob/master/01_Introduction_to_PyTorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6AKjW7v57s7u",
        "colab_type": "text"
      },
      "source": [
        "# Introduction to PyTorch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qtHN-y4S7x0n",
        "colab_type": "text"
      },
      "source": [
        "## Import packages and libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJ2YAki77WKU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch \n",
        "import torchvision\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "import torchvision.transforms as transforms"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xo491HGs8TNB",
        "colab_type": "text"
      },
      "source": [
        "## Table of Contents\n",
        "\n",
        "1. Basic autograd example 1\n",
        "2. Basic autograd example 2\n",
        "3. Loading data from numpy\n",
        "4. Input pipline\n",
        "5. Input pipline for custom dataset\n",
        "6. Pretrained model\n",
        "7. Save and load model "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sjB_gugN8M_m",
        "colab_type": "text"
      },
      "source": [
        "## 1. Basic autograd example 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O4fOkE3m8gUS",
        "colab_type": "text"
      },
      "source": [
        "### Create tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yAW82_2-7eOw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.tensor(1., requires_grad=True)\n",
        "w = torch.tensor(2., requires_grad=True)\n",
        "b = torch.tensor(3., requires_grad=True)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t4ynuoBI8kQ_",
        "colab_type": "text"
      },
      "source": [
        "### Build a computational graph"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cgx31uq08mJu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = w * x + b    # y = 2 * x + 3"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vrOxNVYv9IYR",
        "colab_type": "text"
      },
      "source": [
        "### Compute gradients"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z8BObTif89YO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y.backward()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gFnMZRMr9AFR",
        "colab_type": "text"
      },
      "source": [
        "### Print out the gradients"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fAK90pW19Baw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "5f1abb82-3a0d-49a1-b0de-926fb79b567a"
      },
      "source": [
        "print(x.grad)    # x.grad = 2 \n",
        "print(w.grad)    # w.grad = 1 \n",
        "print(b.grad)    # b.grad = 1 "
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(2.)\n",
            "tensor(1.)\n",
            "tensor(1.)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "smnUrjim9L5Q",
        "colab_type": "text"
      },
      "source": [
        "## 2. Basic autograd example 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4_wRRuyP9R25",
        "colab_type": "text"
      },
      "source": [
        "### Create tensors of shape (10, 3) and (10, 2)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWoUxV7d9Vl_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.randn(10, 3)\n",
        "y = torch.randn(10, 2)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mSSw5U829Uuw",
        "colab_type": "text"
      },
      "source": [
        "### Build a fully connected layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8LZzdwYS9ZMG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "a0a01a49-2f1e-4e89-ac59-efb5de20d393"
      },
      "source": [
        "linear = nn.Linear(3, 2)\n",
        "print ('w: ', linear.weight)\n",
        "print ('b: ', linear.bias)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "w:  Parameter containing:\n",
            "tensor([[-0.5365,  0.0821, -0.4799],\n",
            "        [-0.0822, -0.4136,  0.2844]], requires_grad=True)\n",
            "b:  Parameter containing:\n",
            "tensor([ 0.3979, -0.0299], requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tnisZ_W59VAX",
        "colab_type": "text"
      },
      "source": [
        "### Build loss function and optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTcpb2QK9eZ4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = torch.optim.SGD(linear.parameters(), lr=0.01)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eZa6l07O9StY",
        "colab_type": "text"
      },
      "source": [
        "### Forward pass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gGeFjjVe9lOW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred = linear(x)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZzbxqUh29oS_",
        "colab_type": "text"
      },
      "source": [
        "### Compute Loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b-dw2IXr9qXA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "37e43429-1624-43dd-9f58-fdc9a97377e1"
      },
      "source": [
        "loss = criterion(pred, y)\n",
        "print('loss: ', loss.item())"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss:  1.7813125848770142\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mjjHezoJ9vD_",
        "colab_type": "text"
      },
      "source": [
        "### Backward pass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqqajpah9wrA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss.backward()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g48tZnrc90yV",
        "colab_type": "text"
      },
      "source": [
        "### Print out the gradients"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "13QxAAzW92lA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "4b9e14ce-6a96-4f7b-be43-395cf96da9c0"
      },
      "source": [
        "print ('dL/dw: ', linear.weight.grad) \n",
        "print ('dL/db: ', linear.bias.grad)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dL/dw:  tensor([[-0.2016, -0.1495, -1.2145],\n",
            "        [ 0.2762, -0.2103,  0.6792]])\n",
            "dL/db:  tensor([-0.2657,  0.7464])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vdy66rRa95PP",
        "colab_type": "text"
      },
      "source": [
        "### 1-step gradient descent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xUvQv14k979A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer.step()\n",
        "\n",
        "# You can also perform gradient descent at the low level.\n",
        "# linear.weight.data.sub_(0.01 * linear.weight.grad.data)\n",
        "# linear.bias.data.sub_(0.01 * linear.bias.grad.data)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kHSGseja-Dk3",
        "colab_type": "text"
      },
      "source": [
        "### Print out the loss after 1-step gradient descent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TXG7wKKk9_BY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "74d916e4-5273-4786-cb98-2aa5fd47e00d"
      },
      "source": [
        "pred = linear(x)\n",
        "loss = criterion(pred, y)\n",
        "print('loss after 1 step optimization: ', loss.item())"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss after 1 step optimization:  1.7540979385375977\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l2gtQLiG-05w",
        "colab_type": "text"
      },
      "source": [
        "## 3. Loading data from numpy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0L0xgJmA-bh4",
        "colab_type": "text"
      },
      "source": [
        "### Create a numpy array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6LZg-pDf-hbI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = np.array([[1, 2], [3, 4]])"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oR8VPc06-i1G",
        "colab_type": "text"
      },
      "source": [
        "### Convert the numpy array to a torch tensor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GtHBw6cH-m_-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = torch.from_numpy(x)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZhXEczdf-pg3",
        "colab_type": "text"
      },
      "source": [
        "### Convert the torch tensor to a numpy array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hrQUUO1-oiu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "z = y.numpy()"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qzPoIjVc_RTp",
        "colab_type": "text"
      },
      "source": [
        "## 4. Input pipeline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6rhkilp_cFw",
        "colab_type": "text"
      },
      "source": [
        "### Download and construct CIFAR-10 dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z5NdtF1l_Sah",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "f826069f97ec412fa2d85071c4669667",
            "4f0c7a702a9048e28204a9eb24219ed4",
            "5fce3507651d431b9c29f6b1ee8a8c8a",
            "23878f753b8f4a8d9a34e148e95b8e7e",
            "8933ddbf6bb048f4899d70d4a60aef9a",
            "35df55b7e64349f28614142361d94ef5",
            "4168a3c5786b489f9ddaf0be1cf4603a",
            "64840b767f894d1c85de6674b57f2d0e"
          ]
        },
        "outputId": "9aa701d3-fe93-4e90-de9d-39bdc1e7e0a9"
      },
      "source": [
        "train_dataset = torchvision.datasets.CIFAR10(root='../../data/', train=True, transform=transforms.ToTensor(), download=True)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f826069f97ec412fa2d85071c4669667",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../../data/cifar-10-python.tar.gz to ../../data/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yDZK9EgL_jFO",
        "colab_type": "text"
      },
      "source": [
        "### Fetch one data pair (read data from disk)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qz9Eb7NX_ldR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "05b563d9-eac4-4546-ab93-0b07d10a2acc"
      },
      "source": [
        "image, label = train_dataset[0]\n",
        "print (image.size())\n",
        "print (label)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([3, 32, 32])\n",
            "6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OjzbcddC_o_e",
        "colab_type": "text"
      },
      "source": [
        "### Data loader (this provides queues and threads in a very simple way)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lPmWyqb8_qwG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=64, shuffle=True)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TbKdn-Av_6Zn",
        "colab_type": "text"
      },
      "source": [
        "### When iteration starts, queue and thread start to load data from files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kqxyPV5C_7eg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_iter = iter(train_loader)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1RujS2lm_984",
        "colab_type": "text"
      },
      "source": [
        "### Mini-batch images and labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cIjdsCwg_-35",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "images, labels = data_iter.next()"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r7sOwgHfADmA",
        "colab_type": "text"
      },
      "source": [
        "### Actual usage of the data loader is as below"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b9JhVgZNAEbL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for images, labels in train_loader:\n",
        "    # Training code should be written here.\n",
        "    pass"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pTK7QsKCALkn",
        "colab_type": "text"
      },
      "source": [
        "## 5. Input pipeline for custom dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iquno3ZhAN5P",
        "colab_type": "text"
      },
      "source": [
        "### You should build your custom dataset as below"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yes1gAIhAUdP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CustomDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self):\n",
        "        # TODO\n",
        "        # 1. Initialize file paths or a list of file names. \n",
        "        pass\n",
        "    def __getitem__(self, index):\n",
        "        # TODO\n",
        "        # 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).\n",
        "        # 2. Preprocess the data (e.g. torchvision.Transform).\n",
        "        # 3. Return a data pair (e.g. image and label).\n",
        "        pass\n",
        "    def __len__(self):\n",
        "        # You should change 0 to the total size of your dataset.\n",
        "        return 0 "
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k_NeGm7aARxh",
        "colab_type": "text"
      },
      "source": [
        "### You can then use the prebuilt data loader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6pj6EUwfAl2a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "custom_dataset = CustomDataset()\n",
        "train_loader = torch.utils.data.DataLoader(dataset=custom_dataset,\n",
        "                                           batch_size=64, \n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tDWNZxrYA42A",
        "colab_type": "text"
      },
      "source": [
        "## 6. Pretrained model "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h8IfA-RmA7e3",
        "colab_type": "text"
      },
      "source": [
        "### Download and load the pretrained ResNet-18"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6wXcQZq4BD0A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "7018f942c34b4089b7fc5b7af6a77c11",
            "dab0843350124c26850caa474b7a6dba",
            "f2daa3a1e5454cdb8f9cb78d16d37cda",
            "c38cec0f0a3849209bfdb2db39d7d57b",
            "910f2a756bf64b7fab50d42bfc3a588b",
            "5b6c3bc7dd7a47449beb81015386fa15",
            "868c132e2e0746bbba5db5bddf844817",
            "46e7c8992d6e4d418a8a55800be1891f"
          ]
        },
        "outputId": "6e0d564c-dccb-4ecd-bd76-e9229878567e"
      },
      "source": [
        "resnet = torchvision.models.resnet18(pretrained=True)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /root/.cache/torch/checkpoints/resnet18-5c106cde.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7018f942c34b4089b7fc5b7af6a77c11",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=46827520.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ZU2dNLJBHVl",
        "colab_type": "text"
      },
      "source": [
        "### If you want to finetune only the top layer of the model, set as below"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vH_aIzxqBLS3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for param in resnet.parameters():\n",
        "    param.requires_grad = False"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TGiV_kYTBNXw",
        "colab_type": "text"
      },
      "source": [
        "### Replace the top layer for finetuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltOpMVV8BPD-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "resnet.fc = nn.Linear(resnet.fc.in_features, 100)  # 100 is an example."
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H6mSI_bmBR54",
        "colab_type": "text"
      },
      "source": [
        "### Forward pass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9pk-bjOuBTZO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "dc06c071-007a-48b3-d02c-e33d367fc7ff"
      },
      "source": [
        "images = torch.randn(64, 3, 224, 224)\n",
        "outputs = resnet(images)\n",
        "print (outputs.size())     # (64, 100)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([64, 100])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hvpUo9BLBalt",
        "colab_type": "text"
      },
      "source": [
        "## 7. Save and load the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eIPOQF5MBfkm",
        "colab_type": "text"
      },
      "source": [
        "### Save and load the entire model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q70R2t1FBX16",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.save(resnet, 'model.ckpt')\n",
        "model = torch.load('model.ckpt')"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cJIMDixSBh71",
        "colab_type": "text"
      },
      "source": [
        "### Save and load only the model parameters (recommended)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J97BlQOyBjBQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "fa389783-e585-42d1-c9e5-ba17144f30b2"
      },
      "source": [
        "torch.save(resnet.state_dict(), 'params.ckpt')\n",
        "resnet.load_state_dict(torch.load('params.ckpt'))"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    }
  ]
}