deep_learning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/npcrites/DS_002/blob/main/deep_learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Clone your code into a directory called `byscratch`\n",
        "# \n",
        "! rm -Rf byscratch ;\n",
        "!git clone https://github.com/douglasgoodwin/byscratch.git byscratch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d1txXq45lJaP",
        "outputId": "cfd82f85-92d0-40a0-802a-43752a34a924"
      },
      "id": "d1txXq45lJaP",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'byscratch'...\n",
            "remote: Enumerating objects: 46, done.\u001b[K\n",
            "remote: Counting objects: 100% (46/46), done.\u001b[K\n",
            "remote: Compressing objects: 100% (36/36), done.\u001b[K\n",
            "remote: Total 46 (delta 16), reused 28 (delta 7), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (46/46), done.\n",
            "Checking out files: 100% (20/20), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install mnist"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ejz-yJ4elVB-",
        "outputId": "5ec209f9-6d1c-4c65-fdc3-18fc21e6dfa7"
      },
      "id": "ejz-yJ4elVB-",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: mnist in /usr/local/lib/python3.7/dist-packages (0.2.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from mnist) (1.21.5)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "357266c5-c228-4ed9-b0cc-7af9ef8254da",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "357266c5-c228-4ed9-b0cc-7af9ef8254da",
        "outputId": "51c07c97-ca30-4c0f-870e-2ffd7ae89f36"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "from byscratch.deep_learning import *"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "68d05542-7c47-4f54-ab3c-b34847f8c12a",
      "metadata": {
        "id": "68d05542-7c47-4f54-ab3c-b34847f8c12a"
      },
      "outputs": [],
      "source": [
        "# python imports\n",
        "from typing import List, Callable, Iterable, Tuple\n",
        "import random\n",
        "import math\n",
        "import json\n",
        "import operator\n",
        "\n",
        "# local imports\n",
        "from byscratch.neural_networks import sigmoid\n",
        "from byscratch.probability import inverse_normal_cdf\n",
        "from byscratch.linear_algebra import dot\n",
        "from byscratch.linear_algebra import squared_distance\n",
        "from byscratch.neural_networks import binary_encode, fizz_buzz_encode, argmax\n",
        "\n",
        "# 3rd party imports\n",
        "import matplotlib.pyplot as plt\n",
        "import tqdm\n",
        "import mnist\n",
        "\n",
        "# not a byscratch\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f9a65cc0-83e6-4e11-ae19-1f26b9dab28d",
      "metadata": {
        "id": "f9a65cc0-83e6-4e11-ae19-1f26b9dab28d"
      },
      "source": [
        "# Deep Neural Network Keras way\n",
        "\n",
        "https://www.kaggle.com/code/poonaml/deep-neural-network-keras-way"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a3e3fd63-1bac-458a-ac2c-0a515a1f3898",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a3e3fd63-1bac-458a-ac2c-0a515a1f3898",
        "outputId": "6259a8e8-7f38-443a-b95f-01da7a570bc6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "adult_9ksample.csv\n",
            "adult_original.csv\n",
            "adult_stratified_income.csv\n",
            "bike_sharing_daily.csv\n",
            "bike_sharing_hourly.csv\n",
            "mnist_test.csv\n",
            "mnist_train.csv\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense , Dropout , Lambda, Flatten\n",
        "# from keras.optimizers import Adam ,RMSprop\n",
        "from tensorflow.keras.optimizers import Adam,RMSprop\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras import  backend as K\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "# Input data files are available in the \"byscratch/data/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"./byscratch/data\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "110339a0-439c-4465-933c-c8034ce2e630",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317
        },
        "id": "110339a0-439c-4465-933c-c8034ce2e630",
        "outputId": "ef715b6e-d6f2-43b0-e76f-5266e589c5a0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(42000, 785)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
              "0      1       0       0       0       0       0       0       0       0   \n",
              "1      0       0       0       0       0       0       0       0       0   \n",
              "2      1       0       0       0       0       0       0       0       0   \n",
              "3      4       0       0       0       0       0       0       0       0   \n",
              "4      0       0       0       0       0       0       0       0       0   \n",
              "\n",
              "   pixel8  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
              "0       0  ...         0         0         0         0         0         0   \n",
              "1       0  ...         0         0         0         0         0         0   \n",
              "2       0  ...         0         0         0         0         0         0   \n",
              "3       0  ...         0         0         0         0         0         0   \n",
              "4       0  ...         0         0         0         0         0         0   \n",
              "\n",
              "   pixel780  pixel781  pixel782  pixel783  \n",
              "0         0         0         0         0  \n",
              "1         0         0         0         0  \n",
              "2         0         0         0         0  \n",
              "3         0         0         0         0  \n",
              "4         0         0         0         0  \n",
              "\n",
              "[5 rows x 785 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bb84a8fa-9d90-4e6e-884d-417d5859af31\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>pixel0</th>\n",
              "      <th>pixel1</th>\n",
              "      <th>pixel2</th>\n",
              "      <th>pixel3</th>\n",
              "      <th>pixel4</th>\n",
              "      <th>pixel5</th>\n",
              "      <th>pixel6</th>\n",
              "      <th>pixel7</th>\n",
              "      <th>pixel8</th>\n",
              "      <th>...</th>\n",
              "      <th>pixel774</th>\n",
              "      <th>pixel775</th>\n",
              "      <th>pixel776</th>\n",
              "      <th>pixel777</th>\n",
              "      <th>pixel778</th>\n",
              "      <th>pixel779</th>\n",
              "      <th>pixel780</th>\n",
              "      <th>pixel781</th>\n",
              "      <th>pixel782</th>\n",
              "      <th>pixel783</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 785 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bb84a8fa-9d90-4e6e-884d-417d5859af31')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bb84a8fa-9d90-4e6e-884d-417d5859af31 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bb84a8fa-9d90-4e6e-884d-417d5859af31');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "# create the training & test sets, skipping the header row with [1:]\n",
        "train = pd.read_csv(\"byscratch/data/mnist_train.csv\")\n",
        "print(train.shape)\n",
        "train.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b20e4081-f24f-4134-8714-e50801ed07b2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317
        },
        "id": "b20e4081-f24f-4134-8714-e50801ed07b2",
        "outputId": "b6fbb907-da9d-4eb5-83c3-3c18cda3c64b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(28000, 784)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n",
              "0       0       0       0       0       0       0       0       0       0   \n",
              "1       0       0       0       0       0       0       0       0       0   \n",
              "2       0       0       0       0       0       0       0       0       0   \n",
              "3       0       0       0       0       0       0       0       0       0   \n",
              "4       0       0       0       0       0       0       0       0       0   \n",
              "\n",
              "   pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
              "0       0  ...         0         0         0         0         0         0   \n",
              "1       0  ...         0         0         0         0         0         0   \n",
              "2       0  ...         0         0         0         0         0         0   \n",
              "3       0  ...         0         0         0         0         0         0   \n",
              "4       0  ...         0         0         0         0         0         0   \n",
              "\n",
              "   pixel780  pixel781  pixel782  pixel783  \n",
              "0         0         0         0         0  \n",
              "1         0         0         0         0  \n",
              "2         0         0         0         0  \n",
              "3         0         0         0         0  \n",
              "4         0         0         0         0  \n",
              "\n",
              "[5 rows x 784 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8fabe6b7-bce4-4478-9980-970f1290a8ce\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pixel0</th>\n",
              "      <th>pixel1</th>\n",
              "      <th>pixel2</th>\n",
              "      <th>pixel3</th>\n",
              "      <th>pixel4</th>\n",
              "      <th>pixel5</th>\n",
              "      <th>pixel6</th>\n",
              "      <th>pixel7</th>\n",
              "      <th>pixel8</th>\n",
              "      <th>pixel9</th>\n",
              "      <th>...</th>\n",
              "      <th>pixel774</th>\n",
              "      <th>pixel775</th>\n",
              "      <th>pixel776</th>\n",
              "      <th>pixel777</th>\n",
              "      <th>pixel778</th>\n",
              "      <th>pixel779</th>\n",
              "      <th>pixel780</th>\n",
              "      <th>pixel781</th>\n",
              "      <th>pixel782</th>\n",
              "      <th>pixel783</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 784 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8fabe6b7-bce4-4478-9980-970f1290a8ce')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8fabe6b7-bce4-4478-9980-970f1290a8ce button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8fabe6b7-bce4-4478-9980-970f1290a8ce');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "test= pd.read_csv(\"byscratch/data/mnist_test.csv\")\n",
        "print(test.shape)\n",
        "test.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b85f91e-d345-4fb6-afc4-c601dea24f40",
      "metadata": {
        "id": "8b85f91e-d345-4fb6-afc4-c601dea24f40"
      },
      "outputs": [],
      "source": [
        "X_train = (train.iloc[:,1:].values).astype('float32') # all pixel values\n",
        "y_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits\n",
        "X_test = test.values.astype('float32')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "deb82679-db66-4eac-96c2-88fd8b3e66c9",
      "metadata": {
        "id": "deb82679-db66-4eac-96c2-88fd8b3e66c9"
      },
      "source": [
        "# Data Visualization\n",
        "Lets look at 3 images from data set with their labels."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e0ebd8fa-2593-4590-b77c-38f7403ff514",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        },
        "id": "e0ebd8fa-2593-4590-b77c-38f7403ff514",
        "outputId": "f2e376c9-2daf-4d18-f734-bb74b0e2990b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 3 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAABvCAYAAACD1ClOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2daWzk53nYf+/c98EZcjjDY5dLLkUu5dVqtbuyaq0OG7YC22iCNIhjF6kLuAhQNEAD9EOMfOmXFEm/BC0aNIWDBFbQoFasBLABGzYSxZYV2Za10t5acQ/e15Ccg3Pfbz8s/6/IvUkOZ3i8P4AgZzg7/2fn4fv83/c5hZQSjUaj0Tw5pnYLoNFoNPsNbTg1Go1mi2jDqdFoNFtEG06NRqPZItpwajQazRbRhlOj0Wi2iDacGo1Gs0UOvOEUQuTu+aoLIf5Xu+XS7BwhxP8VQiwKITJCiJtCiP/Qbpk0O0cI8VMhRGnDmh1vt0z3cuANp5TSY3wB3UAR+G6bxdI0hz8BjkopfcC/Bv5YCPFcm2XSNIff37B2n2q3MPdy4A3nPfwbYBl4p92CaHaOlPK6lLJsPFz/GmyjSJpDwmEznF8H/kbqOtMDgxDifwshCsDHwCLwwzaLpGkOfyKEWBVCvCuEeKXdwtyLOCw2RAhxBJgAhqSUk+2WR9M8hBBm4AXgFeC/Symr7ZVIsxOEEM8DHwEV4HeAPwdOSSnvtFWwDRymHefvAv+ijebBQ0pZl1L+C9AL/Md2y6PZGVLK96SUWSllWUr5OvAu8MV2y7WRw2Q4/x3weruF0OwqFrSP8yAiAdFuITZyKAynEOJfAT3oaPqBQQjRJYT4HSGERwhhFkK8BnwVeKvdsmm2jxAiIIR4TQjhEEJYhBD/FngJ+FG7ZduIpd0CtIivA/8gpcy2WxBN05DcPZb/H+5uAKaBP5BSfr+tUml2ihX4Y2AEqHM36PcbUsqbbZXqHg5NcEij0WiaxaE4qms0Gk0z0YZTo9FotsiODKcQ4teEEONCiNtCiG82SyhNe9F6Pbho3TaHbfs415OObwKfB+aA94GvSik/ap54mlaj9Xpw0bptHjvZcZ4DbkspJ6SUFeA7wK83RyxNG9F6Pbho3TaJnaQj9QCzGx7PAc8/6h+Ew2F59OjRHVxyf/PBBx+sSik72y3HY9B63SL7RK+wRd1qvT5cr7uexymE+D3g9wD6+/u5cOHCbl9yzyKEmG63DM1C6/UTtF4PJo/S606O6vNA34bHvevPbUJK+S0p5Rkp5ZnOzv1wUz70aL0eXB6rW63XJ2MnhvN94LgQYkAIYeNuFxNdtbH/0Xo9uGjdNoltH9WllDUhxO8DPwbMwF9LKa83TTJNW9B6Pbho3TaPHfk4pZQ/RDeOPXBovR5ctG6bg64c0mg0mi2iDadGo9FsEW04NRqNZotow6nRaDRb5LA0MtZoNPsIKSWNRuOBz2/8vhEh7k7XMJlMmx4b35uJNpwajWbPUCqVyGazZLNZbt68ST6fp9FoIKWkUqlQLpfJ5XIsLi5SqVSAu4YxEAjg9XoJBoP09PTgcrmIxWI4HA4CgQAOh6OpcmrDqdFo9gyVSoVUKkU8HufnP/85q6ur1Ot1arUaxWKRXC7HysoKly5dolQqAXcNZ19fH93d3fT39/PMM88QCoVoNBrKaGrDqTnwGAsln8+ztLREuVwmk8lQqVTI5XLk8/lNrzeZTJhMJgKBAE6nE7vdjt1ux2Kx4HQ61dENUDsTIQRmsxnYnaOc5smo1+s0Gg0WFhZYXl4mHo8zMTFBKpXi2rVrZLNZ6vU6UkpKpRKlUolcLnffMb5QKLC6ugrc1afH42F5eRmv18vQ0BCdnZ3EYjH6+/ubom9tODV7jkqlQqFQYG5ujp/+9Kckk0lu3bpFOp1mamqKhYUF9VohBEIIbDYbJ06coLu7m46ODsLhMG63m66uLqxWq3rt6OgoAwMDWCwW7Hb7JgOqaS3G8btSqfDBBx/wi1/8gomJCS5cuKCO7BsNpJRSfdXr9U3vlUwmSafTzM3NceXKFcxms7qBnj59mr6+Pj7/+c/T29vbFH3vecNZrVYpFovU63WKxSJSSpxOJ1arVe00DIxFIIRQfhHj61HU63XK5bI6DtRqNSwWCxaLBYfDgc/nw2QyqffW7Bzjj9/Qq/HZVyoVstks6XSapaUlJicnWVtbY2lpiVwuRzabVUc0AyEEtVpN7TiKxSKFQgGHw0Eul1MLxfhbyeVy2Gw2XC4XNpsNn8+H1WrF4/Fgt9tb+0EcQqSU1Go1KpUKs7OzrK2tMTExwdzcHMvLy2QyGWq1GjabDdh8IpBSPvEarNVqSClJJBJYLBYWFhaYn5/H5XIRCASwWLZv/va84UwkEoyPj7O2tsatW7eoVCqMjIwQiUSw2+04nU71WrPZTCAQwGazUSwWKZfL1Ot1qtXqI41nNptlamqKbDbLRx99RDqdJhgMEggEGBoa4jOf+UxTPmzNJxgGMpvNMj4+Tjqd5saNGywtLTE/P8/U1JR6TaPRUDq02WyEw+FN72XsPubm5piZmcFkMqkbq8Vi2bTQHA6HMpbd3d0Eg0Gee+45wuEwZ86c4ciRI5hMJn2D3EWq1SrpdJrl5WX+8i//kmvXrrG4uMjq6irlcplisYjL5WJwcBCPx6M2LBvX8OP0Uy6XWVxcpFwuMzU1xeTkJLVajWw2y8DAAK+99hper3fbm6E9bwUMZ3EymWRmZoZisYjH41E7T6fTqf7jZrOZRqOB3W6nUCionYwRfXsYxhZ/bW2NO3fukEgk6OzsJBgM4vP5KJfL2Gy2x+5cNU9OpVJhbW2NdDrNwsICyWRS7Trm5+eZnr7bCtEwYobP0u124/V61ftIKalWqzQaDeLxOGtra1itViwWy30nEoBGo0Gj0cDv91MoFOjo6KCzs5NSqcTQ0BBdXV3YbDa129E0n2q1SiaTIZFIMDk5yc2bN8lkMmSzWYQQWK1W7HY7XV1d+Hw+zGYzZrN5S4azUCgol08ymaRUKpFMJpmdncXj8Tx2M/U49rzhLBaLLC8vMz8/z69+9StSqRRXr17F6XRisVg2OfjNZjOhUAi73U4+n6dQKNBoNKhUKvdt8Tc+LpfLZLNZZaQrlQorKyu43W4CgcAD88k0O2N8fJw333yTRCLB7du3yeVyJJNJCoUCHo+HsbExPB4P0WgUl8vFwMAAfr+fjo4OAoGAep9yuczS0hLpdJrvfe97XL58mbGxMZ599lk8Hg9dXV3qlCClZHV1lXQ6TSaTYWlpiWKxyFtvvYXFYmFtbY2RkRFGR0d55pln9K6zyRg3rdnZWd58803m5+cZHx8nlUpRrVYBCIfDHDt2jN7eXn7rt36LSCSiboD3GrpH6cfY1RaLRcbHx1laWqJUKrG6ukoikaBcLlOpVJQLbqvsecNZrVbJZrMkk0mmpqaIx+P3GUC4+yFaLBY6OjpwOBzk8/lNO86NhvLeO9eDFJLP53E6nSSTSW04d4Hl5WXeffddVlZWmJ6e3uS3HBoaore3l46ODoaGhggEApw6dYquri51EjAoFArcvn2b5eVlfv7zn2MymYhGo5w7d45gMMixY8dUcEhKyezsrPKdptNp0uk0H3/8MbVaja6uLiqVCoFAgJMnT2rD2WQMv3YymeT9999nbm6OeDyusiSEELjdbvr6+hgaGuLTn/70tqPgjUaDcrlMqVQiHA4zNTXFrVu3uH79OoVCgWq1qqL12+GxhlMI8dfAl4FlKeXT6891AG8AR4Ep4LellKltSfAESCnxer288MILlEollXaSy+VYW1tTrzOZTCpwVKlU1IfzMMNnHA9yuRwLCwtUKhWlpEgkQk9PD0ePHsXv9+N2uw9U9LXdeg2FQpw+fZp0Os3AwAC1Wo3Ozk48Hg+xWIwjR47gcrkIh8M4nU5isRgej+e+fDwjMttoNPjUpz6F1Wrl05/+NCdPnsTtdtPR0bFJb8YRMBaL0dPTw8LCAplMhtXVVVKpFBMTE5w4cYJ6va4i9vuNduv2YczMzDA+Ps6NGzeYmZlRPk0hBJFIhFAoxOjoKJ/97GeJRqObXDJbxdhIORwOtYZjsZiKjwQCAex2+32unCflSXac3wb+HPibDc99E3hLSvmn67OZvwn84bYkeEJ8Ph+f/exnsdvtDA4O0tnZqYIID/J9PMmdxPClLSwsEI/H1b8zm83EYjFOnTrF0NCQ2sUeML5NG/Xa1dXFCy+8QLFYVL6tsbExYrEYwWCQUCi0yXA9zIg1Gg1KpRL1ep3Tp09z4sQJdVS/t/QOIBqNqqhuqVTi448/5r333qNcLrOyskImk+HcuXPU6/VtH+P2AN9mD6zZe7lz5w4//OEPmZ6eZmJigmw2i5QSk8lELBZjdHSUM2fO8KUvfQmPx4PP59v2jcvwlVqtVo4fP76pVFMIsW2DafBYwyml/JkQ4ug9T/868Mr6z68DP2WXlGD4JaSUuFwuFSDw+/3Kwbud7Xaj0SCXy23akQohcDgc2O12otEog4ODdHV17fhD3ou0W6/GLtKIogoh6OzsxOfzKf/1k2CxWNTuwe12U61WCQQCmyLjUkrltjES6LPZLCsrK0xNTZFOp6lUKhw5ckS5A/ZzZL3dun0YRqlkIpGgWq2qLBin08ng4CAjIyP09fXhcrlUjm0z2I2Tw3Z9nBEp5eL6z0tApEny3EcymeTSpUvY7XZCoRA+n49gMEgsFiMSifDUU09t+72z2SzvvPMOhUIBKSVWq5VwOIzH4+Hs2bN8+ctfxuPxKB/ZIaBleu3s7MTv92+68Vmt1i2nhzgcDoaHhzfl7Vqt1k3vUavViMfjKt3s9u3bKtE6m80Sj8exWq0899xznD9/nuPHj6uo/AGiZbp9GPF4nAsXLpDP56lWq9jtdp5++mmi0Shf/OIXOX/+vArI7vUb146DQ1JKKYR46Jbv3nGjW6Ver1MqlZQvq1arKf+FUf2xTbkxm81Uq1UV0TOZTPh8Pjo6OggGg/j9/qbe+fYTu61Xs9m8KQd3uxhVQxsx8j9rtRrlcplyuazSzWZnZ5mfn2dpaYmVlRUajQYdHR0qgh+JRFRJ5kHlUbrdqV4fRLVapVarUSgUKBQKlEolGo0GZrOZcDhMb28vXV1ddHR0qJvnPfJu+r5B1rbpabuGMy6EiEopF4UQUWD5YS+UUn4L+BbAmTNntp04Va1WN1WG7AQjaJTL5UgkEirJ2uVy8fzzz3P8+HFGR0fxer0HbdfxOFqu12ZiuF1u3rzJtWvXWFlZ4cqVK2SzWZaWltSiLZfLmM1muru7iUaj/OZv/iY9PT3Kd+5wOA6i4Xwi3TZbr/V6XQWCZmdnVX6l4Xr73Oc+x9mzZ4nFYrhcrgd+7saGyThVGBjB4Hb4obdrOL8PfB340/Xv32uaRPdgHL+MIIARCNgJjUZDpSmVSiUVTbfZbMRiMQYGBtTd75DRMr0+iO36q41/Y+xsEomESqb/1a9+pZKtS6USNpsNq9VKMBikq6uL7u5unn/+eY4ePYrD4TjIOm+LbqWUrK2tsby8zNramjKAADabjd7eXo4fP47D4dhkADeue+PUYOSBwid524YvvNVH+ydJR/p/3HUqh4UQc8B/5e6H/3dCiG8A08BvN1uwXC5HoVBgeXmZ5eVlHA4HXq+XcDi8owh3o9Egk8mo9lQrKyuYzWZVNTI2NsaJEycIhUJN/N/sPdql1wdhdEOanJxUp4qHpZAZJZPVapVSqaS6KFWrVebn50mn00xPT3Pnzh3q9Tper5eOjg5efPFFvF4vgUAAn8+Hx+MhHA4TDAaJRqPY7fb9GkG/j72iW8PNdvXqVS5evMjt27dVf4J1ObHZbDgcDmUADb3m83lVUTQ+Pk48HqdarVIulwFUqtHo6KhKYzp27FhTIuZPwpNE1b/6kF99rsmybKJQKJBIJEgmk6RSKfx+Px6PR9WibxcppapSMd47GAzS39+vEm+HhoYO8s4DaJ9eHyCHarIyNTXF7du3H9j9Bu4uNK/Xi9vtplgskslkKJfLJBIJCoUCV69eZX5+nkwmQzqdpqOjg0996lN0dnbyuc99jr6+PmKxGJ2dndjtdjwez0E8ku8J3Rq7xUqlwq1bt3jvvfdYXFy8T69Wq3XTejZ8oaurq1y8eJHFxUXefvttbt++TbFYJJ/Pq14EXq+XV155hb6+PtxuN0eOHGnZzW/PVw4Z7aFsNltT/sjr9To3b95kfHycqakpVdve3d1NJBLB5XJhNpsPm2+zbVSrVVZWVshms9y+fZuPP/5YVXw8CKPfZqVSUSlGtVqNer2udpPGsS0QCDA8PIzf72dwcJBQKITf71c7nINoNPcKGzsgGQUG+XweKSV2u13l6q6urnLr1i3i8bhyp2QyGdbW1rhy5QqpVEqVSNZqNfXexhF+ZmaGfD6v3GsdHR309fXt+vrdF4bT5XLd15B2u1QqFd5++21+9KMfsbKyQr1ex+l0MjQ0RF9fH36/Xzd4aCHlcpnJyUlVMnnx4kXW1tZIpR5e1LKxTNbwV7pcLl599VVGRkbo6enhyJEjeL1eotHopqYfRiRWG83dxciCKRaLLC0tMT09rXabXq+XkZERQqEQU1NT5HI53nnnHS5evKgaFVerVfL5vLopbiyPNIxyPp/n0qVLatdaKBQYGxsjGo3u+hres4bT8GGEw2GGh4dV+Z3f79/RMVpKSaFQIJVKqfxNw8ms+222HiMyaqQDGQ040un0I3sEVKtVcrmcemwED41djcvlUl+6FWB7MPyNhh/T8FEavVez2Sxzc3NkMhmVGG90NDKMpGEHDB02Gg3Vw9VoN9hoNEilUiwsLBCNRqlUKqqj0m6xZ/+ifD4fbrebl19+mcHBQYQQ+Hw+bDYbHR0d237fRqOhmuTq5h3tx+l0MjIyQq1WY3BwcFNlz6OYnZ3l0qVLpNNprl+/TqlUYmJiYlOwz2hkq2k9JpMJh8OBx+Ohr6+P48ePqy5nuVyOjz76CJPJxOXLl7FYLORyOZXf2Wg0cDgcKkXJ6GFgkEqluH79+qaeuzdv3iSZTGI2m3n11Vep1+uql+dusGcNp5HgHgqFcDgcm7ogbXcbbkRvjTSkjXXQxh1K7zhbi8lkwu12A3eNqLGbMHaOD8PhcJBKpXC73SwuLqpc3Hw+rzrFm81m8vm88pEflKj5fmBjupCRDZPP51W3/kwms+n1RuK74VZxOp2EQiHlbvH7/eq1RuFLtVpVOaFGQCmdTlMul3fcb/Nx7FnDaWDUpt/bPm6rlMtlFhYWWF1dVbsZo+Df6XRy9OhRent7m1LNotkehrvEZDI9tiLMKMEtl8u88sorm9rLxeNx/uIv/gKfz8fAwAChUEjNm9nrpXwHCUOPZ86cwefz8c///M9MTU0po2Zsgkwmk1p/vb29qrOVkSbmdDo3bZauXLnC7du3AVRVoZGPnU6nWVlZQUqJx+PZNTfNnjecho9kp9RqNZLJJCsrK/dFbG02G6FQiFAopANDbcQI/hmZFI/C7XYTiURU6lKpVMLpdKrUl5/97Gd4PB5mZmbo7e3lueeeIxaL6cBQCzE6FB05cgSn08n4+LjqOGUYTovFgtVqpbu7m6GhIZ5++mm+8IUv4PF4CIVCD4xn1Go1fD4fa2tr6hRhTHkwRgi7XK5ddcXtecPZLGq1GmtraySTSfUhB4NBOjs7OXr0qEqGPuj5m+3G6MxvjEcwjmc7aR9m3FyPHTuGz+fD6/XS3d2t+mtOTk7ygx/8gKtXrzI2Nsbg4KAa1qbZXYQQdHR0YLPZePHFF1WE3DBqdrsdq9Wqur5v7JX5KNfKvYnuhivGCDS6XK5dTUk6VIZzY06Y0cZsbGyM4eFhotHoga8WajdSSjKZzKaxJIZvayf+R8NwPvXUUwwPD/PUU09x7tw5Ll68yKVLl0ilUqysrOByufja175GMBjE4/Fsmlel2R1MJhOdnZ2Ew2F8Ph+nTp1SkXFDb2azmWAwqJqrPM7gbdSZcYIwIvderxev17urgSE4RIazXq+rNBejbMvlctHV1UUwGNSBg13GSFSfnp7mxo0bxGIxxsbGgCdrOv0kGIvI4XAQCASIRCKMjo6SSCSU3icnJ3n//ffp6elR5ZuHtQNWKzEG7vl8PpXAbhzVDV/o49ag0cksn88rn6aRG2oYTbfbjdvt3nWdHhrDWS6XVefpTCaDlJJIJMKzzz5Lf3+/9m3uIo1GQ7UU+/GPf8wbb7zByy+/rG5azXbiu91uNULlG9/4BouLi7z55pvcuXOHt956i5/85Ce8/PLLBAIB5a7R+t99DL08aGLDkxyr8/k8iUSCpaUlUqmUahoihCAYDKrgUjgcxu1266N6MzAWr9H1HVB3wIe1s9I0ByPqaaQK5fN5yuWyml7Y7M/eeF+Xy0UkEkEIQVdXF+l0WvUnMBq8SCkJBoPacLYAI0Vpu+TzeeLxOKlUSiXSG0bY6XQSCARwu90taUJ9aAyn0c/TGAlrTNQzfJv6qL57VCoV7ty5w/LyMna7nVOnTjE6OkpPT8+udth3uVwcPXqUSCSC1WolkUjwgx/8gLfffpvZ2Vm+853vcOTIEb72ta8Ri8UOYtf3A8WVK1f47ne/y8LCgjKe9Xodi8VCT08Pp0+fZmBgoCVr+VAYTiNlxTguGs0CjMjqAW1cu2cwWvklk0lMJhOhUEgd0XdzCJ7ZbFZBoJGREXK5HB988AE2m41cLsfNmzdVN556va5vnnsUwye6srLCjRs3SKVSqi+vEUzy+/10d3fvaMDbVniSfpx93J2WFwEk8C0p5f/cC+NGn4RCoUA6nWZ+fp75+XkWFxcpFApA84IS+5F26dVqtba8YbDJZFI725MnT5LL5Zienuby5csATE1NYTabiUaj+Hy+lsm1G+z39XovRjllPB7n8uXLqot8vV7HbrerAoczZ85w5syZ+8ZB7xZPci6pAf9FSnkC+DTwn4QQJ/hk3Ohx4K31x3uOUqmkqkmWl5dVAvxhNprrtEWvZrP5vm7fu41RHebz+Th+/DgvvPACPT09LC4uMjc3p2YQ7XQkyx5hX6/XezEaXH/44YfcunWLxcVFUqkUtVpN5X8+/fTTPP3004yNjdHT09MSd8tjryClXJRSfrj+cxa4AfRwd9zo6+svex34jd0SUtN8WqlXIxXF4XCoG1kmk1GzZFqFEX3t6+sjEomoTlvxeJzZ2Vny+XzLZNktDsp6NSr9lpaWuHHjBhcuXGB2dlZVHFmtVjVWeGxsrOUjnbfk41yf1fws8B57YNzodtnY3EOz+3o1Gnl4vV6y2Sx37tzh6NGjFAoFVZveil2CEEJ1gJ+bmyMajWI2m7lz5w7ZbJbR0dFdl6GV7Of1ajQpXl5eVilkRhTdSJz3+/08//zznDt3jnA43NJTzBMbTiGEB/h74A+klJmNRqfV40Y1zaMVejWOykZliJGalMlkNjV6aAVGBywj8d3oqLS2tqbGRB8E9tt6NVxnRiPjTCbD5OSkSj/a2F/C6XQSiUSIRqMEAgG8Xm/L08meyHAKIazcVcLfSin/Yf3ptowbbQYPm9N82GiVXq1WK319fYRCIex2O4lEgrm5Oa5du0ZXVxcnT55s2SgL4zjncDjw+/3kcjk1c/1xPUD3C/txvRotH6enp7lw4QKLi4v80z/9E/F4nJmZGeCT/Nyenh6+9KUvqT6frQoIbeSxt3lx96/5r4AbUso/2/ArY9wotGGU7E4wmUyqhdlhnS/USr0aO06jFM5sNlOpVFhZWVGzaJox9vlJMPqxbryW0Un8INxI9+J6NWrTa7Ua1Wp102dv6MM4gRjz12dmZpiammJ6eppCoaD8mg6HA5/PR39/P729vSpbotVr+El2nJ8Bfhe4KoS4tP7cH9GmUbI7xWw2q2ohY0Cb2+0+jDl8LdOrUZPscDg4deoUmUyGVCrFG2+8QTAYZGlpie7ubk6dOqV6Zu7GQpBScvPmTSYmJnj//feZnp7Gbrfz7LPPEolECIfDTb9mG9hT69XIv8zlchSLRUqlkuqT2mg0+Oijj0gkEnz88cdMTEywvLzM+Pg4uVyOeDy+6Yba39/P6OgoIyMjnD9/nnA4TCAQaMV/4z6eZDzwvwAPO0O1dJRsMxBC4HK58Pv9+Hw+fD7foWzy0Gq9Grv8vr4+Tp8+rQazeb1eIpEIyWSS/v5+otGoen2zaTQaxONxbty4wdTUFOl0mkAgQDQaVSNm9zt7bb0a47hXV1eV77Kzs5O+vj7q9Trz8/PMzs7y3nvvqVEoS0tLDzx9hEIhRkZGeOqppzh+/Hhbc24PfOVQKpXi2rVrTExMUCqVMJvNDA8PMzQ0pPoy6lGxrUEIoWrHa7WayqmdnJxkbm6OcrnMxYsX6e/vZ3BwEKfTSTAY3PYgPWOgmxGhTafTvPvuu3z44YdUq1VGRkaIRCI888wzxGIxgsHgLv3PDy9SSjUK5caNG1y9ehW3282FCxdoNBrcuHGDRCLB9PQ0q6urau6QxWLB5XJhs9no6ekhEAhw7tw5zp8/TyQSaXtvgQNvOBOJBB9++CHz8/MUCgUsFgsnT57kM5/5DMPDw4dyt9kuhBD09PTQ09ODz+fDYrEwOTnJt7/9beLxONevX8fv9/PSSy/xhS98QQWTHA7HE7Udu5dKpUIikSCVSvH2228zMzPDL3/5Sy5fvszo6Civvvoq/f39nDt3jkgk8tiu85rtUS6XyWazXLt2je9+97vA3QY7UkrW1tZUY/GNPmaz2awi5mfPnmVwcJCzZ89y/vz5PbHROfCG00i4XllZoVarYTKZCAQCqq5V01qMP3iPx0NPTw8Ap06dUp2KjGbH4+PjeL1eVlZW1M7TmJpodLMymUxqfnej0aBYLKpZ3kZfgqWlJVVimUwmcbvdDA0NMTQ0xPDwMN3d3WqEcLsX40HFCOqYTCbV/b3RaKj56MZUS6MZscfjwePxcOzYMfx+P6Ojo6pd3F7R04E3nIlEgkuXLpFIJCgWi7hcLgYGBjhz5syeUcJhpLOzU6UDHT16lHQ6zS9/+UsmJiZUtUi9XqdareJwOBgdHVU+roGBATWrplqtqjw/wzjOzFoXF6gAAATeSURBVMwwOTlJqVQilUohpVS5fs888wyvvfYaw8PDvPzyyzgcDpxO52EMDrYEIQRer5dKpaIGIdZqtU2jgAECgQChUIhYLMaJEycIh8O8+OKLhEIhwuGw0t9eWa8H1nAau5eNHaMNJRl3Nk37MObDmEwmYrEYHo+H7u5uCoUCJpOJUqlEsVgkk8lQrVZJJpM0Gg38fr/yS9tsNtUusFQqsbS0RDqdJpVKkc1mqVar1Go11SXJGPBmzLYxSi41u8fGsRahUIienh6VlrTxaB6JROjo6FCBunA4TDQaJRgM4vP59tx6PbCGs1wuUy6XyWQyrK2tkc/nNzU+1ewNjAmHxrA844idy+VYXFzkF7/4BdlslnQ6zfLyMrOzs8oY2u12yuUyyWSSWq1GR0cHLpcLt9vNuXPnlKF0u90cP35c7WqMdLTdGh2r+QSjjaDf7+crX/kKL774ojqmb8QY3me329Xu0phJtRf1tPckahL1ep1KpaJ2HdVqVY+G3YMYndoB5XM2kqKnp6dZXFxUeYC1Wo3V1VVWVlbUgqrVaiQSCdWb0WKx0NnZqaLkQ0ND+Hw+Tpw4oSpMDmPBQzsxdosej4fBwcE2S9McDqzhhM3NPKxWK36/n1AotOe2/ZrNGE0curq6eOmllygWi6rjdz6fp1gsqiR5IygkpSQQCOByufB4POp4Z4ya9Xq920pp0mgexIE2nPDJ5EOjYkjPl9n7GEYxFArpkc2aPcmBNZxGid/w8DBf+cpXqFarytFspMFoNBrNdjiwhtNut2Oz2Th//jxnz54FPtl96kRnjUazEw6s4YRPfJs65USj0TQT0cr0HCHECpAHVlt20eYRZudyH5FSdjZDmL2E1qvW6x5kV/XaUsMJIIS4IKU809KLNoH9Kner2K+fz36Vu1Xs189nt+XWCW0ajUazRbTh1Gg0mi3SDsP5rTZcsxnsV7lbxX79fPar3K1iv34+uyp3y32cGo1Gs9/RR3WNRqPZIi0znEKIXxNCjAshbgshvtmq624VIUSfEOInQoiPhBDXhRD/ef35DiHEPwohbq1/13MW1tkPutV63Tpar4+4biuO6kIIM3AT+DwwB7wPfFVK+dGuX3yLrM+cjkopPxRCeIEPgN8A/j2QlFL+6fofUVBK+YdtFHVPsF90q/W6NbReH02rdpzngNtSygkpZQX4DvDrLbr2lpBSLkopP1z/OQvcAHq4K+/r6y97nbvK0ewT3Wq9bhmt10fQKsPZA8xueDy3/tyeRghxFHgWeA+ISCkX13+1BETaJNZeY9/pVuv1idB6fQQ6OPQQhBAe4O+BP5BSZjb+Tt71b+h0hH2I1uvBpNV6bZXhnAf6NjzuXX9uTyKEsHJXCX8rpfyH9afj6/4Uw6+y3C759hj7Rrdar1tC6/URtMpwvg8cF0IMCCFswO8A32/RtbeEuNsi/K+AG1LKP9vwq+8DX1//+evA91ot2x5lX+hW63XLaL0+6rqtSoAXQnwR+B+AGfhrKeV/a8mFt4gQ4kXgHeAq0Fh/+o+46zf5O6AfmAZ+W0qZbIuQe4z9oFut162j9fqI6+rKIY1Go9kaOjik0Wg0W0QbTo1Go9ki2nBqNBrNFtGGU6PRaLaINpwajUazRbTh1Gg0mi2iDadGo9FsEW04NRqNZov8f8caU8cwzIrxAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "#Convert train datset to (num_images, img_rows, img_cols) format \n",
        "X_train = X_train.reshape(X_train.shape[0], 28, 28)\n",
        "\n",
        "for i in range(6, 9):\n",
        "    plt.subplot(330 + (i+1))\n",
        "    plt.imshow(X_train[i], cmap=plt.get_cmap('gray_r'))\n",
        "    plt.title(y_train[i]);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "50e22553-f70b-4ff5-b3ca-82719ef5ece2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "50e22553-f70b-4ff5-b3ca-82719ef5ece2",
        "outputId": "37fc9d0e-76fc-48b7-95ac-ae5dc8df1828"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(42000, 28, 28, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "#expand 1 more dimention as 1 for colour channel gray\n",
        "X_train = X_train.reshape(X_train.shape[0], 28, 28,1)\n",
        "X_train.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "738bc42d-2ec0-4409-8458-b956bf0a289e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "738bc42d-2ec0-4409-8458-b956bf0a289e",
        "outputId": "ac38a757-f244-4824-c3ad-712d76282f7a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(28000, 28, 28, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "X_test = X_test.reshape(X_test.shape[0], 28, 28,1)\n",
        "X_test.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "636c9dd1-3ef2-4197-aaf3-7fd6b80c7fb2",
      "metadata": {
        "id": "636c9dd1-3ef2-4197-aaf3-7fd6b80c7fb2"
      },
      "source": [
        "# Preprocess the digit images\n",
        "## Feature Standardization\n",
        "\n",
        "It is important preprocessing step. It is used to centre the data around zero mean and unit variance."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5a5597e4-27be-4c6a-a836-ca498e5c478a",
      "metadata": {
        "id": "5a5597e4-27be-4c6a-a836-ca498e5c478a"
      },
      "outputs": [],
      "source": [
        "mean_px = X_train.mean().astype(np.float32)\n",
        "std_px = X_train.std().astype(np.float32)\n",
        "\n",
        "def standardize(x): \n",
        "    return (x-mean_px)/std_px"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3461c170-ddf9-4d00-ba95-65d400e1d576",
      "metadata": {
        "id": "3461c170-ddf9-4d00-ba95-65d400e1d576"
      },
      "source": [
        "# One Hot encoding of labels.\n",
        "\n",
        "## This is another term for the `dummy()` function we saw last week.\n",
        "\n",
        "A one-hot vector is a vector which is 0 in most dimensions, and 1 in a single dimension. In this case, the nth digit will be represented as a vector which is 1 in the nth dimension.\n",
        "\n",
        "For example, 3 would be [0,0,0,1,0,0,0,0,0,0]."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f440748c-5036-4be6-822c-522e5292fcf1",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f440748c-5036-4be6-822c-522e5292fcf1",
        "outputId": "9f9058e8-77a3-4e30-ab94-09f2a0429897"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "from keras.utils.np_utils import to_categorical\n",
        "y_train= to_categorical(y_train)\n",
        "num_classes = y_train.shape[1]\n",
        "num_classes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f419096e-5cf4-4772-8b5f-d69854ab3cb5",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "id": "f419096e-5cf4-4772-8b5f-d69854ab3cb5",
        "outputId": "757f7624-43d7-4ffe-cd4b-b3c67db4f503"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/matplotlib/text.py:1165: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
            "  if s != self._text:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfZAkd33f8fd3Z59ud+budLe7s+jupL2TdjaWKEDKRcHGUJQliEQSqWwSlwQ4wYWtUIUcbFxJyRCDIRWXiV2OyxUBlgFDwJIsy4BVjkAkQQQqREQnnqwHZvZ0upPupJ3de57ZvX3+5o/pXo32dm5nd2emZ3o+r6opzfR0T3/34T7q/fW3f23ujoiItL6OqAsQEZHaUKCLiMSEAl1EJCYU6CIiMaFAFxGJCQW6iEhMKNBFRGJCgR5zZuZmNm1m/ynqWiR+zOzjwe+Xm1ln1PW0OwV6e3i9u38kfGFmbzCzJ81sJvjvG6r9IDMbMbPHgm1/amY3bWDbXWb21SAAjpnZuzaw7S+b2feC/X672u3Ktv8tM5sws/Nm9nkz69nAtu8K6p02s6+Z2a4NbHtj8H2aCb5vV25g26b/Obn7x4Brq/1sqS8Fepsxs27gb4EvA5cBXwT+NlhejfuBHwK7gY8AD5nZYJXb3gPMA2ng3cCnzazaMDgN/AnwB1Wuv8LM/glwN3AjcCVwAPh4ldteC/wZ8CuU6p4BPlXltgPAV4DfBXYBh4C/qnLbVv05SZTcXY8YPwAHri57/XbgBGBly14Abq7iszLAHJAqW/Zd4P1VbNtPKSQyZcu+BPzBBr+eXwO+vcFt7gN+v+z1jcBEldv+PnBf2eurgq8jVcW2dwLfW/U9uAD8gyq2bZmfEzAS/J511vN3WY/1HzpCbz/XAj/x4F9i4CdU92fztcARdy+ULftxldtmgEV3z21i2626NthX+X7TZrZ7o9u6+3MEgbeJbaeB56j+e91uPyfZIgV6+0kC51YtOwekGrDt+U1uu1Wr6w6fN+JrbsVto/o5yRYp0NtPEdi+atl2oLDGus2y7Vat3nf4vJm/5lbcViKmQG8/TwOvMzMrW/a6YHk12x4ws/KjtddXuW0O6DSz0U1su1VPB/sq32/e3U9tdFszOwD0UPp6NrptP6Ux+Gq/1+32c5KtinoQX4/6Prj4pGg3cAz4IKVguit43V3l5z0O/BHQC/wicBYYrHLbByh1X/QDb6L0p/y1VW6bCPb5fuA7wfOuKre9GZgArgF2At+iypOxlMaOzwNvDur+MvBAldsOBl/jO4N6Pwk8XuW2LfNzQidFm+YReQF61PkHvCrQg2XXAU9S6rj4AXBd2XsfBr5+ic8bAb4dbJsFbip7793A05fYdhfwNWCaUsfGu8reezNQvMS27w2+lvLHF8reLwJvvsT2HwLyQTj/BdBT9t7TwLsvse27gnqnKbUS7ip77+vAhy+x7U3AT4Pv17eBkbL3PgN85hLbtsTPSYHePA8LfiASU2Y2S6mF7U/d/Xejrkfixcw+Rul/lj1Av7svRVxSW1Ogi4jEhE6KiojEhAJdRCQmIpsdbWBgwEdGRqLavYhIS3ryySdPuvua8/JEFugjIyMcOnQoqt2LiLQkMztW6T0NuYiIxIQCXUQkJhToIiIxoUAXEYkJBbqISEysG+jB/RcnzeypCu+bmf2pmR02s5+Y2fW1L1NERNZTzRH6FyjNVlfJLcBo8LgT+PTWyxIRkY1aN9Dd/TuUbtBbyW3Af/OSx4GdZvaaWhUoreEbT73MxLnZqMsQaWu1GEPfA7xY9vp4sOwiZnanmR0ys0NTU1M12LU0g3MXFnj/l3/AZ797JOpSRNpaQ0+Kuvu97n7Q3Q8ODq555aq0oPF86e5k2bzuUiYSpVoE+glgX9nrvcEyaRNhkOcU6CKRqkWgPwz8q6Db5Y3AOXd/uQafKy0iN1EK8vz5Oc7OzEdcjUj7WndyLjO7H3grMGBmx4GPAV0A7v4Z4BHgHcBhYAb41XoVK80pmy+Q6DCWlp1cvsgN+3dFXZJIW1o30N39jnXed+ADNatIWs54vsjPXbWb746fJJcvKNBFIqIrRWVLThbnODU9z1vHhkj2dGocXSRCCnTZknD8fCydYjSdJDuhQBeJigJdtiTscMkMJxlLp8jlC+jG4yLRUKDLluTyBS7r62Iw2UMmneLMzAJTxbmoyxJpSwp02ZJcvkgmncLMGBtOAaWTpCLSeAp02TR3JzdRWAnyTLr0X42ji0RDgS6b9vK5WQpzi4wGQT6Q7Oayvi51uohERIEumxaeEB0LAt3MyKRTmtNFJCIKdNm0sGUxk06uLBsbTjGeL6rTRSQCCnTZtFy+SHp7Dzv7uleWZdIpinOLvKS50UUaToEum5bLF1ZOhIbCE6Q5nRgVaTgFumzK0rIzPnlxoGeGgk4XjaOLNJwCXTblxdMzzC4sr5wQDe3o6yK9vUdH6CIRUKDLpuRWLvlPXfReJp0iN6lAF2k0BbpsShjoo0PJi94bS5c6XZaW1eki0kgKdNmUbL7Ivl3b6O+5eEr9zHCKucVlXjg9E0FlIu1LgS6bkpsorJwAXU1TAIhEQ4EuG7awtMyRk8U1x8/hlWEYTQEg0lgKdNmwoyenWVjyizpcQv09nezbtU2BLtJgCnTZsJWbWlQIdGDlZhci0jgKdNmw3ESBRIdxYLC/4jqZdIojU9PMLy43sDKR9qZAlw3L5gtcubuP3q5ExXUy6RSLy87zJ6cbWJlIe1Ogy4bl8sWK4+ehlU4XDbuINIwCXTZkdmGJY6emLzl+DnBgsJ9EhzGuQBdpGAW6bMjhySLL/sqsipX0diUY2d2nXnSRBlKgy4bkquhwCY0Nq9NFpJEU6LIh2XyB7kQHI7v71l13dCjFsdMzXJhfakBlIqJAlw3JTRQ4MNhPZ2L9X52x4RTupWEaEak/BbpsSC5fXHf8PBQOy2jYRaQxFOhStcLsAifOXqhq/BxgZHcf3YkOBbpIgyjQpWrjwdBJtYHemejgqqGketFFGqSqQDezm80sa2aHzezuNd6/wsweM7MfmtlPzOwdtS9VohbeVm69i4rKZdJJ3Y5OpEHWDXQzSwD3ALcA1wB3mNk1q1b7D8CD7n4dcDvwqVoXKtHL5gts60qw97JtVW+TSad46dws52cX6liZiEB1R+g3AIfd/Yi7zwMPALetWseB7cHzHcBLtStRmsV4vkgmnaSjw6reJjyaH8+r00Wk3qoJ9D3Ai2WvjwfLyv0e8B4zOw48AvzGWh9kZnea2SEzOzQ1NbWJciVK2Xyh6vHzUNgRoxOjIvVXq5OidwBfcPe9wDuAL5nZRZ/t7ve6+0F3Pzg4OFijXUsjnJ6eZ6owt+FA37NzG33dCU0BINIA1QT6CWBf2eu9wbJy7wMeBHD3/wv0AgO1KFCaw8ol/1X2oIc6OozRoaSO0EUaoJpAfwIYNbP9ZtZN6aTnw6vWeQG4EcDMfoZSoGtMJUbCQN5Ih0soo7sXiTTEuoHu7ovAXcCjwLOUulmeNrNPmNmtwWq/Dfy6mf0YuB94r7t7vYqWxsvlC2zv7SS9vWfD244NpzhZnOdUca4OlYlIqLOaldz9EUonO8uXfbTs+TPAm2pbmjST3ETpkn+z6jtcQq9MAVDkZ5Mb/x+CiFRHV4rKutydbL7A6CaGW0CdLiKNokCXdU0W5jh3YWFT4+cAQ6ketvd2agoAkTpToMu6wpbDjbYshsysdLMLtS6K1JUCXdb1yl2Kkpv+jLDTRefKRepHgS7ryuULDCR72L2FE5pjwynOzy6SP69OF5F6UaDLurLBHC5bEQ7XaBxdpH4U6HJJy8vO+CbmcFltpXVR4+gidaNAl0s6cfYCM/NLVd92rpJd/d0MJHt0hC5SRwp0uaRXTohuLdABxoaTjCvQRepGgS6XlK1Bh0uo1OlSZHlZnS4i9aBAl0vKTRS4fEcvqd6uLX/WWDrFhYUljp+5UIPKRGQ1BbpcUjZf3PCUuZWMqtNFpK4U6FLR4tIyz00WN33J/2rhsI3mdBGpDwW6VHTs9AzzS8s1OSEKkOrtYs/ObQp0kTpRoEtFYc/4VlsWy2XSSd2OTqROFOhSUTZfwAyuGtx6h0soM5ziyNQ0C0vLNftMESlRoEtFuXyBK3f1sa07UbPPzAylmF9a5tip6Zp9poiUKNClouzE1i/5Xy0cvslOFGv6uSKiQJcK5haXOHpqpqbj5wBXDyUxU6eLSD0o0GVNR6amWVr2mh+h93YlGNndr0AXqQMFuqyplnO4rJZJJ3VxkUgdKNBlTdmJAp0dxv6B/pp/diad4ujJaWYXlmr+2SLtTIEua8rlixwY7Ke7s/a/Ipl0imUvDeuISO0o0GVNuRrc1KKS8ESrxtFFakuBLheZmV/khdMzNZvDZbWR3f10JUzj6CI1pkCXi4znSz3io3UK9O7ODg4MJHU7OpEaU6DLRcIj51r3oJcbVaeLSM0p0OUi4/kCPZ0dXLGrr277GEunOH7mAtNzi3Xbh0i7UaDLRbL5IqPpJIkOq9s+wptmjE9qCgCRWlGgy0VydZjDZbXwhKvG0UVqp6pAN7ObzSxrZofN7O4K6/yymT1jZk+b2X21LVMa5dzMAhPnZ+se6Pt29dHb1aFxdJEa6lxvBTNLAPcAbwOOA0+Y2cPu/kzZOqPA7wBvcvczZjZUr4KlvnKTwQnROgd6osO4eiipXnSRGqrmCP0G4LC7H3H3eeAB4LZV6/w6cI+7nwFw98nalimNsjKHSx07XEKZdEqBLlJD1QT6HuDFstfHg2XlMkDGzP6PmT1uZjev9UFmdqeZHTKzQ1NTU5urWOoqN1Eg2dPJ5Tt6676vsXSK/Pk5zs7M131fIu2gVidFO4FR4K3AHcCfm9nO1Su5+73uftDdDw4ODtZo11JL2XyBTDqJWf06XEKZlSkA1OkiUgvVBPoJYF/Z673BsnLHgYfdfcHdnwdylAJeWoi71+UuRZWE4/Q6MSpSG9UE+hPAqJntN7Nu4Hbg4VXrfI3S0TlmNkBpCOZIDeuUBjhZnOfMzELDAv01O3pJ9XSqdVGkRtYNdHdfBO4CHgWeBR5096fN7BNmdmuw2qPAKTN7BngM+HfufqpeRUt9jDfgkv9yZsZoWp0uIrWybtsigLs/AjyyatlHy5478KHgIS0qW8e7FFUyNpziG09N4O4NGbcXiTNdKSorcvkCl/V1MZDsbtg+M+kUZ2YWmCrONWyfInGlQJcV4QnRRh4pvzIFgDpdRLZKgS5AqcNlPF9s2Ph5KGxdVKeLyNYp0AWAl8/NUphbbOj4OcBAsodd/d0rJ2RFZPMU6AI05qYWlWR0swuRmlCgC/DKNLaZocYH+lg6RW6iQKlZSkQ2S4EuQOkIPb29hx19XQ3fd2Y4xfT8EifOXmj4vkXiRIEuQKllsdHj56GVThcNu4hsiQJdWFp2Dk8W6z4HeiWjaU3SJVILCnThxdMzzC4sN2QO9LXs2NbF8PZezekiskUKdInkkv/VMsMpdbqIbJECXVaOjEeHkpHVMJZOMj5ZZGlZnS4im6VAF7L5Avt2baO/p6q52uoik04xv7jMsVPTkdUg0uoU6FK65D/C4RZ4ZbhHJ0ZFNk+B3ubmF5d5bqoY6fg5wGi6NNyj1kWRzVOgt7mjp6ZZXPbIA72vu5MrdvXpxKjIFijQ21x2IvoOl1AmmAJARDZHgd7mcvkCiQ7jwGB/1KUwNpzk+ZPTzC8uR12KSEtSoLe5XL7AyO4+ersSUZdCJp1icdl5/qQ6XUQ2Q4He5nIR3NSiknDYR+PoIpujQG9jswtLHD01zWgEU+au5cBgP4kO0zi6yCYp0NvY4cki7tHc1GItPZ0J9g/06whdZJMU6G2smTpcQmPplHrRRTZJgd7GcpMFuhMdjOzui7qUFaPpJC+cnuHC/FLUpYi0HAV6G8tNFLhqKElnonl+DcbSKdxLw0EisjHN8y9ZGi6XL5JJRzfD4lrCOdk1ji6ycQr0NlWYXeDE2QtNNX4OcOWuPro7OzSOLrIJCvQ2NR4MaUQ9y+JqnYkOrh5MKtBFNkGB3qbCXu9maVksl0kn1YsusgkK9DaVzRfo606wZ+e2qEu5SGY4xUvnZjk/uxB1KSItRYHepnL5AqNDSTo6LOpSLhIOA41r2EVkQ6oKdDO72cyyZnbYzO6+xHrvNDM3s4O1K1HqITsR/U0tKlmZ02VCrYsiG7FuoJtZArgHuAW4BrjDzK5ZY70U8EHg+7UuUmrr9PQ8J4tzTTl+DrBn5zb6uxM6MSqyQdUcod8AHHb3I+4+DzwA3LbGev8R+CQwW8P6pA7CoGzWI/SODuNqTQEgsmHVBPoe4MWy18eDZSvM7Hpgn7v/90t9kJndaWaHzOzQ1NTUhouV2giDslmP0AHG0mpdFNmoLZ8UNbMO4I+B315vXXe/190PuvvBwcHBre5aNik7UWB7bydDqZ6oS6kok05xslgaGhKR6lQT6CeAfWWv9wbLQingtcC3zewo8EbgYZ0YbV65fIGx4RRmzdfhEgr/etBRukj1qgn0J4BRM9tvZt3A7cDD4Zvufs7dB9x9xN1HgMeBW939UF0qli1x92AOl+YdboHy1kV1uohUa91Ad/dF4C7gUeBZ4EF3f9rMPmFmt9a7QKmtycIc5y4sNPX4OcBgqocd27o0SZfIBnRWs5K7PwI8smrZRyus+9atlyX10ow3tViLmZVudqEpAESqpitF20yztyyWywwnyeYLuHvUpYi0BAV6m8lOFBhI9rCrvzvqUtY1lk5RmF1k4rwubRCphgK9zeQmi4wNN9dNLSoJ/4rI6cSoSFUU6G1kedkZzxdaYrgFygJd4+giVVGgt5ETZy8wM7/UMoF+WX83g6kedbqIVEmB3kZapcOl3JjmdBGpmgK9jWRXOlxaYwwdSv/zGc8XWV5Wp4vIehTobWQ8X2DPzm2keruiLqVqY8NJLiwscfzMhahLEWl6CvQ2ks0XW+roHGA0vNmFhl1E1qVAbxOLS8s8N9n8c7isNjpU+h+QxtFF1qdAbxNHT80wv7TccoGe6u1iz85tKyd0RaQyBXqbaIWbWlQyNqxOF5FqKNDbRC5fwAyuHmqtMXQodbocmZpmYWk56lJEmpoCvU3k8gVGdvfT25WIupQNy6STzC8tc+zUdNSliDQ1BXqbyE4UVk4wtppw3D87oTldRC5Fgd4GZheWOHpqpiXHz6E0TNRhal0UWY8CvQ0cmZpmadlbrsMl1NuVYGR3vybpElmHAr0NjE+2bodLKJNOkZtUoItcigK9DWQnCnQljJHd/VGXsmmZdJKjJ6eZXViKuhSRpqVAbwO5fIH9A/10d7bujzsznGLZ4bkpnRgVqaR1/4VL1bItdFOLSsZW7l6kYReRShToMTc9t8iLpy+sBGKrGhnopythal0UuQQFeswdniwFYKaFT4gCdCU6uGowybiO0EUqUqDHXNi73epH6FCaSle96CKVKdBjLjdRoKezg327+qIuZcvG0kmOn7lAcW4x6lJEmpICPeay+QKj6SSJDou6lC0LT+xq2EVkbQr0mMvFoMMlFF4YpU4XkbUp0GPs3MwC+fNzsRg/B9h3WR+9XR3k8up0EVmLAj3GwkvlW73DJdTRYYwO6WYXIpUo0GMsvG1bXIZcoPS16HZ0ImurKtDN7GYzy5rZYTO7e433P2Rmz5jZT8zsf5nZlbUvVTYqly+Q7Onk8h29UZdSM2PDSSYLc5yZno+6FJGms26gm1kCuAe4BbgGuMPMrlm12g+Bg+7+OuAh4D/XulDZuOxEgUw6iVnrd7iEMpoCQKSiao7QbwAOu/sRd58HHgBuK1/B3R9z95ng5ePA3tqWKRvl7uTyhZaeMnctK50ukzoxKrJaNYG+B3ix7PXxYFkl7wO+vtYbZnanmR0ys0NTU1PVVykbdrI4z5mZhViNnwMMb+8l1dOpm12IrKGmJ0XN7D3AQeAP13rf3e9194PufnBwcLCWu5ZVwiGJuAW6mZEZ1hQAImupJtBPAPvKXu8Nlr2Kmd0EfAS41d3nalOebFYcO1xCmXSpddHdoy5FpKlUE+hPAKNmtt/MuoHbgYfLVzCz64A/oxTmk7UvUzZqfLLArv5uBpLdUZdSc2PpJGdnFpgq6rhBpNy6ge7ui8BdwKPAs8CD7v60mX3CzG4NVvtDIAn8tZn9yMwervBx0iBx7HAJhRdK5TQ3usirdFazkrs/AjyyatlHy57fVOO6ZAtKHS5F3nn9pc5dt65wGCmbL/DzowMRVyPSPHSlaAy9dG6W4twiozEcPwcYSPawu79bnS4iqyjQYygMurj1oJfL6GYXIhdRoMfQSsviUHwDfWw4xbg6XUReRYEeQ9l8geHtvezo64q6lLrJpFNMzy9x4uyFqEsRaRoK9BjK5QuxmTK3kkw6CWhOF5FyCvSYWVp2xvNFMkPJqEupq/CEb1atiyIrFOgx88LpGeYWl2N/hL5jWxev2dGrI3SRMgr0mAkDLi63nbuUcAoAESlRoMdM2LI4mo73kAsEnS6TRZaW1ekiAgr02MnmC1yxq4++7qouAm5po0NJ5heXOXZqOupSRJqCAj1mcvnCSgdI3K3c7ELDLiKAAj1W5heXOTI1Hcspc9dy9VASM3W6iIQU6DFy9NQ0i8se60v+y/V1d3LFrj5ykzpCFwEFeqzE+aYWlWTSKU3SJRJQoMdILl8g0WEcGOyPupSGyaSTPH9ymrnFpahLEYmcAj1GshMFRnb30dOZiLqUhsmkUywuO8+fVKeLiAI9RnL5QtuMn4fCrzerYRcRBXpczC4scez0TFuNnwMcGEjS2WGM59XpIqJAj4nDk0Xc2+OS/3LdnR3sH+jXzS5EUKDHRnblkv/2CnTQnC4iIQV6TOTyBboTHYzs7ou6lIbLpFO8cHqGmfnFqEsRiZQCPSay+QJXDSXpTLTfj3RsOIl7adhJpJ2137/+mBrPFxlrkzlcVgtPBOd0YlTanAI9BgqzC5w4eyH2N7Wo5Mrd/XR3dmgcXdqeAj0GwiPTzFB7Bnqiw7h6MKledGl7CvQYWLlLUZseoUPpa9cRurQ7BXoMZCcK9HUn2LNzW9SlRCaTTvHyuVnOXViIuhSRyCjQY2B8ssBoOkVHh0VdSmTGhksnhA9rKl1pYwr0GMhOtG+HSyjsdNHNLqSdKdBb3KniHCeLc203h8tqe3Zuo787oXF0aWsK9Ba30uHS5oFuZoymU+p0kbZWVaCb2c1mljWzw2Z29xrv95jZXwXvf9/MRmpdqKxNHS6vGNOcLtLm1g10M0sA9wC3ANcAd5jZNatWex9wxt2vBv4L8MlaFypry+UL7NjWxVCqJ+pSIpcZTnFqep6TxbmoSxGJRGcV69wAHHb3IwBm9gBwG/BM2Tq3Ab8XPH8I+K9mZu7uNawVgAefeJE//+6RWn9sy3rp7AWuvXwHZu3b4RIKpw7+pU99j55OjSZK8/q3N47yz19/ec0/t5pA3wO8WPb6OPCPK63j7otmdg7YDZwsX8nM7gTuBLjiiis2VfDOvi5G27yjo9xoOskvXrc36jKawsGRy7j9H+3j/Kx60aW57djWVZfPrSbQa8bd7wXuBTh48OCmjt7ffu0wb792uKZ1STz0diX4g3e+LuoyRCJTzd+lJ4B9Za/3BsvWXMfMOoEdwKlaFCgiItWpJtCfAEbNbL+ZdQO3Aw+vWudh4F8Hz/8F8K16jJ+LiEhl6w65BGPidwGPAgng8+7+tJl9Ajjk7g8DnwO+ZGaHgdOUQl9ERBqoqjF0d38EeGTVso+WPZ8F/mVtSxMRkY1Qb5eISEwo0EVEYkKBLiISEwp0EZGYsKi6C81sCji2yc0HWHUVakRUx6upjuaqAVTHanGo40p3H1zrjcgCfSvM7JC7H1QdqqNZ62iGGlRH+9WhIRcRkZhQoIuIxESrBvq9URcQUB2vpjpe0Qw1gOpYLdZ1tOQYuoiIXKxVj9BFRGQVBbqISEy0XKCvd8PqBtXweTObNLOnoth/UMM+M3vMzJ4xs6fN7IMR1dFrZv/PzH4c1PHxKOooqydhZj80s7+LsIajZvb3ZvYjMzsUYR07zewhM/upmT1rZj8bQQ1jwfchfJw3s9+MoI7fCn4/nzKz+82st9E1BHV8MKjh6bp8H9y9ZR6Upu99DjgAdAM/Bq6JoI63ANcDT0X4vXgNcH3wPAXkIvpeGJAMnncB3wfeGOH35UPAfcDfRVjDUWAgqv2X1fFF4NeC593AzojrSQATlC6MaeR+9wDPA9uC1w8C743g638t8BTQR2mm2/8JXF3LfbTaEfrKDavdfR4Ib1jdUO7+HUrzvkfG3V929x8EzwvAs5R+cRtdh7t7MXjZFTwiOdNuZnuBfwp8Nor9NxMz20HpwONzAO4+7+5no62KG4Hn3H2zV4hvRSewLbijWh/wUgQ1/AzwfXefcfdF4H8Dv1TLHbRaoK91w+qGh1izMbMR4DpKR8dR7D9hZj8CJoH/4e6R1AH8CfDvgeWI9h9y4Jtm9mRwY/Qo7AemgL8IhqA+a2b9EdUSuh24v9E7dfcTwB8BLwAvA+fc/ZuNroPS0fmbzWy3mfUB7+DVt/fcslYLdFnFzJLA3wC/6e7no6jB3Zfc/Q2U7jd7g5m9ttE1mNk/Aybd/clG73sNP+/u1wO3AB8ws7dEUEMnpWHBT7v7dcA0EMk5J4Dg9pW3An8dwb4vo/SX/H7gcqDfzN7T6Drc/Vngk8A3gW8APwKWarmPVgv0am5Y3TbMrItSmP+lu38l6nqCP+kfA26OYPdvAm41s6OUhuJ+wcy+HEEd4REh7j4JfJXSUGGjHQeOl/219BClgI/KLcAP3D0fwb5vAp539yl3XwC+AvxcBHXg7p9z93/o7m8BzlA691UzrRbo1dywui2YmVEaH33W3f84wjoGzWxn8Hwb8Dbgp42uw91/x933uvsIpd+Lb7l7w4/CzKzfzFLhc+DtlP7Ubih3nwBeNLOxYNGNwDONrqPMHUQw3BJ4AXijmfUF/25upHTOqeHMbCj47xWUxs/vq+XnV3VP0WbhFW5Y3eg6zOx+4K3AgJkdBz7m7p9rcBlvAn4F+Ptg/Brgw9Bn5ZYAAACISURBVF66/2sjvQb4opklKB0gPOjukbUMNoE08NVSbtAJ3Ofu34iolt8A/jI4+DkC/GoURQT/Y3sb8G+i2L+7f9/MHgJ+ACwCPyS6KQD+xsx2AwvAB2p9olqX/ouIxESrDbmIiEgFCnQRkZhQoIuIxIQCXUQkJhToIiIxoUAXEYkJBbqISEz8f0RFX2LIXOJTAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Lets plot 10th label.\n",
        "\n",
        "plt.title(y_train[9])\n",
        "plt.plot(y_train[9])\n",
        "plt.xticks(range(10));"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "fe277620-bb3b-4218-b616-0e49535739ed",
      "metadata": {
        "id": "fe277620-bb3b-4218-b616-0e49535739ed"
      },
      "source": [
        "# Design Neural Network Architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "acc3a523-3f8e-4df2-98ef-21cd159af0f1",
      "metadata": {
        "id": "acc3a523-3f8e-4df2-98ef-21cd159af0f1"
      },
      "outputs": [],
      "source": [
        "# fix random seed for reproducibility\n",
        "seed = 43\n",
        "np.random.seed(seed)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d786e63a-624a-48a8-b682-29cb1b088e1a",
      "metadata": {
        "id": "d786e63a-624a-48a8-b682-29cb1b088e1a"
      },
      "outputs": [],
      "source": [
        "# Linear Model\n",
        "from keras.models import  Sequential\n",
        "from keras.layers.core import  Lambda , Dense, Flatten, Dropout\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.layers import BatchNormalization, Convolution2D , MaxPooling2D"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "79bbb861-c1e4-442e-b586-80374fb79201",
      "metadata": {
        "id": "79bbb861-c1e4-442e-b586-80374fb79201"
      },
      "source": [
        "# Create a simple model from Keras Sequential layer.\n",
        "\n",
        "1. Lambda layer performs simple arithmetic operations like sum, average, exponentiation etc.\n",
        "In 1st layer of the model we have to define input dimensions of our data in (rows,columns,colour channel) format. (In theano colour channel comes first)\n",
        "2. Flatten will transform input into 1D array.\n",
        "3. Dense is fully connected layer that means all neurons in previous layers will be connected to all neurons in fully connected layer. In the last layer we have to specify output dimensions/classes of the model. Here it's 10, since we have to output 10 different digit labels."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1ee9901e-b8b6-49f8-8fde-0b34c77774aa",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1ee9901e-b8b6-49f8-8fde-0b34c77774aa",
        "outputId": "573f93f3-831b-496b-d15b-00d489955326"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input shape  (None, 28, 28, 1)\n",
            "output shape  (None, 10)\n"
          ]
        }
      ],
      "source": [
        "model= Sequential()\n",
        "model.add(Lambda(standardize,input_shape=(28,28,1)))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(10, activation='softmax'))\n",
        "print(\"input shape \",model.input_shape)\n",
        "print(\"output shape \",model.output_shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "042d9462-3e5d-425d-a9be-9fd28b608687",
      "metadata": {
        "id": "042d9462-3e5d-425d-a9be-9fd28b608687"
      },
      "source": [
        "# Compile network\n",
        "Before making network ready for training we have to make sure to add below things:\n",
        "\n",
        "1. A loss function: to measure how good the network is\n",
        "2. An optimizer: to update network as it sees more data and reduce loss value\n",
        "3. Metrics: to monitor performance of network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cf5b8656-cf11-4629-82ff-63b387c6ad4e",
      "metadata": {
        "id": "cf5b8656-cf11-4629-82ff-63b387c6ad4e"
      },
      "outputs": [],
      "source": [
        "# from keras.optimizers import RMSprop\n",
        "model.compile(optimizer=RMSprop(learning_rate=0.001),\n",
        " loss='categorical_crossentropy',\n",
        " metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ef093f4f-d899-43d9-ac79-d112c55cada7",
      "metadata": {
        "id": "ef093f4f-d899-43d9-ac79-d112c55cada7"
      },
      "outputs": [],
      "source": [
        "from keras.preprocessing import image\n",
        "gen = image.ImageDataGenerator()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "66c3c45e-30ea-484c-b04b-13c800ddd877",
      "metadata": {
        "id": "66c3c45e-30ea-484c-b04b-13c800ddd877"
      },
      "source": [
        "# Cross Validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "184efe41-5535-4e41-97a5-5b3407a3a08b",
      "metadata": {
        "id": "184efe41-5535-4e41-97a5-5b3407a3a08b"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X = X_train\n",
        "y = y_train\n",
        "batch_size=64\n",
        "\n",
        "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\n",
        "batches = gen.flow(X_train, y_train, batch_size=batch_size)\n",
        "val_batches=gen.flow(X_val, y_val, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a83d4488-d81b-495f-bd63-85b7fbb7d5bb",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a83d4488-d81b-495f-bd63-85b7fbb7d5bb",
        "outputId": "4107b701-cda2-4d7e-921f-137f3a31e7d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "  590/37800 [..............................] - ETA: 2:31 - loss: 0.4524 - accuracy: 0.8647WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 189000 batches). You may need to use the repeat() function when building your dataset.\n",
            "WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 37800 batches). You may need to use the repeat() function when building your dataset.\n",
            "37800/37800 [==============================] - 4s 86us/step - loss: 0.4522 - accuracy: 0.8648 - val_loss: 0.3286 - val_accuracy: 0.9081\n"
          ]
        }
      ],
      "source": [
        "steps_per_epoch = len(X_train)//batch_size\n",
        "validation_steps = len(X_test)//batch_size # if you have validation data \n",
        "\n",
        "history=model.fit(batches, \n",
        "                            steps_per_epoch=batches.n, \n",
        "                            epochs=5, \n",
        "                            validation_data=val_batches, \n",
        "                            validation_steps=batches.n\n",
        "                           )"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "(Don't worry is the first try runs out of data)"
      ],
      "metadata": {
        "id": "O09Kdgi2HuMe"
      },
      "id": "O09Kdgi2HuMe"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "94f4592b-a9f8-4898-ab4c-bf93e3dcdf81",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "94f4592b-a9f8-4898-ab4c-bf93e3dcdf81",
        "outputId": "09fd967a-d301-4be0-dbdc-b47bee067d61"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy'])"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ],
      "source": [
        "history_dict = history.history\n",
        "history_dict.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "64a411f7-12cc-4818-9f37-86ce55ee156e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "64a411f7-12cc-4818-9f37-86ce55ee156e",
        "outputId": "82ba0390-dcc3-433e-f616-ccc57ef3460b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUBUlEQVR4nO3df7DddZ3f8eeLBJRfCjW3uysJBN040yi7lD1ldbtUS9XFOgVW9geY6eJOpyy7pLpT2YKjUxG705Htsi0lu212il13oRnrVCcdRLSuSLXo5mTlh4FSQhQI0nqpCJsi8sN3/zjfq4fkc5Ob5H7vuTd5PmbO3O/n8/mek/cnIbzy+X6/5/tNVSFJ0u6OmHQBkqTFyYCQJDUZEJKkJgNCktRkQEiSmpZPuoD5smLFilq9evWky5CkJWXr1q2PV9VUa+yQCYjVq1czHA4nXYYkLSlJHpptzENMkqQmA0KS1GRASJKaDAhJUpMBIUlqMiCkHt14I6xeDUccMfp5442Trkiau0PmMldpsbnxRrjkEnj66VH7oYdGbYB16yZXlzRXriCknrz//T8KhxlPPz3ql5YCA0LqycMP71+/tNgYEFJPTj55//qlxcaAkHryu78Lxxzz4r5jjhn1S0uBASH1ZN062LgRTjkFktHPjRs9Qa2lw6uYpB6tW2cgaOlyBSFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNfUaEEnOSXJ/ku1JrtzLfhckqSSD3fpPTrIryeV91ilJ2lNvAZFkGbABeBuwFrgoydrGfscD7wG+2viYa4Fb+qpRkjS7PlcQZwLbq2pHVT0LbALOa+z3YeAjwDPjnUnOB74BbOuxRknSLPoMiJOAR8baO7u+H0pyBrCqqm7erf844ArgQ3v7BZJckmSYZDg9PT0/VUuSgAmepE5yBKNDSO9tDF8F/EFV7drbZ1TVxqoaVNVgamqqhyol6fDV5xPlHgVWjbVXdn0zjgdeB9yWBODHgc1JzgV+FvilJNcAJwA/SPJMVV3fY72SpDF9BsQWYE2SUxkFw4XAO2cGq+pJYMVMO8ltwOVVNQTOGuu/CthlOEjSwurtEFNVPQ+sB24F7gM+XlXbklzdrRIkSYtYqmrSNcyLwWBQw+Fw0mVI0pKSZGtVDVpjfpNaktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDX1GhBJzklyf5LtSa7cy34XJKkkg679liRbk9zT/Ty7zzolSXta3tcHJ1kGbADeAuwEtiTZXFX37rbf8cB7gK+OdT8O/IOq+laS1wG3Aif1VaskaU99riDOBLZX1Y6qehbYBJzX2O/DwEeAZ2Y6quprVfWtrrkNODrJS3qsVZK0mz4D4iTgkbH2TnZbBSQ5A1hVVTfv5XMuAP6yqr6/+0CSS5IMkwynp6fno2ZJUmdiJ6mTHAFcC7x3L/u8ltHq4jda41W1saoGVTWYmprqp1BJOkz1GRCPAqvG2iu7vhnHA68DbkvyTeD1wOaxE9UrgU8Cv1ZVD/ZYpySpoc+A2AKsSXJqkqOAC4HNM4NV9WRVraiq1VW1GvgKcG5VDZOcANwMXFlVX+6xRknSLHoLiKp6HljP6Aqk+4CPV9W2JFcnOXcfb18P/CTwz5Pc2b3+el+1SpL2lKqadA3zYjAY1HA4nHQZkrSkJNlaVYPWmN+kliQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNvQZEknOS3J9ke5Ir97LfBUkqyWCs733d++5P8gt91ilJ2tPyvj44yTJgA/AWYCewJcnmqrp3t/2OB94DfHWsby1wIfBa4JXAf0vymqp6oa96JUkv1ucK4kxge1XtqKpngU3AeY39Pgx8BHhmrO88YFNVfb+qvgFs7z5PkrRA+gyIk4BHxto7u74fSnIGsKqqbt7f93bvvyTJMMlwenp6fqqWJAETPEmd5AjgWuC9B/oZVbWxqgZVNZiampq/4iRJ/Z2DAB4FVo21V3Z9M44HXgfclgTgx4HNSc6dw3slST3rcwWxBViT5NQkRzE66bx5ZrCqnqyqFVW1uqpWA18Bzq2qYbffhUlekuRUYA3wFz3WKknaTW8riKp6Psl64FZgGXBDVW1LcjUwrKrNe3nvtiQfB+4Fngcu8womSVpYqapJ1zAvBoNBDYfDSZchSUtKkq1VNWiN+U1qSVLTnAIiybHdVUckeU2Sc5Mc2W9pkqRJmusK4nbgpUlOAj4L/EPgP/ZVlCRp8uYaEKmqp4F3AH9YVb/M6DYYkqRD1JwDIskbgHXAzLeel/VTkiRpMZhrQPw28D7gk90lqK8CvtBfWZKkSZvT9yCq6ovAF+GHt8h4vKre3WdhkqTJmutVTDcleVmSY4GvA/cm+Z1+S5MkTdJcDzGtraqngPOBW4BTGV3JJEk6RM01II7svvdwPrC5qp4DDo2vYEuSmuYaEP8e+CZwLHB7klOAp/oqSpI0eXM9SX0dcN1Y10NJ/m4/JUmSFoO5nqR+eZJrZ57eluT3Ga0mJEmHqLkeYroB+CvgV7rXU8BH+ypKkjR5c30exKur6oKx9oeS3NlHQZKkxWGuK4jvJfn5mUaSvw18r5+SJEmLwVxXEJcCH0vy8q79BHBxPyVJkhaDuV7FdBfw00le1rWfSvLbwN19FidJmpz9eqJcVT3VfaMa4J/2UI8kaZE4mEeOZt6qkCQtOgcTEN5qQ5IOYXs9B5Hkr2gHQYCje6lIkrQo7DUgqur4hSpEkrS4HMwhpn1Kck6S+5NsT3JlY/zSJPckuTPJl5Ks7fqPTPIn3dh9Sd7XZ52SpD31FhBJlgEbgLcBa4GLZgJgzE1VdVpVnQ5cA1zb9f8y8JKqOg34GeA3kqzuq1ZJ0p76XEGcCWyvqh1V9SywCThvfIexS2ZhdPO/mfMdBRybZDmjcx3P4u3FJWlB9RkQJwGPjLV3dn0vkuSyJA8yWkHMPOf6E8D/Ax4DHgb+VVV9p/HeS2buMDs9PT3f9UvSYa3XcxBzUVUbqurVwBXAB7ruM4EXgFcyerzpe5O8qvHejVU1qKrB1NTUgtUsSYeDPgPiUWDVWHtl1zebTYweaQrwTuAzVfVcVX0b+DIw6KVKSVJTnwGxBViT5NQkRwEXApvHd0iyZqz5duCBbvth4Oxun2OB1wP/s8daJUm7mevdXPdbVT2fZD1wK7AMuKGqtiW5GhhW1WZgfZI3A8/x4jvEbgA+mmQboy/lfbSqvDGgJC2gVB0ad8wYDAY1HA4nXYYkLSlJtlZV8xD+xE9SS5IWJwNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpqdeASHJOkvuTbE9yZWP80iT3JLkzyZeSrB0b+6kkdyTZ1u3z0j5rlSS9WG8BkWQZsAF4G7AWuGg8ADo3VdVpVXU6cA1wbffe5cCfAZdW1WuBNwHP9VWrJGlPfa4gzgS2V9WOqnoW2AScN75DVT011jwWqG77rcDdVXVXt9//raoXeqxVkrSbPgPiJOCRsfbOru9FklyW5EFGK4h3d92vASrJrUn+Msk/a/0CSS5JMkwynJ6enufyJenwNvGT1FW1oapeDVwBfKDrXg78PLCu+/mLSf5e470bq2pQVYOpqakFq1mSDgd9BsSjwKqx9squbzabgPO77Z3A7VX1eFU9DXwaOKOXKiVJTX0GxBZgTZJTkxwFXAhsHt8hyZqx5tuBB7rtW4HTkhzTnbB+I3Bvj7VKknazvK8Prqrnk6xn9D/7ZcANVbUtydXAsKo2A+uTvJnRFUpPABd3730iybWMQqaAT1fVzX3VKknaU6pq33stAYPBoIbD4aTLkKQlJcnWqhq0xiZ+klqStDgZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU29BkSSc5Lcn2R7kisb45cmuSfJnUm+lGTtbuMnJ9mV5PI+65Qk7am3gEiyDNgAvA1YC1y0ewAAN1XVaVV1OnANcO1u49cCt/RVoyRpdn2uIM4EtlfVjqp6FtgEnDe+Q1U9NdY8FqiZRpLzgW8A23qsUZI0iz4D4iTgkbH2zq7vRZJcluRBRiuId3d9xwFXAB/a2y+Q5JIkwyTD6enpeStckrQITlJX1YaqejWjQPhA130V8AdVtWsf791YVYOqGkxNTfVcqSQdXpb3+NmPAqvG2iu7vtlsAv6o2/5Z4JeSXAOcAPwgyTNVdX0vlUqS9tBnQGwB1iQ5lVEwXAi8c3yHJGuq6oGu+XbgAYCqOmtsn6uAXYaDJC2s3gKiqp5Psh64FVgG3FBV25JcDQyrajOwPsmbgeeAJ4CL+6pHkrR/UlX73msJGAwGNRwOJ12GJC0pSbZW1aA1NvGT1JKkxcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAWwFVXTboCaf8ZENIC+NBen2wiLU4GhCSpyYCQenLVVZCMXvCjbQ83aanwbq7SAkjgEPmrpkOMd3OVJO03A0JaAB/84KQrkPafASEtAM87aCkyICRJTQaEJKnJgJAkNRkQkqQmA0KS1HTIfFEuyTTw0KTrOAArgMcnXcQCc86Hh8Ntzkt1vqdU1VRr4JAJiKUqyXC2bzEeqpzz4eFwm/OhOF8PMUmSmgwISVKTATF5GyddwAQ458PD4TbnQ26+noOQJDW5gpAkNRkQkqQmA6InSc5Jcn+S7UmubIyfkuTzSe5OcluSlWNjJyf5bJL7ktybZPVC1n6gDnLO1yTZ1s35umTmOWyLW5Ibknw7yddnGU83n+3dvM8YG7s4yQPd6+KFq/rgHOick5ye5I7uz/nuJL+6sJUfuIP5c+7GX5ZkZ5LrF6bieVJVvub5BSwDHgReBRwF3AWs3W2f/wxc3G2fDfzp2NhtwFu67eOAYyY9pz7nDPwc8OXuM5YBdwBvmvSc5jjvvwOcAXx9lvG/D9wCBHg98NWu/68BO7qfJ3bbJ056Pj3P+TXAmm77lcBjwAmTnk+fcx4b/zfATcD1k57L/rxcQfTjTGB7Ve2oqmeBTcB5u+2zFvjzbvsLM+NJ1gLLq+pzAFW1q6qeXpiyD8oBzxko4KWMguUlwJHA/+m94nlQVbcD39nLLucBH6uRrwAnJPkJ4BeAz1XVd6rqCeBzwDn9V3zwDnTOVfW/quqB7jO+BXwbaH6Dd7E5iD9nkvwM8GPAZ/uvdH4ZEP04CXhkrL2z6xt3F/CObvsXgeOTvILRv7K+m+S/JPlakt9Lsqz3ig/eAc+5qu5gFBiPda9bq+q+nutdKLP9vszl92up2ufckpzJ6B8EDy5gXX1qzjnJEcDvA5dPpKqDZEBMzuXAG5N8DXgj8CjwArAcOKsb/1uMDtm8a0I1zrfmnJP8JPA3gJWM/qKdneSsyZWpPnX/sv5T4Ner6geTrqdnvwV8uqp2TrqQA7F80gUcoh4FVo21V3Z9P9Qtsd8BkOQ44IKq+m6SncCdVbWjG/sUo2Oa/2EhCj8IBzPnfwx8pap2dWO3AG8A/vtCFN6z2X5fHgXetFv/bQtWVb9m/W8hycuAm4H3d4diDhWzzfkNwFlJfovR+cSjkuyqqj0u4liMXEH0YwuwJsmpSY4CLgQ2j++QZEW3/AR4H3DD2HtPSDJzbPZs4N4FqPlgHcycH2a0slie5EhGq4tD5RDTZuDXuqtcXg88WVWPAbcCb01yYpITgbd2fYeC5py7/y4+yehY/ScmW+K8a865qtZV1clVtZrRCvpjSyUcwBVEL6rq+STrGf2FXwbcUFXbklwNDKtqM6N/Pf7LJAXcDlzWvfeFJJcDn+8u9dwK/PEk5rE/DmbOwCcYBeE9jE5Yf6aq/utCz+FAJPlPjOa1olv9fZDRSXaq6t8Bn2Z0hct24Gng17ux7yT5MKNgBbi6qvZ2EnTRONA5A7/C6GqgVyR5V9f3rqq6c8GKP0AHMeclzVttSJKaPMQkSWoyICRJTQaEJKnJgJAkNRkQkqQmA0LahyQvJLlz7DVv17EnWT3bHUKlSfN7ENK+fa+qTp90EdJCcwUhHaAk38zoORb3JPmL7p5SM6uCP++eC/D5JCd3/T+W5JNJ7upeP9d91LIkf9w9J+GzSY7u9n93Rs8DuTvJpglNU4cxA0Lat6N3O8Q0/qCbJ6vqNOB64F93ff8W+JOq+ingRuC6rv864ItV9dOMni2wretfA2yoqtcC3wUu6PqvBP5m9zmX9jU5aTZ+k1rah+7masc1+r8JnF1VO7p7SP3vqnpFkseBn6iq57r+x6pqRZJpYGVVfX/sM1Yzei7Emq59BXBkVf2LJJ8BdgGfAj41czNDaaG4gpAOTs2yvT++P7Y9c8t3gLcDGxitNrYk8ZyhFpQBIR2cXx37eUe3/T8Y3c0WYB0/um3554HfBEiyLMnLZ/vQ7q63q6rqC8AVwMsZ3S5aWjD+i0Tat6OTjN9x9DNjt2w+McndjFYBF3V9/wT4aJLfAab50Z093wNsTPKPGK0UfpPRE/RalgF/1oVIgOuq6rvzNiNpDjwHIR2g7hzEoKoen3QtUh88xCRJanIFIUlqcgUhSWoyICRJTQaEJKnJgJAkNRkQkqSm/w8zf67qbas/cAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "loss_values = history_dict['loss']\n",
        "val_loss_values = history_dict['val_loss']\n",
        "epochs = range(1, len(loss_values) + 1)\n",
        "\n",
        "# \"bo\" is for \"blue dot\"\n",
        "plt.plot(epochs, loss_values, 'bo')\n",
        "# b+ is for \"blue crosses\"\n",
        "plt.plot(epochs, val_loss_values, 'b+')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b2c004f-1bab-4b38-b3bc-ba8774e464d1",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "8b2c004f-1bab-4b38-b3bc-ba8774e464d1",
        "outputId": "e8cccb5b-c66d-4a60-e566-7d1fc7c6e8b0"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEJCAYAAACOr7BbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUf0lEQVR4nO3dfdBedX3n8feHhJQnRUpSVwkhdBYr6WhB740POwoLKwLdwgjTShYVOgzRWtjdLnTE0VkwXceZaqetFbcbZ9HKUiiyK0O7CLIRVkdpy00hkYdFI6uQwK5hAV2kFYHv/nFO5PLml+TKw7mvO8n7NXPNfc7vPFzfXx7uz3XO71znpKqQJGmmfSZdgCRpbjIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUNGhAJDk5yQNJ1ie5pLH8iCRrkqxLcluSxSPLbkryZJK/GrJGSVLbYAGRZB5wOXAKsAxYkWTZjNU+AXy+ql4LrAI+NrLs48C7h6pPkrR18wfc93JgfVU9CJDkGuB04L6RdZYB/7afvhW4fvOCqlqT5Phx32zhwoW1dOnSnSxZkvYud95552NVtai1bMiAOAx4eGR+A/CGGeusBc4A/hh4B/CSJIdW1f/d3jdbunQp09PTO1qrJO2VknxvS8smPUh9MXBckruA44CNwHPjbpxkZZLpJNObNm0aqkZJ2isNGRAbgcNH5hf3bT9VVY9U1RlVdSzwob7tyXHfoKpWV9VUVU0tWtQ8QpIk7aAhA+IO4KgkRyZZAJwF3DC6QpKFSTbX8EHgigHrkSRth8ECoqqeBS4AbgbuB66tqnuTrEpyWr/a8cADSb4FvBz46Obtk3wN+AJwYpINSd4+VK2SpBfLnnK776mpqXKQWpK2T5I7q2qqtWzSg9TSXuGyyyZdgbT9DAhpFnzkI5OuQNp+BoQkqcmAkAZy2WWQdC94YdrTTdpdOEgtzYIE9pD/atrDOEgtSdpuBoQ0Cy69dNIVSNvPgJBmgeMO2h0ZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJaho0IJKcnOSBJOuTXNJYfkSSNUnWJbktyeKRZeck+Xb/OmfIOiVJLzZYQCSZB1wOnAIsA1YkWTZjtU8An6+q1wKrgI/12/48cCnwBmA5cGmSQ4aqVZL0YkMeQSwH1lfVg1X1DHANcPqMdZYBX+mnbx1Z/nbglqp6vKqeAG4BTh6wVknSDEMGxGHAwyPzG/q2UWuBM/rpdwAvSXLomNuSZGWS6STTmzZt2mWFS5ImP0h9MXBckruA44CNwHPjblxVq6tqqqqmFi1aNFSNkrRXmj/gvjcCh4/ML+7bfqqqHqE/gkhyEHBmVT2ZZCNw/IxtbxuwVknSDEMeQdwBHJXkyCQLgLOAG0ZXSLIwyeYaPghc0U/fDJyU5JB+cPqkvk2SNEsGC4iqeha4gO4X+/3AtVV1b5JVSU7rVzseeCDJt4CXAx/tt30c+D26kLkDWNW3SZJmSapq0jXsElNTUzU9PT3pMiRpt5Lkzqqaai2b9CC1JGmOMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1bTMgkvxakh0KkiQnJ3kgyfoklzSWL0lya5K7kqxLcmrfviDJZ5N8M8naJMfvyPtLknbcOL/43wl8O8nvJ3n1uDtOMg+4HDgFWAasSLJsxmofBq6tqmOBs4BP9+3nA1TVa4C3AX+woyElSdox2/ylW1XvAo4FvgN8LsntSVYmeck2Nl0OrK+qB6vqGeAa4PSZuwde2k8fDDzSTy8DvtK///eBJ4GpMfojSdpFxvpUXlU/BK6j+yX/CuAdwN8luXArmx0GPDwyv6FvG3UZ8K4kG4Abgc37WwuclmR+kiOB1wOHj1OrJGnXGGcM4rQkXwRuA/YFllfVKcCvABft5PuvAD5XVYuBU4Er+1NJV9AFyjTwR8A3gOcata1MMp1ketOmTTtZiiRp1Pwx1jkT+MOq+upoY1U9neS8rWy3kZ/91L+4bxt1HnByv7/bk+wHLOxPK/3O5pWSfAP41sw3qKrVwGqAqampGqMvkqQxjXOK6TLgbzfPJNk/yVKAqlqzle3uAI5KcmSSBXSD0DfMWOch4MR+v0cD+wGbkhyQ5MC+/W3As1V13zgdkiTtGuMExBeA50fmn+vbtqqqngUuAG4G7qe7WuneJKuSnNavdhFwfpK1wNXAuVVVwC/QjXHcD3wAePe4HZIk7RrjnGKa31+FBEBVPdMfEWxTVd1IN/g82vbvRqbvA/5pY7vvAr80zntIkoYxzhHEppFP/CQ5HXhsuJIkSXPBOEcQ7wOuSvIpIHSXrr5n0KokSRO3zYCoqu8Ab0xyUD//1OBVSZImbpwjCJL8KvDLwH5JAKiqVQPWJUmasHG+KPendPdjupDuFNOvA0cMXJckacLGGaR+c1W9B3iiqj4CvAl41bBlSZImbZyA+If+59NJXgn8hO5+TJKkPdg4YxB/meRlwMeBv6O7A+tnBq1KkjRxWw2I/sZ5a6rqSeC/JPkrYL+q+sGsVCdJmpitnmKqqufpHvqzef7HhoMk7R3GGYNYk+TMbL6+VZK0VxgnIN5Ld3O+Hyf5YZL/l+SHA9clSZqwcb5Jva1Hi0qS9kDbDIgkb221z3yAkCRpzzLOZa6/OzK9H7AcuBM4YZCKJElzwjinmH5tdD7J4XTPiZYk7cHGGaSeaQNw9K4uRJI0t4wzBvEndN+ehi5QjqH7RrUkaQ82zhjE9Mj0s8DVVfX1geqRJM0R4wTEdcA/VNVzAEnmJTmgqp4etjRJ0iSN9U1qYP+R+f2B/z5MOZKkuWKcgNhv9DGj/fQBw5UkSZoLxgmIHyV53eaZJK8H/n64kiRJc8E4YxD/BvhCkkfoHjn6j+geQSpJ2oON80W5O5K8GvilvumBqvrJsGVJkiZtm6eYkvw2cGBV3VNV9wAHJXn/8KVJkiZpnDGI8/snygFQVU8A5w9XkiRpLhgnIOaNPiwoyTxgwXAlSZLmgnEGqW8C/iLJf+zn3wt8abiSJElzwTgB8QFgJfC+fn4d3ZVMkqQ92DZPMVXV88DfAN+lexbECcD9w5YlSZq0LR5BJHkVsKJ/PQb8BUBV/bPZKU2SNElbO8X0P4GvAf+iqtYDJPmdWalKkjRxWzvFdAbwKHBrks8kOZHum9SSpL3AFgOiqq6vqrOAVwO30t1y4xeS/IckJ81WgZKkyRhnkPpHVfXn/bOpFwN30V3ZJEnag23XM6mr6omqWl1VJw5VkCRpbtiugNheSU5O8kCS9UkuaSxfkuTWJHclWZfk1L593yR/luSbSe5P8sEh65QkvdhgAdHfkuNy4BRgGbAiybIZq30YuLaqjgXOAj7dt/868HNV9Rrg9cB7kywdqlZJ0osNeQSxHFhfVQ9W1TPANcDpM9Yp4KX99MHAIyPtByaZT/eI02eAHw5YqyRphiED4jDg4ZH5DX3bqMuAdyXZANwIXNi3Xwf8iO4y24eAT1TV4zPfIMnKJNNJpjdt2rSLy5ekvdugYxBjWAF8rqoWA6cCVybZh+7o4znglcCRwEVJfnHmxv2A+VRVTS1atGg265akPd6QAbEROHxkfnHfNuo84FqAqrod2A9YCPxL4Kaq+klVfR/4OjA1YK2SpBmGDIg7gKOSHJlkAd0g9A0z1nkIOBEgydF0AbGpbz+hbz8QeCPdrT8kSbNksICoqmeBC4Cb6e7+em1V3ZtkVZLT+tUuAs5Psha4Gji3qoru6qeDktxLFzSfrap1Q9UqSXqxdL+Pd39TU1M1PT096TIkabeS5M6qap7Cn/QgtSRpjjIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqSmQQMiyclJHkiyPskljeVLktya5K4k65Kc2refneTukdfzSY4ZslZJ0s8aLCCSzAMuB04BlgErkiybsdqHgWur6ljgLODTAFV1VVUdU1XHAO8G/ldV3T1UrZKkFxvyCGI5sL6qHqyqZ4BrgNNnrFPAS/vpg4FHGvtZ0W8rSZpF8wfc92HAwyPzG4A3zFjnMuDLSS4EDgT+eWM/7+TFwQJAkpXASoAlS5bsZLmSpFGTHqReAXyuqhYDpwJXJvlpTUneADxdVfe0Nq6q1VU1VVVTixYtmp2KJWkvMWRAbAQOH5lf3LeNOg+4FqCqbgf2AxaOLD8LuHrAGiVJWzBkQNwBHJXkyCQL6H7Z3zBjnYeAEwGSHE0XEJv6+X2A38DxB0maiMECoqqeBS4Abgbup7ta6d4kq5Kc1q92EXB+krV0RwrnVlX1y94KPFxVDw5VoyRpy/LC7+Pd29TUVE1PT0+6DEnarSS5s6qmWssmPUgtSZqjDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQhrQVVfB0qWwzz7dz6uumnRF0vjmT7oAaU911VWwciU8/XQ3/73vdfMAZ589ubqkcXkEIQ3kQx96IRw2e/rprl3aHRgQ0kAeemj72qW5xoCQBrJkyfa1S3ONASEN5KMfhQMO+Nm2Aw7o2qXdgQEhDeTss2H1ajjiCEi6n6tXO0Ct3YdXMUkDOvtsA0G7L48gJElNBoQkqcmAkCQ1GRCSpCYDQpLUlKqadA27RJJNwPcmXccOWAg8NukiZpl93jvY593DEVW1qLVgjwmI3VWS6aqamnQds8k+7x3s8+7PU0ySpCYDQpLUZEBM3upJFzAB9nnvYJ93c45BSJKaPIKQJDUZEANKcnKSB5KsT3JJY/kRSdYkWZfktiSLR5YtSfLlJPcnuS/J0tmsfUftZJ9/P8m9fZ8/mSSzW/32S3JFku8nuWcLy9P3ZX3f59eNLDsnybf71zmzV/XO2dE+Jzkmye393/G6JO+c3cp33M78PffLX5pkQ5JPzU7Fu0hV+RrgBcwDvgP8IrAAWAssm7HOF4Bz+ukTgCtHlt0GvK2fPgg4YNJ9GrLPwJuBr/f7mAfcDhw/6T6N0ee3Aq8D7tnC8lOBLwEB3gj8Td/+88CD/c9D+ulDJt2fgfv8KuCofvqVwKPAyybdnyH7PLL8j4E/Bz416b5sz8sjiOEsB9ZX1YNV9QxwDXD6jHWWAV/pp2/dvDzJMmB+Vd0CUFVPVdWMpxvPSTvcZ6CA/eiC5eeAfYH/M3jFO6mqvgo8vpVVTgc+X52/Bl6W5BXA24FbqurxqnoCuAU4efiKd96O9rmqvlVV3+738QjwfaD5Ba25Zif+nknyeuDlwJeHr3TXMiCGcxjw8Mj8hr5t1FrgjH76HcBLkhxK90nryST/NcldST6eZN7gFe+8He5zVd1OFxiP9q+bq+r+geudDVv6Mxnnz2p3tc2+JVlO92HgO7NY15CafU6yD/AHwMUTqWonGRCTdTFwXJK7gOOAjcBzdA9yeku//J/QnbI5d0I17mrNPif5x8DRwGK6/2wnJHnL5MrUUPpP1lcCv1lVz0+6noG9H7ixqjZMupAd4RPlhrMROHxkfnHf9lP9YfYZAEkOAs6sqieTbADurqoH+2XX053X/E+zUfhO2Jk+nw/8dVU91S/7EvAm4GuzUfiAtvRnshE4fkb7bbNW1bC2+O8gyUuB/wZ8qD8Vs6fYUp/fBLwlyfvpxhIXJHmqql50Acdc5BHEcO4AjkpyZJIFwFnADaMrJFnYH4ICfBC4YmTblyXZfH72BOC+Wah5Z+1Mnx+iO7KYn2RfuqOLPeEU0w3Ae/qrXN4I/KCqHgVuBk5KckiSQ4CT+rY9QbPP/b+JL9Kdq79usiXucs0+V9XZVbWkqpbSHT1/fncJB/AIYjBV9WySC+j+088Drqiqe5OsAqar6ga6T5AfS1LAV4Hf7rd9LsnFwJr+Us87gc9Moh/bY2f6DFxHF4TfpBuwvqmq/nK2+7C9klxN16eF/ZHfpXQD7FTVnwI30l3hsh54GvjNftnjSX6PLlQBVlXV1gZB54wd7TPwG3RXAx2a5Ny+7dyqunvWit9BO9Hn3ZrfpJYkNXmKSZLUZEBIkpoMCElSkwEhSWoyICRJTQaEtA1Jnkty98hrl13HnmTplu4QKk2a34OQtu3vq+qYSRchzTaPIKQdlOS76Z5h8c0kf9vfT2rzUcFX+ucCrEmypG9/eZIvJlnbv97c72peks/0z0n4cpL9+/X/VbpngaxLcs2Euqm9mAEhbdv+M04xjT7o5gdV9RrgU8Af9W1/AvxZVb0WuAr4ZN/+SeB/VNWv0D1b4N6+/Sjg8qr6ZeBJ4My+/RLg2H4/7xuqc9KW+E1qaRv6m6sd1Gj/LnBCVT3Y3z/qf1fVoUkeA15RVT/p2x+tqoVJNgGLq+rHI/tYSvdciKP6+Q8A+1bVv09yE/AUcD1w/eYbGUqzxSMIaefUFqa3x49Hpjff7h3gV4HL6Y427kjimKFmlQEh7Zx3jvy8vZ/+Bt2dbAHO5oVblq8BfgsgybwkB29pp/0dbw+vqluBDwAH090uWpo1fiKRtm3/JKN3HL1p5JbNhyRZR3cUsKJvuxD4bJLfBTbxwp09/zWwOsl5dEcKv0X39LyWecB/7kMkwCer6sld1iNpDI5BSDuoH4OYqqrHJl2LNARPMUmSmjyCkCQ1eQQhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1PT/Abugmul2HhH3AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.clf()   # clear figure\n",
        "acc_values = history_dict['accuracy']\n",
        "val_acc_values = history_dict['val_accuracy']\n",
        "\n",
        "plt.plot(epochs, acc_values, 'bo')\n",
        "plt.plot(epochs, val_acc_values, 'b+')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "cba02bab-5e7a-4647-9ace-06b8980807da",
      "metadata": {
        "id": "cba02bab-5e7a-4647-9ace-06b8980807da"
      },
      "source": [
        "# Fully Connected Model¶\n",
        "Neurons in a fully connected layer have full connections to all activations in the previous layer, as seen in regular Neural Networks. Adding another Dense Layer to model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c3de4f6a-c9c9-4635-a03e-f82281ba2701",
      "metadata": {
        "id": "c3de4f6a-c9c9-4635-a03e-f82281ba2701"
      },
      "outputs": [],
      "source": [
        "def get_fc_model():\n",
        "    model = Sequential([\n",
        "        Lambda(standardize, input_shape=(28,28,1)),\n",
        "        Flatten(),\n",
        "        Dense(512, activation='relu'),\n",
        "        Dense(10, activation='softmax')\n",
        "        ])\n",
        "    model.compile(optimizer='Adam', loss='categorical_crossentropy',\n",
        "                  metrics=['accuracy'])\n",
        "    return model\n",
        "\n",
        "fc = get_fc_model()\n",
        "fc.optimizer.lr=0.01"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b29c4dc7-64ea-47f2-9640-5267515b5474",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b29c4dc7-64ea-47f2-9640-5267515b5474",
        "outputId": "8e34de67-179f-4cf6-f78f-6ec919b8de7a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  586/37800 [..............................] - ETA: 4:52 - loss: 0.3346 - accuracy: 0.9116WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 37800 batches). You may need to use the repeat() function when building your dataset.\n",
            "WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 4200 batches). You may need to use the repeat() function when building your dataset.\n",
            "37800/37800 [==============================] - 6s 136us/step - loss: 0.3337 - accuracy: 0.9117 - val_loss: 0.2057 - val_accuracy: 0.9395\n"
          ]
        }
      ],
      "source": [
        "history=fc.fit(batches, \n",
        "             steps_per_epoch=batches.n, \n",
        "             epochs=1, \n",
        "             validation_data=val_batches, \n",
        "             validation_steps=val_batches.n\n",
        "            )"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2f06cbd5-34b5-49f1-8d69-fc2e59b3ab17",
      "metadata": {
        "id": "2f06cbd5-34b5-49f1-8d69-fc2e59b3ab17"
      },
      "source": [
        "# Convolutional Neural Network\n",
        "CNNs are extremely efficient for images."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0726225e-2e79-4918-8775-88f99f1013a2",
      "metadata": {
        "id": "0726225e-2e79-4918-8775-88f99f1013a2"
      },
      "outputs": [],
      "source": [
        "from keras.layers import Convolution2D, MaxPooling2D\n",
        "\n",
        "def get_cnn_model():\n",
        "    model = Sequential([\n",
        "        Lambda(standardize, input_shape=(28,28,1)),\n",
        "        Convolution2D(32,(3,3), activation='relu'),\n",
        "        Convolution2D(32,(3,3), activation='relu'),\n",
        "        MaxPooling2D(),\n",
        "        Convolution2D(64,(3,3), activation='relu'),\n",
        "        Convolution2D(64,(3,3), activation='relu'),\n",
        "        MaxPooling2D(),\n",
        "        Flatten(),\n",
        "        Dense(512, activation='relu'),\n",
        "        Dense(10, activation='softmax')\n",
        "        ])\n",
        "    model.compile(Adam(), loss='categorical_crossentropy',\n",
        "                  metrics=['accuracy'])\n",
        "    return model\n",
        "\n",
        "model= get_cnn_model()\n",
        "model.optimizer.lr=0.01"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "de939f8d-228e-42f6-b71e-637b382fe9db",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "de939f8d-228e-42f6-b71e-637b382fe9db",
        "outputId": "a168ca46-b8d1-4e7e-977e-8f13253bb73b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  591/37800 [..............................] - ETA: 1:34:22 - loss: 0.2696 - accuracy: 0.9181WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 37800 batches). You may need to use the repeat() function when building your dataset.\n",
            "WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 4200 batches). You may need to use the repeat() function when building your dataset.\n",
            "37800/37800 [==============================] - 93s 2ms/step - loss: 0.2696 - accuracy: 0.9181 - val_loss: 0.2009 - val_accuracy: 0.9386\n"
          ]
        }
      ],
      "source": [
        "history=model.fit(batches, \n",
        "                            steps_per_epoch=batches.n, \n",
        "                            epochs=1, \n",
        "                            validation_data=val_batches, \n",
        "                            validation_steps=val_batches.n\n",
        "                           )"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f52a2366-e7e5-47ca-b88f-73ee103a7b08",
      "metadata": {
        "id": "f52a2366-e7e5-47ca-b88f-73ee103a7b08"
      },
      "source": [
        "# Data Augmentation\n",
        "It is tehnique of showing slighly different or new images to neural network to avoid overfitting. And to achieve better generalization. In case you have very small dataset, you can use different kinds of data augmentation techniques to increase your data size. Neural networks perform better if you provide them more data.\n",
        "\n",
        "Different data aumentation techniques are as follows:\n",
        "\n",
        "1. Cropping\n",
        "2. Rotating\n",
        "3. Scaling\n",
        "4. Translating\n",
        "5. Flipping\n",
        "6. Adding Gaussian noise to input images etc."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dd25bb95-1af4-4ecf-9989-d74ed2cb2a11",
      "metadata": {
        "id": "dd25bb95-1af4-4ecf-9989-d74ed2cb2a11"
      },
      "outputs": [],
      "source": [
        "gen =ImageDataGenerator(rotation_range=8, \n",
        "                        width_shift_range=0.08, \n",
        "                        shear_range=0.3,\n",
        "                        height_shift_range=0.08, \n",
        "                        zoom_range=0.08\n",
        "                       )\n",
        "batches = gen.flow(X_train, y_train, batch_size=64)\n",
        "val_batches = gen.flow(X_val, y_val, batch_size=64)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e6d30df8-b8f9-4e77-8d18-db78eeac6652",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e6d30df8-b8f9-4e77-8d18-db78eeac6652",
        "outputId": "d8884d4b-3f4f-47f7-ac94-31d26fbbd83c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  591/37800 [..............................] - ETA: 1:43:31 - loss: 0.1841 - accuracy: 0.9415WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 37800 batches). You may need to use the repeat() function when building your dataset.\n",
            "WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 4200 batches). You may need to use the repeat() function when building your dataset.\n",
            "37800/37800 [==============================] - 104s 3ms/step - loss: 0.1841 - accuracy: 0.9415 - val_loss: 0.1456 - val_accuracy: 0.9583\n"
          ]
        }
      ],
      "source": [
        "model.optimizer.lr=0.001\n",
        "history=model.fit(batches, \n",
        "                            steps_per_epoch=batches.n, \n",
        "                            epochs=1, \n",
        "                            validation_data=val_batches, \n",
        "                            validation_steps=val_batches.n\n",
        "                           )"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "beea23a6-1d51-4cd3-9977-62416133d285",
      "metadata": {
        "id": "beea23a6-1d51-4cd3-9977-62416133d285"
      },
      "source": [
        "# Adding Batch Normalization\n",
        "BN helps to fine tune hyperparameters more better and train really deep neural networks."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "04931b3f-1e19-4bcc-af58-4a1b6b949c70",
      "metadata": {
        "id": "04931b3f-1e19-4bcc-af58-4a1b6b949c70"
      },
      "outputs": [],
      "source": [
        "# from keras.layers.normalization import BatchNormalization\n",
        "from tensorflow.keras.layers import BatchNormalization\n",
        "\n",
        "def get_bn_model():\n",
        "    model = Sequential([\n",
        "        Lambda(standardize, input_shape=(28,28,1)),\n",
        "        Convolution2D(32,(3,3), activation='relu'),\n",
        "        BatchNormalization(axis=1),\n",
        "        Convolution2D(32,(3,3), activation='relu'),\n",
        "        MaxPooling2D(),\n",
        "        BatchNormalization(axis=1),\n",
        "        Convolution2D(64,(3,3), activation='relu'),\n",
        "        BatchNormalization(axis=1),\n",
        "        Convolution2D(64,(3,3), activation='relu'),\n",
        "        MaxPooling2D(),\n",
        "        Flatten(),\n",
        "        BatchNormalization(),\n",
        "        Dense(512, activation='relu'),\n",
        "        BatchNormalization(),\n",
        "        Dense(10, activation='softmax')\n",
        "        ])\n",
        "    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ecdd2dce-56ac-4c5d-b958-b991d4270953",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ecdd2dce-56ac-4c5d-b958-b991d4270953",
        "outputId": "8af9216d-03d5-4cdf-8184-2d484605d066"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  591/37800 [..............................] - ETA: 2:21:12 - loss: 0.2288 - accuracy: 0.9351WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 37800 batches). You may need to use the repeat() function when building your dataset.\n",
            "WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 4200 batches). You may need to use the repeat() function when building your dataset.\n",
            "37800/37800 [==============================] - 140s 4ms/step - loss: 0.2288 - accuracy: 0.9351 - val_loss: 0.2016 - val_accuracy: 0.9548\n"
          ]
        }
      ],
      "source": [
        "model= get_bn_model()\n",
        "model.optimizer.lr=0.01\n",
        "history=model.fit(batches, \n",
        "                  steps_per_epoch=batches.n, \n",
        "                  epochs=1, \n",
        "                  validation_data=val_batches, \n",
        "                  validation_steps=val_batches.n\n",
        "                 )"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "12a2589d-272c-4e40-be1e-7af9cca9c602",
      "metadata": {
        "id": "12a2589d-272c-4e40-be1e-7af9cca9c602"
      },
      "source": [
        "# Get the Predictions\n",
        "Make sure you use full train dataset here to train model and predict on test set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "da059b29-1976-4855-8658-d278c5f0c81f",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "da059b29-1976-4855-8658-d278c5f0c81f",
        "outputId": "d6c51ad9-43f5-49e1-dd7c-2b793cfbf58e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/3\n",
            "  657/42000 [..............................] - ETA: 2:27:53 - loss: 0.0537 - accuracy: 0.9840WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 126000 batches). You may need to use the repeat() function when building your dataset.\n",
            "42000/42000 [==============================] - 141s 3ms/step - loss: 0.0537 - accuracy: 0.9840\n"
          ]
        }
      ],
      "source": [
        "model.optimizer.lr=0.01\n",
        "gen = image.ImageDataGenerator()\n",
        "batches = gen.flow(X, y, batch_size=64)\n",
        "# history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3)\n",
        "history=model.fit(batches, steps_per_epoch=batches.n, epochs=3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "294c30a1-a09a-4717-8c25-c0b885175e44",
      "metadata": {
        "id": "294c30a1-a09a-4717-8c25-c0b885175e44"
      },
      "outputs": [],
      "source": [
        "# predictions = model.predict_classes(X_test, verbose=0)\n",
        "\n",
        "# This function were removed in TensorFlow version 2.6. \n",
        "# According to the keras in rstudio reference\n",
        "\n",
        "predictions=model.predict(X_test) \n",
        "classes_x=np.argmax(predictions,axis=1)\n",
        "\n",
        "submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n",
        "                          \"Label\": classes_x})\n",
        "\n",
        "submissions.to_csv(\"byscratch/data/mnist_DR.csv\", index=False, header=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "51f9aa4f-172b-4818-8c74-74d1cc3ed663",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "51f9aa4f-172b-4818-8c74-74d1cc3ed663",
        "outputId": "d0acaa98-2887-4596-bb60-fcaa7af57cda"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([2, 0, 9, ..., 3, 9, 2])"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ],
      "source": [
        "classes_x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c49df095-dd06-42a9-a603-72c3ac2a096d",
      "metadata": {
        "id": "c49df095-dd06-42a9-a603-72c3ac2a096d"
      },
      "outputs": [],
      "source": [
        "# for i in range(0, 9):\n",
        "#     plt.subplot(330 + (i+1))\n",
        "#     plt.imshow(X_train[i], cmap=plt.get_cmap('gray_r'))\n",
        "#     plt.title(y_train[i]);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2b718a7b-107a-4490-9779-72e18905e2a6",
      "metadata": {
        "id": "2b718a7b-107a-4490-9779-72e18905e2a6"
      },
      "outputs": [],
      "source": [
        "# image_index = 333\n",
        "# img = X_test[image_index].reshape(28, 28)\n",
        "# dig = pred.argmax()\n",
        "\n",
        "# plt.imshow(img,cmap='Greys')\n",
        "# pred = model.predict(X_test[image_index].reshape(1, 28, 28, 1))\n",
        "# print(dig)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bb4ecb1e-fbd8-4d0d-9922-72d330dec5d5",
      "metadata": {
        "id": "bb4ecb1e-fbd8-4d0d-9922-72d330dec5d5"
      },
      "outputs": [],
      "source": [
        "from IPython.display import display\n",
        "\n",
        "image_index = 333\n",
        "\n",
        "image_set = []\n",
        "image_indices = []\n",
        "\n",
        "def compareImgPred(idx):\n",
        "    imgsh = X_test[idx].reshape(28, 28)\n",
        "    image_set.append(imgsh)\n",
        "    image_indices.append(idx)\n",
        "    pred = model.predict(X_test[idx].reshape(1, 28, 28, 1))\n",
        "    dig = pred.argmax()\n",
        "\n",
        "    # img = plt.imshow(imgsh,cmap='Greys')\n",
        "    # display(dig)\n",
        "    # display(img)\n",
        "    # plt.show()\n",
        "    \n",
        "for n in range(500,520):\n",
        "    compareImgPred(n)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6b33173b-4889-43cc-ba2c-ca42bb3d4e41",
      "metadata": {
        "id": "6b33173b-4889-43cc-ba2c-ca42bb3d4e41"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import cv2\n",
        "import json\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "def read_this(image_file, gray_scale=False):\n",
        "    image_src = cv2.imread(image_file)\n",
        "    if gray_scale:\n",
        "        image_src = cv2.cvtColor(image_src, cv2.COLOR_BGR2GRAY)\n",
        "    else:\n",
        "        image_src = cv2.cvtColor(image_src, cv2.COLOR_BGR2RGB)\n",
        "    return image_src\n",
        "\n",
        "def resize_image(image_matrix, nh, nw):\n",
        "    image_size = image_matrix.shape\n",
        "    oh = image_size[0]\n",
        "    ow = image_size[1]   \n",
        "\n",
        "    re_image_matrix = np.array([\n",
        "        np.array([image_matrix[(oh*h // nh)][(ow*w // nw)] for w in range(nw)]) \n",
        "        for h in range(nh)\n",
        "    ])\n",
        "\n",
        "    return re_image_matrix\n",
        "\n",
        "def concat_lib(image_set, how, with_plot=False, image_indices=[]):\n",
        "    # list for predictions\n",
        "    predcts = []\n",
        "    # dimension of each matrix in image_set\n",
        "    shape_vals = [imat.shape for imat in image_set]\n",
        "\n",
        "    # length of dimension of each matrix in image_set\n",
        "    shape_lens = [len(ishp) for ishp in shape_vals]\n",
        "\n",
        "    # if all the images in image_set are read in same mode\n",
        "    channel_flag = True if len(set(shape_lens)) == 1 else False\n",
        "\n",
        "    if channel_flag:\n",
        "        ideal_shape = max(shape_vals)\n",
        "        images_resized = [\n",
        "            # function call to resize the image\n",
        "            resize_image(image_matrix=imat, nh=ideal_shape[0], nw=ideal_shape[1]) \n",
        "            if imat.shape != ideal_shape else imat for imat in image_set\n",
        "        ]\n",
        "    # else:\n",
        "    #     return False\n",
        "\n",
        "    for i,img in enumerate(image_set):\n",
        "        # model.predict(X_test[idx].reshape(1, 28, 28, 1))\n",
        "        idx = image_indices[i]\n",
        "        pred = model.predict(X_test[idx].reshape(1, 28, 28, 1))\n",
        "        dig = pred.argmax()\n",
        "        # print(dig)\n",
        "        predcts.append(dig)\n",
        "\n",
        "    # cv2 library code to concatenate the image matrices\n",
        "    # we use methods like\n",
        "    #     - vconcat() → vertical concat\n",
        "    #     - hconcat() → horizontal concat\n",
        "    if (how == 0) or (how == 'vertical'):\n",
        "        concats = cv2.vconcat(images_resized)\n",
        "    elif (how == 1) or (how == 'horizontal'):\n",
        "        concats = cv2.hconcat(images_resized)\n",
        "    else:\n",
        "        concats = cv2.hconcat(images_resized)\n",
        "\n",
        "    if with_plot:\n",
        "        cmap_val = None if len(concats.shape) == 3 else 'gray_r'\n",
        "        plt.figure(figsize=(10, 6))\n",
        "        plt.axis(\"off\")\n",
        "        plt.imshow(concats, cmap=cmap_val)\n",
        "        # return True\n",
        "\n",
        "\n",
        "    return concats, predcts\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "abac1c18-a09d-483a-9973-e51f989b22aa",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 59
        },
        "id": "abac1c18-a09d-483a-9973-e51f989b22aa",
        "outputId": "c86525b7-cb23-4379-9334-914f0c06bb50"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjwAAAAqCAYAAAC3H4E1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deVRUV57Hf6+soqqkWFMgHEGBEcTmmKCNsZxQKoxGOK5lK5HjigNGJoLodBQOElRGRI1GJRppW23DCRqXuNBRQEGWyCKtRBlNEDQqKEZDoSB7wXf+sOs1xSJQvNJu533OuSdELvd333bf9/3u73cvA4B4eHh4eHh4eN5mBG+6Azw8PDw8PDw8hoYXPDw8PDw8PDxvPbzg4eHh4eHh4Xnr4QUPDw8PDw8Pz1sPL3h4eHh4eHh43np4wcPDw8PDw8Pz1iPs4fd8zjoPDw8PDw/PvwpMd7/gPTw8PDw8PDw8bz284OHh4emR9PR0GjhwIG3YsOFNd4WHh4dHLzgXPC0tLVRTU0PR0dHEMAwxDEMJCQnU2trKtalu0Wg0dObMGZJKpbR169bXZpeH523l5MmT1NDQQOvXr6d169ZRdXX1m+4SDw8PT5/gVPA0NDRQfHw8WVhY0P/8z/+QQCAggUBA69ato9TUVC5NvZL6+nratm3ba7PHNRUVFbR69WqaPHkylZWVvbF+tLS00Ndff01OTk7EMAwtWbKEysvLOWu/ubmZCgoKaNu2bbR06VJycnKipUuXUmlpKWc2XifV1dX0X//1X2RmZkaWlpa0YMECWr58OX333XdUXFxMLS0tb7qLnLBp0yaKiop6093g4eHh6RsAXlV6RX19PW7fvo2dO3dCIBCwxdraGq6urvjuu+9621S/aGlpwblz5/DBBx+AiGBqaoqcnJzXYpsLGhsbkZycDGdnZwiFQowYMQJnz57tVO/27dtYsmQJ5HI5Fi5ciIcPHxqkP2lpaVAoFGAYBgzDYOLEiTh79iyeP3+ud5vNzc24f/8+9u/fjxkzZsDa2hoMw+jcN2FhYSgvL+fkGNRqNbKzs+Hv7w93d3dMnToVoaGhuHHjBjQaDSc27ty5g7CwMLi6usLIyIg9X+2LVCpFQkIC6urqOLHZFU1NTdi4cSPWrVuH2tpaTtt+/Pgx1q5dC0tLSxARLCwscOvWLU5t/DPR0tKC48ePY+zYsaCXyRsQiUTw9vbG559/jra2Nk7sNDU1IT8/Hzt37sTUqVMhl8uhVCrh4+ODpUuXdvn8/ytSVlaGsLAwyOVy9nxaW1tj586dnNtSq9X45ptvsHTpUjg6OrL2tGX69OnIy8vj7Pnn+QcxMTEQCARoamp6k93oVtP0W/DU1dXhz3/+s47IGT16NDw8PHDo0CHuD+XvaDQaXLx4EdHR0YiJicGFCxewZs0a9qYWCoWYOXOmwewDQGtrK0pLS7F06VLY2dnBxMQEH3/8MX799Ve92rt//z6WLVsGhmEgFAqhVCpRVFSkU+fXX3/F+vXrIRKJwDAMzMzMEBYWxsXh6HDhwgV4eXlBJpOxL+1BgwYhLi4O9fX1erVZUVGByMhI2NrasveLSCTCu+++C09PT1hbW0MgEMDJyQnJycn9PoZjx45BoVBAIpF0GvSCgoJw//79ftvQ2rGxsYGFhQWWLl2Kn3/+mR1Mq6qqkJiYCBcXFwiFQnz77bdobGzkxG5HQkJCIBaLYW9vz5lg7EhqaipsbGxAREhKSjKIjTdNS0sLjhw5AolEAoZhYGJiAgsLC9jY2MDFxQVWVlb485//3C8bra2tKCkpgZ+fHyQSCcRiMd577z34+/tj+/bt8Pf3h0KhgFgshru7Oz7++GPk5+f3qu0XL17gL3/5C4YPH441a9bgwIEDKCkp6Vd/9aWkpATR0dFwd3eHSCQCEbHjiUwmQ3h4OCfi/MaNG9i7dy/8/Pzg6OgIqVTa5YeHtnh4eCApKQnPnj3j4Ch1aWpqQmVlJR48eIAHDx7g0KFD2LRpE27cuIGWlha92rx58yaOHTuGxMREnbJ3715YWlpiw4YNqKmp4fQ4Nm/eDLFYjPDw8F4JmKysLPYa91XwlJaW4urVq1i7di3++Mc/siUhIQG//PJLX7tuOMFz9epVna/zoKCgvnauz9TX18PT07PTS+x1iZ1nz55h9+7d8PLygrOzs86DZGxsjHnz5vVZFKjVamzfvh1isRjGxsbw8/PDo0ePOtW7ffs25s+fD4ZhYGtri6ioKDx+/JirQ2MpLCzEtGnTIBAI2GMbMmQI9u3bp1d7jx8/RnBwMIyNjdl7xcPDA4cPH4ZarUZjYyPOnDmD999/H1KpFF988UW/BsJjx47B1dUVRASBQICRI0ciMDAQc+bMgVwuh5GREQ4fPsyJx+XYsWOwtbXFvHnzur0WkZGRMDMzQ2xsLOfeFy0TJkyASCTCe++9hydPnuDatWtITk7Gzp07kZGRgdbWVk7sREREgIgwd+5cTtoDXr6kvby8sHDhQvY5dnR0RGBgIAIDA3Ho0CHk5eVhy5YtCAwMRFBQEPvfoKAgTj251dXVsLe3h0QiwaZNm/DkyRP2d62trfjpp59w48YNvdtvbW3F9evX4erqCoFAADc3N1y9erWTx+HOnTsYOXIkey7y8vJ61X5+fr7Oh4pIJIJCoUB6errBxHZXPH/+HDExMRAKhWAYhr2u7cdLlUqF4uLiftn5+eefoVKpdN4BJiYmGD58ODw8PHTK4MGDIRQKQUR49913cfr0ac7OSWZmJmJjY+Ht7Q2ZTNbpvWRqagqFQoHIyEhUVlb22N7WrVuhUqkwdepUuLq6QiwWdyvgZDIZ9uzZw5kHOScnB8bGxiCiXnn0s7OzYWRkBCJCWlparzygTU1NKCoqwvz581lbXRUzMzNMmjQJ2dnZve3+2yN4mpqa2AG3Y5FIJIiJiTHYNFZzczPy8/Ph7u4OU1NT9mYTCoWsx4VhGLi5ueHBgwd9avvHH3/E6NGj2cEpKyury3qXLl2Cvb09jIyM4Ovri59++omLQ9MhPz8fc+bMgbm5uY7g8fLy6rZfPXHp0iWMHz+enb5SqVS4du2aTp28vDx4eXmBYRgEBQXhzp07eh9DUlISXFxcQETYsWMHqqurAbz0XLm5uUEikSAxMVFvb1VHtOKquzYnT54MsViMffv2cWazPe0HqB07drBfkqGhoXBwcIBIJEJoaCguXLiAp0+f9svW1atXwTAMp4KnqakJfn5+Ovfbq0r7555hGCxevJiTfrS2tiI7OxsikQhTp041yLTHw4cP4ebmBnNzc8TExLD3Znvq6upw+PBhWFhY9FnwXL58uZOw0L5wlUol1q5di9OnT2PZsmVQKBRsmTVrFk6cOMHZcd65cwdBQUFsHxwdHbFq1SokJSXh+PHjOH78OAoLC9HQ0NAvO1lZWfDy8oKnpyeio6Nx5MgR7NmzBwUFBZ3qfv/99/Dw8ADDMHjvvfc4EzyFhYV4//33O92bXRVLS8tOnvuuWLBgAYyNjTF9+nTWU/2qZ+LTTz/tV7hBe7Qfu8nJyT0+A9nZ2Zg0aRIYhkFqamqvp3sDAgI6nS8jIyMMGjSILSYmJjqCMTo6ujdNd6tpelp4sFsaGhror3/9K929e5eIiGxtbUmpVNLvf/97fZvskba2NsrMzKQffvihy99LJBJat24d53br6+vp3LlzdOTIETp16hQREQ0cOJD+/d//nfz8/EihUNAPP/xAf/zjH0kmk9GYMWPIwsKi1+0/fvyYjh49Srdu3aJhw4ZRVFQUjR8/vlO9lpYWqq6upubmZjI2NqZ3332XXF1dOTtOIqKHDx/St99+S3/961+pqamJiF6KYiKiIUOGkKOjo95ta2+69j8DIIZh6MmTJ3Tx4kW6c+cOyeVy+vDDD8nJyUlvW1OnTiVnZ2eqra2lYcOGkUwmIyKiR48eUX19PU2YMIE8PDxIKpXqbaM9+/fvp5CQEIqMjKQbN25QeHg4WVpaEhHR6tWrqaCggLy9vWn27Nmc2dRSWFhIn376KTU1NVFoaCj5+/uTUPjy0d61axft2rWLJkyYQF9++SXFx8fTxIkT6ezZs+w56St2dnZcdp+IiIyMjOjbb7+l8ePHd/t8t4dhmC5/7i+tra2UlZVFRkZGtHDhQhowYABnbWtpa2uj2tpaMjU1pY8++ojMzc11fq9Wq2nr1q2UkJBAH330ES1dupQ2b95MarW6T3YGDhxIzs7OZGJiQn/729+opqaGfvjhB/b8ap/F9ucPAP3hD3/o5xG+5OnTp3T//n0ievk8xsbG0siRIzlpuz3jx4+njIyMHusVFBTQ6dOnqby8nACQra0t2dnZkVgs7ncf/vd//5eePHnCnlNzc3NSqVRkY2NDAwcOJKlUStnZ2ZSTk0NTpkxhx4ZXYWVlRf/5n/9J//3f/00DBgygyspK0mg0OnW+/PJLOn36NNXX11NNTQ07ZveHLVu20MWLF6mtra1X9Tdt2kTp6emkVCpJIOh9HlR6ejrJ5XJasWIFCYVCGjt2LL3zzjvk7u7O1qmoqKCUlBTasmULlZWV0c6dO+k//uM/SKlU9vm4iEh/D8+jR49Yr46trS127dr1SsnV1taGlJQUHDhwAIcOHdJrvrG5uRnR0dFYt24dJk6cCF9f3z630VceP36MRYsWsR4dY2NjzJgxA4cPH2br1NbW4tChQ/D19UVubm6f54Xz8vIgl8vZuJ3ugkErKiqwatUqMAwDCwsLrF27tl/H1hUHDx6EpaUlhEIhe30ZhoGnpydSUlL0/uJ9/Pgx4uPj4ebmxrY7evRonDx5Eg0NDSgoKICXlxcEAgH8/Pz67ebuit9++w0qlQoSiQTr16/Xmarggjt37iAxMREXL15kvThTpkyBWCzGyJEjUVxczNm0UnuKi4thY2MDY2Nj7NixA83NzZ3qVFZWIjk5GRKJBAKBAKNGjeqyXm/4/vvvwTAMgoOD+9v1TqjVapSVleHOnTsoKyvDN998g4iIiE4lPj4etra2rCdj0aJFnNivq6uDh4cHTE1N9Y7F64mamhps2LABMpkMwcHBOtdBrVbjs88+g1wux6JFi/Do0SM0NjaiqKgIV65c6VX7Wg+PmZkZPvnkEzx79gzJyckICQmBiYlJJy9Zey8Bl6EADx8+xKpVqyAUCjFx4sS+TElwzrVr16BSqXRie/bs2cPZdNbx48dZjzL93SNXWFjYqd7Nmzfx4sULTmwCwJMnT+Dg4ACGYTBu3Djcvn273wH106dPh1AohKenZ4991U5ZEhEiIyP7FKOUkpLSax2wY8cO9ty2f/d2A/dTWu0Fj5eXl87vioqKsHv3bhQWFuLSpUvYvXs3SkpK4OPjwwaqbty4sU+u/YaGBiiVSjYmY8qUKUhJScHEiRMxadIkbN68uddt9YbGxkakpaWxN5NAIICDgwP+8pe/dHpRPHjwAP7+/li8eHGfXyLV1dX44osvIJFIYG1tjfXr13db95dffsHHH3/MBhBv2LBBr2PritbWVpSXl7OCqv00pampKdatW9fvB1Wj0aCgoACzZ8+GVCqFQCCAjY0NgoKC4O/vzwYtb926lfM4l6qqKsTHx2Po0KHw8vLCzZs3OW2/I2q1GosXL4axsTEYhjFYsHJlZSUUCgVEIhGmTZv2yukBjUaD2bNnQyQS9SuTIiAgACKRCD///LO+3e43ra2tGDJkCPvCjoyM5KTduro6uLq6GlTwAC+zloYMGQJLS0vs3bsXwMuXV1BQEMzMzLBo0SK9sy+rqqqwZ88eLF++vFNM2YMHDxAXF4dVq1YhJiYGubm5yM3NxfDhw0FEGDNmTK+Do3vD/v37IRaLMWLECHzzzTectdsXiouLMXfuXDZ5wcHBAX/60584m/4pLy9n301EBDs7O2zcuNEgsZUdaS94RowY0e+PqpycHDb+aO3ata8cs7KzsyEWi0FEUCqVBs3MunLlCgQCAYgI3t7ePVU3rOBxdHTE0aNHcevWLWzatAnTpk2DUChEXFwcQkJCIBKJcPz4cVbwCAQCiMXiPn1hv3jxose5UZlMxkn8TlVVFUJCQsAwDIyMjODq6org4OBOL0mNRoPi4mJMmzaNzZgKCQnptZ22tjZcvXoVHh4eEAgEUCgUKC0t7bb+xYsXYWlpCYZhMGHCBE69IFVVVYiMjGS/frTXSSqVIjg4uMsA6v7YunTpEqZNm8bGPmntzZ07t18BoV0RHx+PUaNGsQOevb09zp49a7AAzrS0NCiVSnYw8PHxwZo1a1BcXMxpTEhFRQVGjRrFBoX29LWk0WigUqkgFAoRFhamV19++uknGBsbY/bs2fp2mxNKSkpgZmYGhmFgbW2Ne/fucdKuVvC89957Bom10lJfX4/ExETY2trCxsYG0dHRCA8Ph7m5OVasWNHvjBuNRtPrj4bc3Fy4urpy7uEBXgbyenp6wsHBAQkJCZy23RO1tbVIS0tjYwdNTU3h4+OD3NxczmxoNBqEhobC3NwcRAR3d3fk5OS8tpT3/fv3Qy6XcxbD017w9BS/M2XKFNZTmJKSwqn3+unTp7h27RoOHjyITz75BCNGjGDf8xs3buzpz7kVPI2NjQgNDdXxAjg7O0OlUrH/LxQKsWXLFqSlpSEuLg6fffYZnJycDCp46O8ZWjNmzOh1ux159uwZ6+WQSCTw8fHBb7/91uU5+P7771l1zTAMrKys+uRpKi8vZ9PQra2tewzI0gqe3tTtCzU1NYiPj4eNjY2O4JFKpfj4449RUVHBma321NbWsoFr2vtix44d/Q5i7EhkZCQcHBxgamrKul+HDBliENGTlpYGZ2fnLoNv5XI5MjIy9E5N7ciYMWNYb01AQMArz9ujR4+gUqnY+nFxcX0elDUaDfz9/SGVSt/o+lZtbW0ICwtjvb0rV67krG2t4DExMcHu3bs5u1ZdUV9fj+PHj8PX15dN53VwcMD169cNZrMrDCl4srKyWMHh4OCAcePGwcfHB7GxsSgsLORsPaOOlJaWYv369ew6PC4uLti4cSNnwlhLSUkJFAqFzsu4v0kBfWHBggXsNB0XgicuLg4SiQSenp44f/58t2OEj48PK3YiIiI4eU4SExPh4OCAoUOHspm0Xb3jx48fj8uXL7+qKW4FT01NjY7YaV8++OADxMfHY8+ePbh69Sr7N+fPn0d8fDx2794Ne3v7Pgse4B8xPNoSHh7ODnrt09T1TUuvr6/HgQMH2GNxdXXtMtuqpaUF2dnZbFyPVvAdO3asT/bKy8sRFBQEIoKVlRU+++yzV9a/ePEiLCws2LpNTU3Izc2FSqXCyJEjcerUqT7Z15Kfn48PP/yw07y+qakpYmJi9GqzNxQXF8PX11cnjmD79u2cCx4tz58/x/nz5zFp0iRIJBIMHjwYycnJnIoebbaWRCKBSqXCrl27kJCQwK7DY2FhgXPnzukdP6PlxIkTbOqxi4vLK6c/9u/fryPChg8frtegfPLkSTAMg48++qhPLyqNRoMLFy5w9gWoVqtZz6BQKERmZiYn7QL/EDza+9HU1BQBAQGcejg78ssvv7BrGzk4OPQqg4crqqursXv3bnYs41rwJCcns+ezY8yQUqlEeno6p/aAl7Ehvr6+7DHZ2toiISGB09gZLdXV1VCpVKw3d/Lkya91QU6t4LGxsUFaWlq/x5XexO9s2rSJFehExJkntOPSAkQEGxsbODg4sMXU1JSNT+sqRurvvD7B86q09NbWVsTExMDS0lIvwdMd2gG1/cKDlpaWfW7nyZMnGD9+PEQiEcaOHcu+FFpbW1FZWYmMjAxERUVh7ty5cHFxYdc/WLhwoV7ztX3x8NTU1GDfvn3sS04sFsPExIRNVdSuKdFX7t+/j9WrV8PMzEzHK/HBBx/g9OnTnC9mpeX69euYOXMmRCIRxGIxbG1tYW5ubpAprY48ffoUoaGhsLCwgIuLS6f0+P4QHR2NrVu34urVqzqDz8OHDxEUFASZTAa5XN4vr1lqairs7e3ZhIGsrKwuB7rY2Fg4OjqygcoMwyAwMBD37t3Ty+U+btw4yGQyXLx4sdd/U1VVhffffx/Ozs6cTcHm5+frBIdy+SKrq6vD8OHDIZFIkJCQgPnz50MqlWLkyJEG8XQ2NDTgyJEjEIvFkMvlkMvlcHFxwVdffcW5ra6orKxERESEQYKWgX8Inu488kuWLOFUTH799ddsPJK2mJiYYPHixQZJhABeekW0glUgEGDLli2oqqoyiK32nDhxgvXKOzs7c7LivtbDQ0TdJt8YwrsDvFxPaeXKlVixYgXWrVuHy5cvd3r/FBYWwszMjP2o7GZRwtcjeJRKpY5XR0tbWxu2bduGOXPmwMrKSq8prd6gVqv1FjwajQb5+fkwNTWFQCCAlZUVFi5ciOXLl8PDw0Nn6qr911/HLIu+UFVVhbi4OIhEIpiamiIwMFDHJdnW1oYHDx7g8OHD2L59O7y9vTv1QSQSYcSIEdi7d2+fPSN1dXU4ePAgHBwcdDKyGIbBtGnT8OOPP+p1XD3x448/stlSNjY2CAkJwXfffYepU6dCIBDgwIEDBrHbnuvXr8PDwwNyuRzZ2dkGc613RLuybkJCgl5fRrdu3YKfnx/EYjEEAgEiIiJ0PFRNTU3IzMzE5MmT2SA/qVSK6dOnIz8/X+/jLCkpgUwmQ0RERK/qX79+HTExMewX2eeff66X3Y7U1NTA19eX9azu2LGDk3a11NfX48MPP2SDltva2lBaWgoXFxcoFApObQEvhbBSqcTgwYORkpKCGzduQKVSwcHBweCi5969e1i/fj28vb11xs2AgADk5OTg/v37/RYjJSUlCAgIgLGxMUaPHo1Vq1YhNDQUI0eOBMMwnHrPXrx4gbVr1+qskda+qFQqg4xpzc3NCA4OZl/E48aNM5i4ak9SUhK7NQ9XgicnJwc+Pj4QCoXdxvC0XySQ69id3hAfH89uUaJSqbqq8noEz+DBg/HFF1+w9Q4cOABfX1/4+PjAxsaGrWdkZISvv/66V1Hdra2tuHTpUq9SGrWCRyKRYMuWLT3W78jjx4+7FBVdFSMjI0ycOLHf87VFRUVwc3MDw7zcb8nJyQmurq5ssbOz6+QKlkgkcHV1xcqVK5Genq53NsDx48dZT4G22NvbIzQ0FLm5uZzv+9Tc3Iy8vDwsWbKEFb7btm1DQ0MDbt68yangKSkpwYEDB3D37t1u6wQGBkImkyEmJqbLOK2+kpaW1mOgaFZWFmxsbPDuu+/2+bo1NTUhOjqaXa166tSp7DWqrKzE559/jilTpsDIyEhn+oqLr7AdO3aAYRhERUV1W6exsRHZ2dmYOXOmznYen3zySb9d7VqysrJ0Pji4frG0tLRg7969MDU1Zbce0XqQBw8ezOm+dW1tbfj5559hY2MDX19f9hw9ffoUYWFhGD16dJeL53FBWVkZFixY0CktXRsSYGpqCmtrawwaNKjfyw88ffoUN2/eZMfK+/fvs9mmXE8XpqamIiAgAGvXrsX27dvh4+PDLl6nUCg4nf5sT1RUFKysrF6b4GlsbMSwYcPY68aV4AHABmB3zNLatGmTTlzNmxA7WkJDQ9l+dAG3gkej0eDs2bNdih4bGxsolUp4enp2eplqU9ITExN7HTfR2NgILy8vbNq0qds6Go0G58+fZzcN1VfwaLOufH19oVQq2eLr64sdO3YgLCyMFRxeXl6cpG82NzcjOTkZ5ubm3YqrjoKHi6Dle/fuITg4uFMKur29Pb744guDpBjevHkTs2bNYm0tWrSITW3+6quvWE8TF4InNzcXHh4eCAgI6Fb0cCV41Go1fHx8sGvXrh6DBouLi9n1Y/oay5Odnc2ufi0QCHDw4EEcO3YMsbGxcHFxYUWOQCCAr68vsrOzOQugDA4OZjPcYmJicOvWLbZ8/fXXiIqKwowZM3SmEoyNjXHy5EnOxA7wct0P7XMwf/58ztrV0traioKCApiYmOisLfbo0SNIpVK9xpXuUKvVCAkJgUwmw4oVK3R+V1FRgVmzZsHb29sg6fENDQ04e/Ys3NzcdDa87RjD5+PjgzNnznBq+9y5c2zW5MqVK6FWqzltvz27d+/GkCFDwDCMwQRPcXExvLy8IBKJIBKJsH379i5Xz+aSjoJn586dnE3tauN4iAienp6oq6tjt8zQeo2VSqXesY/FxcW4ePEiLly4gLKysj79bVZWFpYsWcLeq+7u7l1VM2xael+KkZFRn7IQ6urqWBHT0cuj0WiQnJzMaZZWV2iDlLUPjr29PS5cuMBZ+8+fP8c333yDKVOm6Cz1ri0LFy5EUlISZsyYAYZhMHToUOzevVtve6WlpQgICOgkpOzs7BAfH8/ZcbWnrKwMq1atYueclUolu01FRkYGPvjgAzYWKTU1td/2/va3v2Hs2LEwNzfHkSNHOk33NTc3Y/HixZDJZNi2bZveg65arYZKpYKbmxtu3rzZ45RRcXExBg8eDIZh+ix4hg4dqvMsdVxCgGEY+Pr64vLly5yKDOAfgqe3RaFQcL5OT21tLZuKLhQKDfYV/eLFCzg4OMDOzo712D169AgSiQTh4eGc2Xn69CkWLlwIiUTSKc2/qqoKERERcHR05DTGrCO3b9/GvHnz2HtJKpVi0qRJOHz4MNLT0zmPRSkpKcGSJUsgFAoxduxYnDt3rs9tNDc3Y/PmzVi2bFm375La2lpkZmay03UmJiZYtWqV3llaiYmJiIuL67TdTUZGBrsOFhFh5syZr2Wj1o6C5/vvv+fsI7V9HE9XJSIiol/jS/spMalUCisrK3bT8ZUrVyIxMREFBQUoKytDQUEBCgoKsH37digUCh1R7uDg0J3o4l7wtLS0ID09vU9iRyQS4cSJE326MI2NjeyiTgzDwNvb+5V7acXFxfW67d7y/PlzxMbGgmFerrQ8f/58gy6y1BX37t3D8uXLwTAMnJyc9N6tuaSkBIGBgTr7smhjqubMmWOwBfmSk5MxZswYMAwDV1dXJCUlobGxEXfv3sWyZctgbm4OExMT7Nu3j7OsKW1G4MiRI5GZmakzrZORkQE3NzdYWVn1y1NXXFwMJycnTJ48GaWlpd0KnocPH+L48eNQKBQwMis9M5sAAAeuSURBVDLCp59+2ueAcO2+Su2/xsViMezs7DB9+nTk5uYazMWck5PDurq7KzKZDP7+/gZbUXfPnj3ssQuFwj7vV9dbGhsbERYWBolEgs2bN0Oj0eDSpUuQSqWcLnBaV1eHr776ChKJBDNnztS5dk1NTTh37pzBBY8hg5abm5t1PjR+++03xMbGsqvKh4WF6RXHVl1djdWrV2Pw4MGIiYnpJMrKysqwefNmDB8+nD2uefPm9Usgz5o1C1KpFFu3bsX169eRkpKCuXPn6jwTH374IW7cuPFa4gE3b97MzgrMnj0b5eXlnNrV7pTe8RlXKpX9zqIdOnRorz6a2u+j1b5YWVkhMjLyVTHA3WoavffSEgqFNH78eEpNTaUpU6a8su6AAQMoISGBpk2bRubm5iQSiXptRyAQkKenJ+Xk5BAAysjI6HLfFJlMRufPnydPT88+H8uraGxspJSUFNq6dSsNHDiQZs+eTV9++SUZGRlxaqcnBgwYQFKplAYMGEA1NTVUVlZGGo2G3TOpN6jVajpz5gydPHmSmpub2X9/5513aMGCBbR69WqD7JNUWlpKSUlJ9OOPPxIRkUKhoJs3b9K3335LRUVFVF5eTr///e8pJCSEpk+fzsneNkRE8+bNo1OnTtGlS5fo6NGj5OjoSEOGDKHm5mY6evQoPXjwgFasWNGv/cHeeecdsra2pszMTPL19SVvb2+aNWsW/frrr2Rvb0/l5eWUnJxMV65coV9//ZU0Gg05OzvT6tWrycTEpE+2Ro0aRT/88AO7n87vfvc72rp1K02ZMsUgez61x9PTk3Jzcyk+Pp4KCgro2rVrRPTyWjo6OtLMmTPJ19eXTE1NDdaHoqIiAsDp3lldIRaLKTIykpKSkmjXrl20bNky2rx5MxkZGXE6vgwcOJB8fX1pw4YNVFBQQKdOnWL3saqpqaHU1FTObL0K7ctA+zMXqNVqSkhIoPv371NUVBTV1NTQtm3b6NChQyQQCGjGjBkUEBCg175y5ubm9Lvf/Y5EIhFFRUWRWq2mZcuWkUajocLCQvruu+/o8uXLVF1dTUQv79H58+eTs7Nzv46poaGB1qxZQ2vWrNH5d4lEQgsWLKDly5fT8OHDDX5/Er3c87C1tZWIiORyORkbG3NqNzw8nIiILly4QJmZmdTW1kZKpZJSU1NJIpH0q+0bN27Qtm3b+rTvl52dHbt/lq2tLdnY2Ohn/FVqqDdqrf1u6X5+fqirq+tU6uvr+7XyZGtrK86fP99J6YnFYkRHR3O+rUR77t27h9GjR0MqlcLPz89gadq9ISMjg50S8vT01Ct9Ozk5GcOGDdOZEnFwcGCXtzcEZ86cwejRo3WCTbWBtwzDYNSoUUhOTuZ8Gkaj0SA8PJxdndrHxwfp6ekIDg6Gubk5XF1dceXKlX5/Gd25cwczZsxgPTBdlcGDB8Pd3R0HDx7Ue92KpqYmFBYWIi8vD3l5eaioqHht2WVvGrVaDUdHR/YeGjNmjMHWawJexrj4+/tDJBLB3d2dvYZcBi0DL70egYGBkEqlmD17NpsKfPv2bYwaNQpz5szhbAuErjCEh+fFixfYtWsXzMzMYGFhATs7OzbYVSQScZItdffuXfj7+7P9NjExYbdxaV+mTp2KS5cu9dtrHBERAUtLy07P9YQJE5Cbm2vQBSq7YsWKFey+aMuWLTNYHFROTg6kUik7lfW6Zzb0hPsprf8PaDQa5OXlQSAQwMXFhdMlyfWhpqYGCQkJsLW1xeLFi/VK679+/Tpmz57Nio0xY8bg2LFjBh1UW1pasG/fPvaFpc3o++ijj5Camornz58bbCn2uro6rF27lk1jbF+4XhX17t27KCoqwtWrVxEUFIQ5c+YgISEB+fn5qKysfGMZDW8D2oQB+vu8/+tYjbiqqoqdstBOJRuC3377DcuXL4dYLMaoUaOwZMkSODo6Qi6Xc5bO3x2GEDwnTpyAvb19p6xWT09PJCYmcrbH1FdffQV7e/suP4THjRvXp/WieqK2thaXL1/Grl27sHHjRsTFxeHKlSsGFd2v4k0IHq73NzQgvODRh+rqanz66aewsLDAtm3b3nR3ePSkvr4eaWlp+OSTTzBo0CBMmzYNhw8fxuPHj//feEj+1dFmFBIRFixY8FpstrW14datW1AoFGw2n6FQq9VYv349u2K8XC7nfI2hrmgveIyMjDjJfNOu5aJdJHXYsGFYv349bt++zUGP/0FFRQWCg4NZz46npyeWL1+Oo0ePGjxL6k2jFTzajGRDiRFtLE94ePi/incHeIWmYfDqOVtuJnT/RXn27BnFxsbS0aNH6fTp0zR69Og33SUenv93AKCBAwdSU1MTyWQyKioqon/7t3970916K6ivr6fc3Fw6ffo0DRo0iObPn09OTk5vuls8PZCfn09z586llpYWunLlCg0ZMuRNd+mfiW6DmXjBw8PD808NAJozZw6dOnWKxo8fT5mZmW+6Szw8PP+88IKHh4eHh4eH562nW8HTU06z4fPreHh4eHh4eHgMjOBNd4CHh4eHh4eHx9DwgoeHh4eHh4fnrYcXPDw8PDw8PDxvPbzg4eHh4eHh4Xnr4QUPDw8PDw8Pz1sPL3h4eHh4eHh43nr+DwdDTefnNoaTAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "c,predcts = concat_lib(image_set, how=\"horizontally\", with_plot=True,image_indices=image_indices)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(predcts)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_wjLhOXYJR65",
        "outputId": "ed57231b-b1b8-4bf5-bd9f-663ce0a20bd8"
      },
      "id": "_wjLhOXYJR65",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[6, 5, 9, 4, 1, 8, 3, 3, 0, 2, 7, 6, 8, 7, 5, 3, 5, 7, 4, 3]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "NK2SSNk8Kne1"
      },
      "id": "NK2SSNk8Kne1",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.11"
    },
    "colab": {
      "name": "deep_learning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}