VGG-esh.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "VGG.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.4"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ewa5uOvRx-Tt"
      },
      "source": [
        "import os \n",
        "import zipfile \n",
        "\n",
        "import tensorflow as tf \n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator \n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.layers import Dense, Input, Dropout,Flatten, Conv2D, MaxPool2D\n",
        "from tensorflow.keras.layers import BatchNormalization, Activation, MaxPooling2D\n",
        "from tensorflow.keras.models import Model, Sequential\n",
        "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "import glob\n",
        "from PIL import Image\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\n",
        "\n",
        "from IPython.display import clear_output\n",
        "from tqdm import tqdm\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import seaborn as sns\n",
        "\n",
        "from keras import backend as K\n",
        "\n",
        "from sklearn.utils import resample"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sOSKOnc3OkVX",
        "outputId": "a0018bbc-ed99-4c1e-db84-5e4684d001b0"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5HyWq_mhW75S"
      },
      "source": [
        "# data_dir = \"images/\"\n",
        "# metadata_path = \"HAM10000_metadata.csv\"\n",
        "data_dir = \"/content/drive/MyDrive/Colab Notebooks/images/\"\n",
        "metadata_path = \"/content/drive/MyDrive/Colab Notebooks/HAM10000_metadata\""
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gf2eWAukQrVe"
      },
      "source": [
        "#Lesion Dictionary\n",
        "lesion_type_dict = {\n",
        "    'nv': 'Melanocytic nevi',\n",
        "    'mel': 'Melanoma',\n",
        "    'bkl': 'Benign keratosis-like lesions ',\n",
        "    'bcc': 'Basal cell carcinoma',\n",
        "    'akiec': 'Actinic keratoses',\n",
        "    'vasc': 'Vascular lesions',\n",
        "    'df': 'Dermatofibroma'\n",
        "}"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GEXRn0E25KCy"
      },
      "source": [
        "# Read metadata into dataframe\n",
        "metadata = pd.read_csv(metadata_path)\n",
        "skin_df = metadata"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZvxnWcZt5KCy"
      },
      "source": [
        "# Dictionary for Image Names\n",
        "imageid_path_dict = {os.path.splitext(os.path.basename(x))[0]: x for x in glob.glob(os.path.join(data_dir, '*.jpg'))}"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VNqP0ahB5KCy"
      },
      "source": [
        "#Create useful Columns - Images Path, Lesion Type and Lesion Categorical Code\n",
        "skin_df['path'] = skin_df['image_id'].map(imageid_path_dict.get)\n",
        "skin_df['cell_type'] = skin_df['dx'].map(lesion_type_dict.get) \n",
        "skin_df['cell_type_idx'] = pd.Categorical(skin_df['cell_type']).codes"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0POOFz2V5KCz",
        "outputId": "f4e6e82f-b710-4084-a0e2-90c9f95cd728"
      },
      "source": [
        "# Lesion code dictionary\n",
        "lesion_code_dict = pd.Series(skin_df.dx.values,index=skin_df.cell_type_idx).to_dict()\n",
        "\n",
        "lesion_code_dict"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'akiec', 1: 'bcc', 2: 'bkl', 3: 'df', 4: 'nv', 5: 'mel', 6: 'vasc'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 589
        },
        "id": "hqAIozEF5KCz",
        "outputId": "2ef6b5df-6434-4d50-db88-3fe6c8691621"
      },
      "source": [
        "skin_df"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>lesion_id</th>\n",
              "      <th>image_id</th>\n",
              "      <th>dx</th>\n",
              "      <th>dx_type</th>\n",
              "      <th>age</th>\n",
              "      <th>sex</th>\n",
              "      <th>localization</th>\n",
              "      <th>dataset</th>\n",
              "      <th>path</th>\n",
              "      <th>cell_type</th>\n",
              "      <th>cell_type_idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>HAM_0000118</td>\n",
              "      <td>ISIC_0027419</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>HAM_0000118</td>\n",
              "      <td>ISIC_0025030</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HAM_0002730</td>\n",
              "      <td>ISIC_0026769</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>HAM_0002730</td>\n",
              "      <td>ISIC_0025661</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>HAM_0001466</td>\n",
              "      <td>ISIC_0031633</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>75.0</td>\n",
              "      <td>male</td>\n",
              "      <td>ear</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10010</th>\n",
              "      <td>HAM_0002867</td>\n",
              "      <td>ISIC_0033084</td>\n",
              "      <td>akiec</td>\n",
              "      <td>histo</td>\n",
              "      <td>40.0</td>\n",
              "      <td>male</td>\n",
              "      <td>abdomen</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Actinic keratoses</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10011</th>\n",
              "      <td>HAM_0002867</td>\n",
              "      <td>ISIC_0033550</td>\n",
              "      <td>akiec</td>\n",
              "      <td>histo</td>\n",
              "      <td>40.0</td>\n",
              "      <td>male</td>\n",
              "      <td>abdomen</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Actinic keratoses</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10012</th>\n",
              "      <td>HAM_0002867</td>\n",
              "      <td>ISIC_0033536</td>\n",
              "      <td>akiec</td>\n",
              "      <td>histo</td>\n",
              "      <td>40.0</td>\n",
              "      <td>male</td>\n",
              "      <td>abdomen</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Actinic keratoses</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10013</th>\n",
              "      <td>HAM_0000239</td>\n",
              "      <td>ISIC_0032854</td>\n",
              "      <td>akiec</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>face</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Actinic keratoses</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10014</th>\n",
              "      <td>HAM_0003521</td>\n",
              "      <td>ISIC_0032258</td>\n",
              "      <td>mel</td>\n",
              "      <td>histo</td>\n",
              "      <td>70.0</td>\n",
              "      <td>female</td>\n",
              "      <td>back</td>\n",
              "      <td>vidir_modern</td>\n",
              "      <td>/content/drive/MyDrive/Colab Notebooks/images/...</td>\n",
              "      <td>Melanoma</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10015 rows × 11 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         lesion_id      image_id  ...                       cell_type cell_type_idx\n",
              "0      HAM_0000118  ISIC_0027419  ...  Benign keratosis-like lesions              2\n",
              "1      HAM_0000118  ISIC_0025030  ...  Benign keratosis-like lesions              2\n",
              "2      HAM_0002730  ISIC_0026769  ...  Benign keratosis-like lesions              2\n",
              "3      HAM_0002730  ISIC_0025661  ...  Benign keratosis-like lesions              2\n",
              "4      HAM_0001466  ISIC_0031633  ...  Benign keratosis-like lesions              2\n",
              "...            ...           ...  ...                             ...           ...\n",
              "10010  HAM_0002867  ISIC_0033084  ...               Actinic keratoses             0\n",
              "10011  HAM_0002867  ISIC_0033550  ...               Actinic keratoses             0\n",
              "10012  HAM_0002867  ISIC_0033536  ...               Actinic keratoses             0\n",
              "10013  HAM_0000239  ISIC_0032854  ...               Actinic keratoses             0\n",
              "10014  HAM_0003521  ISIC_0032258  ...                        Melanoma             5\n",
              "\n",
              "[10015 rows x 11 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8pFORH3joGg2",
        "outputId": "56a4b71d-14bb-4d41-d5ab-925b71daed2d"
      },
      "source": [
        "# Attach images to the skin_df Dataframe\n",
        "temp_arr, count = [], 0\n",
        "for path in tqdm(skin_df['path']):\n",
        "  temp_arr.append(np.asarray( Image.open(path).resize((100,75)) ))\n",
        "  count += 1\n",
        "  # print(f\"Processed images: {count}\")\n",
        "  # if count % 10 == 0: clear_output()\n",
        "\n",
        "skin_df['image'] = temp_arr\n",
        "\n",
        "\n",
        "# skin_df['image'] = skin_df['path'].map(lambda x: np.asarray(Image.open(x).resize((100,75))))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 10015/10015 [37:44<00:00,  4.42it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "raUm2J0PgXYL"
      },
      "source": [
        "# Separate the dataframe into Features and Targets data\n",
        "features = skin_df.drop(columns=['cell_type_idx','dx'],axis=1)\n",
        "target = skin_df['cell_type_idx']"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yx5DqaAK5KCz",
        "outputId": "fe2eaa34-0390-41ef-a97a-ea2f451cf37e"
      },
      "source": [
        "# Create First Train and Test sets\n",
        "x_train_o, x_test_o, y_train_o, y_test_o = train_test_split(features, target, test_size=0.20, random_state=1)\n",
        "print(x_train_o.shape)\n",
        "print(y_train_o.shape)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(8012, 10)\n",
            "(8012,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sbX83-Yp5KCz",
        "outputId": "57096755-4d05-455c-d375-f7348f56963e"
      },
      "source": [
        "# Create image arrays from the train and test sets\n",
        "x_train = np.asarray(x_train_o['image'].tolist())\n",
        "x_test = np.asarray(x_test_o['image'].tolist())\n",
        "\n",
        "print(x_train.shape)\n",
        "print(x_test.shape)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(8012, 75, 100, 3)\n",
            "(2003, 75, 100, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UUJSBhG-5KC0",
        "outputId": "84553684-e698-426f-f329-a19274d167c9"
      },
      "source": [
        "# Class distribution before data augmentation\n",
        "train_value_counts = y_train_o.value_counts()\n",
        "print(train_value_counts)\n",
        "print(f'Total images: {train_value_counts.sum()}')"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4    5378\n",
            "5     891\n",
            "2     871\n",
            "1     418\n",
            "0     266\n",
            "6     110\n",
            "3      78\n",
            "Name: cell_type_idx, dtype: int64\n",
            "Total images: 8012\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFaa1RBfgE_B"
      },
      "source": [
        "# Data Augmentation\n",
        "# Image data generator is used to generate the possible versions of data as given below.\n",
        "\n",
        "dataaugment = ImageDataGenerator(\n",
        "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
        "        samplewise_center=False,  # set each sample mean to 0\n",
        "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
        "        samplewise_std_normalization=False,  # divide each input by its std\n",
        "        zca_whitening=False,  # apply ZCA whitening\n",
        "        rotation_range=90,  # randomly rotate images in the range (degrees, 0 to 180)\n",
        "        zoom_range = 0.1, # Randomly zoom image \n",
        "        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
        "        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
        "        horizontal_flip=True,  # randomly flip images\n",
        "        vertical_flip=True,  # randomly flip images\n",
        "        shear_range = 10\n",
        "    ) "
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1xOuqdLf5KC0",
        "outputId": "74c99c12-0db6-454f-8c01-837e09f97c29"
      },
      "source": [
        "# For each lesion code != 4, get the indices in y_train_o.\n",
        "# For the same indices in x_train_o, augment the image present in the row.\n",
        "reset_y_train_o = y_train_o.reset_index(drop=True)\n",
        "reset_index = reset_y_train_o.index\n",
        "\n",
        "print(len(y_train_o))"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "8012\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvfAsDIGhnEx"
      },
      "source": [
        "# The normalisation is done using the training set Mean and Std. Deviation as reference\n",
        "x_train_mean = np.mean(x_train)\n",
        "x_train_std = np.std(x_train)\n",
        "\n",
        "x_train = (x_train - x_train_mean) / x_train_std\n",
        "x_test = (x_test - x_train_mean) / x_train_std"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQ0EFbN5joCo"
      },
      "source": [
        "# Perform one-hot encoding on the labels\n",
        "y_train = to_categorical(y_train_o, num_classes = 7)\n",
        "y_test = to_categorical(y_test_o, num_classes = 7)"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vRrRWy_Njsnp"
      },
      "source": [
        "# Splitting training into Train and Validatation sets\n",
        "x_train, x_validate, y_train, y_validate = train_test_split(x_train, y_train, test_size = 0.1, random_state=1)\n",
        "\n",
        "# Reshaping the Images into 3 channels (RGB)\n",
        "# height = 75, width = 100, canal = 3\n",
        "x_train = x_train.reshape(x_train.shape[0], *(75, 100, 3))\n",
        "x_test = x_test.reshape(x_test.shape[0], *(75, 100, 3))\n",
        "x_validate = x_validate.reshape(x_validate.shape[0], *(75, 100, 3))"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z9XLCqXO5KC1",
        "outputId": "d5f9f10f-453c-4c89-fe58-56a4d395a00a"
      },
      "source": [
        "input_shape = (75, 100, 3)\n",
        "num_classes = 7\n",
        "\n",
        "# VGGNet 16\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Conv2D(input_shape=input_shape,filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n",
        "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n",
        "model.add(Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n",
        "model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n",
        "model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n",
        "model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n",
        "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n",
        "\n",
        "model.add(Flatten())\n",
        "model.add(Dense(units=4096,activation=\"relu\"))\n",
        "model.add(Dropout(0.25))\n",
        "model.add(Dense(units=4096,activation=\"relu\"))\n",
        "model.add(Dropout(0.25))\n",
        "model.add(Dense(units=7, activation=\"softmax\"))\n",
        "\n",
        "# from keras.optimizers import Adam\n",
        "# opt = Adam(lr=0.001)\n",
        "# model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "model.summary()\n"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d_26 (Conv2D)           (None, 75, 100, 64)       1792      \n",
            "_________________________________________________________________\n",
            "conv2d_27 (Conv2D)           (None, 75, 100, 64)       36928     \n",
            "_________________________________________________________________\n",
            "max_pooling2d_10 (MaxPooling (None, 37, 50, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_28 (Conv2D)           (None, 37, 50, 128)       73856     \n",
            "_________________________________________________________________\n",
            "conv2d_29 (Conv2D)           (None, 37, 50, 128)       147584    \n",
            "_________________________________________________________________\n",
            "max_pooling2d_11 (MaxPooling (None, 18, 25, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_30 (Conv2D)           (None, 18, 25, 256)       295168    \n",
            "_________________________________________________________________\n",
            "conv2d_31 (Conv2D)           (None, 18, 25, 256)       590080    \n",
            "_________________________________________________________________\n",
            "conv2d_32 (Conv2D)           (None, 18, 25, 256)       590080    \n",
            "_________________________________________________________________\n",
            "max_pooling2d_12 (MaxPooling (None, 9, 12, 256)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_33 (Conv2D)           (None, 9, 12, 512)        1180160   \n",
            "_________________________________________________________________\n",
            "conv2d_34 (Conv2D)           (None, 9, 12, 512)        2359808   \n",
            "_________________________________________________________________\n",
            "conv2d_35 (Conv2D)           (None, 9, 12, 512)        2359808   \n",
            "_________________________________________________________________\n",
            "max_pooling2d_13 (MaxPooling (None, 4, 6, 512)         0         \n",
            "_________________________________________________________________\n",
            "conv2d_36 (Conv2D)           (None, 4, 6, 512)         2359808   \n",
            "_________________________________________________________________\n",
            "conv2d_37 (Conv2D)           (None, 4, 6, 512)         2359808   \n",
            "_________________________________________________________________\n",
            "conv2d_38 (Conv2D)           (None, 4, 6, 512)         2359808   \n",
            "_________________________________________________________________\n",
            "max_pooling2d_14 (MaxPooling (None, 2, 3, 512)         0         \n",
            "_________________________________________________________________\n",
            "flatten_2 (Flatten)          (None, 3072)              0         \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 4096)              12587008  \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 4096)              0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 4096)              16781312  \n",
            "_________________________________________________________________\n",
            "dropout_5 (Dropout)          (None, 4096)              0         \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 7)                 28679     \n",
            "=================================================================\n",
            "Total params: 44,111,687\n",
            "Trainable params: 44,111,687\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0oFIjDtr5KC2"
      },
      "source": [
        "# Define metrics \n",
        "\n",
        "def recall_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def precision_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "def f1_m(y_true, y_pred):\n",
        "    precision = precision_m(y_true, y_pred)\n",
        "    recall = recall_m(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2QceAAB5KC2"
      },
      "source": [
        "# New baseline\n",
        "# Define the optimizer\n",
        "optimizer = Adam(learning_rate=0.001)\n",
        "\n",
        "# Compile the model\n",
        "model.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\", f1_m,precision_m, recall_m])\n",
        "\n",
        "# Set a learning rate annealer\n",
        "learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', \n",
        "                                            patience=3, \n",
        "                                            verbose=1, \n",
        "                                            factor=0.5, \n",
        "                                            min_lr=0.00001)\n",
        "early_stopping_monitor = EarlyStopping(patience=20,monitor='val_accuracy')"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5WSfybEBiz-W"
      },
      "source": [
        "x_train = np.asarray(x_train)\n",
        "y_train = np.asarray(y_train)\n",
        "x_validate = np.asarray(x_validate)\n",
        "y_validate = np.asarray(y_validate)"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTGRcxtb5KC2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "150036b6-77fe-448d-c5e5-3bf0460ce4b2"
      },
      "source": [
        "# Fit the model\n",
        "\n",
        "epochs = 30\n",
        "batch_size = 10\n",
        "# history = model.fit(dataaugment.flow(x_train,y_train, batch_size=batch_size),\n",
        "#                     epochs = epochs, validation_data = (x_validate,y_validate),\n",
        "#                     verbose = 0, steps_per_epoch=x_train.shape[0] // batch_size\n",
        "#                               , callbacks=[learning_rate_reduction])\n",
        "\n",
        "history = model.fit(x=x_train, y=y_train,\n",
        "                              epochs = epochs, validation_data = (x_validate,y_validate),\n",
        "                              verbose = 1, steps_per_epoch=x_train.shape[0] // batch_size\n",
        "                              , callbacks=[learning_rate_reduction])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n",
            "137/721 [====>.........................] - ETA: 29:49 - loss: 1.6406 - accuracy: 0.6047 - f1_m: 0.4904 - precision_m: 0.5091 - recall_m: 0.4802"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "C8o3b0Pn5KC2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c4fe28c4-22b4-4775-9f32-a6377eae358c"
      },
      "source": [
        "# plot graphs from history\n",
        "print(history.history.keys())\n",
        "print(f'Accuracy: {history.history[\"accuracy\"][-1]}')\n",
        "print(f'loss: {history.history[\"loss\"][-1]}')\n",
        "print(f'f1_m: {history.history[\"f1_m\"][-1]}')\n",
        "print(f'precision_m: {history.history[\"precision_m\"][-1]}')\n",
        "print(f'recall_m: {history.history[\"recall_m\"][-1]}')\n",
        "\n",
        "plt.plot(history.history[\"accuracy\"], label='accuracy')\n",
        "plt.plot(history.history[\"val_accuracy\"], label='Validaion accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'])\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history[\"f1_m\"], label='F1_score')\n",
        "plt.plot(history.history[\"val_f1_m\"], label='Validaion F1_score')\n",
        "plt.ylabel('F1_score')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'])\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history[\"precision_m\"], label='Precision')\n",
        "plt.plot(history.history[\"val_precision_m\"], label='Validaion Precision')\n",
        "plt.ylabel('Precision')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'])\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history[\"recall_m\"], label='Recall')\n",
        "plt.plot(history.history[\"val_recall_m\"], label='Validaion Recall')\n",
        "plt.ylabel('Recall')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'])\n",
        "plt.show()"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_keys(['loss', 'accuracy', 'f1_m', 'precision_m', 'recall_m', 'val_loss', 'val_accuracy', 'val_f1_m', 'val_precision_m', 'val_recall_m', 'lr'])\n",
            "Accuracy: 0.8999999761581421\n",
            "loss: 0.5843114256858826\n",
            "f1_m: 0.8999999165534973\n",
            "precision_m: 0.8999999761581421\n",
            "recall_m: 0.8999999761581421\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAayElEQVR4nO3df5QV5Z3n8fdHRFsjavPDaGi02138gTGDyZVxo5k1cVTUo5h1FBydoxkTJjNq1MzkDJ5kR4dkTzhn92x2csbEkBxmkqwBDUbTyZqwqKAnUSOXhIigCBINjRpbBCNZUcHv/lHVprg8QDV03f7B53XOPVQ99VTdb1df7qernnurFBGYmZk12q+/CzAzs4HJAWFmZkkOCDMzS3JAmJlZkgPCzMyS9u/vAvrK6NGjo729vb/LMDMbVJYuXfpKRIxJLRsyAdHe3k69Xu/vMszMBhVJz+9smU8xmZlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJVUaEJImS1olaY2kGYnlx0h6QNITkhZLaissu0rS6vxxVZV1mpnZjioLCEnDgNuA84AJwOWSJjR0+x/AdyLiA8BM4Mv5uiOBW4A/BSYBt0hqrapWMzPbUZXfg5gErImItQCS5gFTgJWFPhOAz+bTi4B78+lzgYUR8Wq+7kJgMjC3ikL/+UcrWPnC76vYtJlZ5Sa871BuufCkPt9ulaeYxgLrCvNdeVvRr4H/kk9/HBghaVTJdZE0XVJdUr27u7vPCjczs/7/JvU/AP8q6WrgYWA9sK3syhExG5gNUKvV9vjOR1Ukr5nZYFdlQKwHxhXm2/K2d0XEC+RHEJIOAS6JiE2S1gNnNqy7uMJazcysQZWnmJYA4yV1SDoAmAZ0FjtIGi2pp4abgTn59ALgHEmt+eD0OXmbmZk1SWUBERFbgevI3tifAu6KiBWSZkq6KO92JrBK0jPAe4H/lq/7KvBFspBZAszsGbA2M7PmUMQen7ofUGq1WvhqrmZmvSNpaUTUUsv8TWozM0tyQJiZWZIDwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLMkBYWZmSQ4IMzNLckCYmVmSA8LMzJIcEGZmluSAMDOzJAeEmZklOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJlQaEpMmSVklaI2lGYvnRkhZJ+pWkJySdn7e3S3pD0rL8cXuVdZqZ2Y72r2rDkoYBtwFnA13AEkmdEbGy0O0LwF0R8XVJE4D7gPZ82bMRMbGq+szMbNeqPIKYBKyJiLUR8RYwD5jS0CeAQ/Ppw4AXKqzHzMx6ocqAGAusK8x35W1FtwJXSuoiO3q4vrCsIz/19JCkj6SeQNJ0SXVJ9e7u7j4s3czM+nuQ+nLg3yOiDTgf+K6k/YAXgaMj4hTgs8D3JB3auHJEzI6IWkTUxowZ09TCzcyGuioDYj0wrjDflrcVXQPcBRARjwItwOiIeDMiNuTtS4FngeMqrNXMzBpUGRBLgPGSOiQdAEwDOhv6/BY4C0DSiWQB0S1pTD7IjaRjgfHA2gprNTOzBpV9iikitkq6DlgADAPmRMQKSTOBekR0An8PfFPSTWQD1ldHREj6M2CmpLeBd4BPR8SrVdVqZmY7UkT0dw19olarRb1e7+8yzMwGFUlLI6KWWtbfg9RmZjZAOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLMkBYWZmSQ4IMzNLckCYmVmSA8LMzJIcEGZmluSAMDOzpEoDQtJkSaskrZE0I7H8aEmLJP1K0hOSzi8suzlfb5Wkc6us08zMdrR/VRuWNAy4DTgb6AKWSOqMiJWFbl8A7oqIr0uaANwHtOfT04CTgPcB90s6LiK2VVWvmZltr8ojiEnAmohYGxFvAfOAKQ19Ajg0nz4MeCGfngLMi4g3I+I3wJp8e2Zm1iRVBsRYYF1hvitvK7oVuFJSF9nRw/W9WBdJ0yXVJdW7u7v7qm4zM6P/B6kvB/49ItqA84HvSipdU0TMjohaRNTGjBlTWZFmZvuiysYggPXAuMJ8W95WdA0wGSAiHpXUAowuua6ZmVWoyiOIJcB4SR2SDiAbdO5s6PNb4CwASScCLUB33m+apAMldQDjgccrrNXMzBpUdgQREVslXQcsAIYBcyJihaSZQD0iOoG/B74p6SayAeurIyKAFZLuAlYCW4Fr/QkmM7PmUvZ+PPjVarWo1+v9XYaZ2aAiaWlE1FLL+nuQ2szMBigHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkkoFhKQfSLqgN9dJMjOzwa3sG/7XgL8EVkuaJen4CmsyM7MBoFRARMT9EXEF8EHgObIb+Dwi6ROShldZoJmZ9Y/Sp4wkjQKuBj4J/Ar4F7LAWFhJZWZm1q9KXaxP0j3A8cB3gQsj4sV80Z2SfAEkMxu03n77bbq6utiyZUt/l1KplpYW2traGD68/Emfsldz/WpELEot2NlFnszMBoOuri5GjBhBe3s7kvq7nEpEBBs2bKCrq4uOjo7S65U9xTRB0uE9M5JaJf1db4s0MxtotmzZwqhRo4ZsOABIYtSoUb0+SiobEJ+KiE09MxGxEfhUr57JzGyAGsrh0GNPfsayATFMha1LGgYc0OtnMzOz7WzatImvfe1rvV7v/PPPZ9OmTbvvuBfKBsRPyQakz5J0FjA3bzMzs72ws4DYunXrLte77777OPzww3fZZ2+VHaT+R+BvgL/N5xcC36qkIjOzfciMGTN49tlnmThxIsOHD6elpYXW1laefvppnnnmGS6++GLWrVvHli1buOGGG5g+fToA7e3t1Ot1Nm/ezHnnnccZZ5zBI488wtixY/nhD3/IQQcdtNe1lQqIiHgH+Hr+MDMbkv75RytY+cLv+3SbE953KLdceNJOl8+aNYsnn3ySZcuWsXjxYi644AKefPLJdz9tNGfOHEaOHMkbb7zBqaeeyiWXXMKoUaO228bq1auZO3cu3/zmN7nsssu4++67ufLKK/e69rLfgxgPfBmYALT0tEfEsXtdgZmZvWvSpEnbfRT1q1/9Kvfccw8A69atY/Xq1TsEREdHBxMnTgTgQx/6EM8991yf1FL2FNO/AbcAXwE+CnwCXwnWzIaYXf2l3yzvec973p1evHgx999/P48++igHH3wwZ555ZvKjqgceeOC708OGDeONN97ok1rKvskfFBEPAIqI5yPiVuCCPqnAzGwfNmLECF5//fXkstdee43W1lYOPvhgnn76aR577LGm1lb2COLN/FLfqyVdB6wHDqmuLDOzfcOoUaM4/fTTef/7389BBx3Ee9/73neXTZ48mdtvv50TTzyR448/ntNOO62ptSkidt9JOhV4Cjgc+CJwKPDfI6K5cbYLtVot6nVfFsrMeuepp57ixBNP7O8ymiL1s0paurNLJu32CCL/UtzUiPgHYDPZ+EMpkiaTXfV1GPCtiJjVsLxnTAPgYOCIiDg8X7YNWJ4v+21EXFT2ec3MbO/tNiAiYpukM3q74TxYbgPOBrqAJZI6I2JlYds3FfpfD5xS2MQbETGxt89rZmZ9o+wYxK8kdQLfB/7Q0xgRP9jFOpOANRGxFkDSPGAKsHIn/S8n+6SUmZkNAGUDogXYAHys0BbArgJiLLCuMN8F/Gmqo6RjgA7gweJz5vea2ArMioh7E+tNB6YDHH300bv/KczMrLSy36QuPe6wh6YB8yNiW6HtmIhYL+lY4EFJyyPi2Ya6ZgOzIRukrrhGM7N9StlvUv8b2RHDdiLir3ex2npgXGG+LW9LmQZc27Dt9fm/ayUtJhufeHbHVc3MrAplvyj3Y+D/5I8HyD7munk36ywBxkvqkHQAWQh0NnaSdALQCjxaaGuVdGA+PRo4nZ2PXZiZ7TMOOaR5X0Ere4rp7uK8pLnAz3azztb8S3ULyD7mOiciVkiaCdQjoicspgHzYvsvZJwIfEPSO2QhNqv46SczM6te2UHqRuOBI3bXKSLuA+5raPunhvlbE+s9Apy8h7WZmQ0aM2bMYNy4cVx7bXaW/dZbb2X//fdn0aJFbNy4kbfffpsvfelLTJkypem1lR2DeJ3txyBeIrtHhJnZ0PGTGfDS8t33640jT4bzZu108dSpU7nxxhvfDYi77rqLBQsW8JnPfIZDDz2UV155hdNOO42LLrqo6bdGLXuKaUTVhZiZ7YtOOeUUXn75ZV544QW6u7tpbW3lyCOP5KabbuLhhx9mv/32Y/369fzud7/jyCOPbGptZY8gPg48GBGv5fOHA2emvptgZjZo7eIv/SpdeumlzJ8/n5deeompU6dyxx130N3dzdKlSxk+fDjt7e3Jy3xXreynmG7pCQeAiNiEv/VsZtYnpk6dyrx585g/fz6XXnopr732GkcccQTDhw9n0aJFPP/88/1SV9lB6lSQ7OkAt5mZFZx00km8/vrrjB07lqOOOoorrriCCy+8kJNPPplarcYJJ5zQL3WVfZOvS/qfZBffg+xLbUurKcnMbN+zfPkfB8dHjx7No48+muy3efPuvoLWd8qeYroeeAu4E5gHbKHhm89mZja0lP0U0x+AGRXXYmZmA0ipIwhJC/NPLvXMt0paUF1ZZmbW38qeYhqdf3IJgIjYSIlvUpuZDQZlbr082O3Jz1g2IN6R9O4NFyS1k7i6q5nZYNPS0sKGDRuGdEhEBBs2bKClpaVX65X9FNPngZ9JeggQ8BHyG/WYmQ1mbW1tdHV10d3d3d+lVKqlpYW2trZerVN2kPqnkmpkofAr4F7gjV5XaGY2wAwfPpyOjo7+LmNAKnupjU8CN5Dd9GcZcBrZ/Rs+tqv1zMxs8Co7BnEDcCrwfER8lOzubpt2vYqZmQ1mZQNiS0RsAZB0YEQ8DRxfXVlmZtbfyg5Sd+Xfg7gXWChpI9A/V48yM7OmKDtI/fF88lZJi4DDgJ9WVpWZmfW7Xl+RNSIeqqIQMzMbWMqOQZiZ2T7GAWFmZkkOCDMzS3JAmJlZkgPCzMySKg0ISZMlrZK0RtIONxyS9BVJy/LHM5I2FZZdJWl1/riqyjrNzGxHvf6Ya1mShpHdw/psoAtYIqkzIlb29ImImwr9rye7hAeSRgK3ADWyy4ovzdfdWFW9Zma2vSqPICYBayJibUS8RXYv6ym76H85MDefPhdYGBGv5qGwEJhcYa1mZtagyoAYC6wrzHflbTuQdAzQATzYm3UlTZdUl1Qf6tdyNzNrtoEySD0NmB8R23qzUkTMjohaRNTGjBlTUWlmZvumKgNiPTCuMN+Wt6VM44+nl3q7rpmZVaDKgFgCjJfUIekAshDobOwk6QSglewGRD0WAOdIapXUCpyTt5mZWZNU9immiNgq6TqyN/ZhwJyIWCFpJlCPiJ6wmAbMi8IdwyPiVUlfJAsZgJkR8WpVtZqZ2Y5UeF8e1Gq1WtTr9f4uw8xsUJG0NCJqqWUDZZDazMwGGAeEmZklOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLMkBYWZmSQ4IMzNLckCYmVmSA8LMzJIcEGZmllRpQEiaLGmVpDWSZuykz2WSVkpaIel7hfZtkpblj84q6zQzsx3tX9WGJQ0DbgPOBrqAJZI6I2Jloc944Gbg9IjYKOmIwibeiIiJVdVnZma7VuURxCRgTUSsjYi3gHnAlIY+nwJui4iNABHxcoX1mJlZL1QZEGOBdYX5rryt6DjgOEk/l/SYpMmFZS2S6nn7xaknkDQ971Pv7u7u2+rNzPZxlZ1i6sXzjwfOBNqAhyWdHBGbgGMiYr2kY4EHJS2PiGeLK0fEbGA2QK1Wi+aWbmY2tFV5BLEeGFeYb8vbirqAzoh4OyJ+AzxDFhhExPr837XAYuCUCms1M7MGVQbEEmC8pA5JBwDTgMZPI91LdvSApNFkp5zWSmqVdGCh/XRgJWZm1jSVnWKKiK2SrgMWAMOAORGxQtJMoB4RnfmycyStBLYBn4uIDZI+DHxD0jtkITar+OknMzOrniKGxqn7Wq0W9Xq9v8swMxtUJC2NiFpqmb9JbWZmSQ4IMzNLckCYmVmSA8LMzJIcEGZmluSAMDOzJAeEmZklOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLKnSgJA0WdIqSWskzdhJn8skrZS0QtL3Cu1XSVqdP66qsk4zM9vR/lVtWNIw4DbgbKALWCKpMyJWFvqMB24GTo+IjZKOyNtHArcANSCApfm6G6uq18zMtldZQACTgDURsRZA0jxgCrCy0OdTwG09b/wR8XLefi6wMCJezdddCEwG5lZS6U9mwEvLK9m0mVnljjwZzpvV55ut8hTTWGBdYb4rbys6DjhO0s8lPSZpci/WRdJ0SXVJ9e7u7j4s3czMqjyCKPv844EzgTbgYUknl105ImYDswFqtVrscRUVJK+Z2WBX5RHEemBcYb4tbyvqAjoj4u2I+A3wDFlglFnXzMwqVGVALAHGS+qQdAAwDehs6HMv2dEDkkaTnXJaCywAzpHUKqkVOCdvMzOzJqnsFFNEbJV0Hdkb+zBgTkSskDQTqEdEJ38MgpXANuBzEbEBQNIXyUIGYGbPgLWZmTWHIvb81P1AUqvVol6v93cZZmaDiqSlEVFLLfM3qc3MLMkBYWZmSQ4IMzNLckCYmVnSkBmkltQNPL8XmxgNvNJH5fQl19U7rqt3XFfvDMW6jomIMakFQyYg9pak+s5G8vuT6+od19U7rqt39rW6fIrJzMySHBBmZpbkgPij2f1dwE64rt5xXb3junpnn6rLYxBmZpbkIwgzM0tyQJiZWdKQDwhJkyWtkrRG0ozE8gMl3Zkv/4Wk9sKym/P2VZLObXJdn5W0UtITkh6QdExh2TZJy/JH4yXUq67rakndhef/ZGHZVZJW54+rmlzXVwo1PSNpU2FZlftrjqSXJT25k+WS9NW87ickfbCwrMr9tbu6rsjrWS7pEUl/Ulj2XN6+TFKfXgGzRF1nSnqt8Pv6p8KyXb4GKq7rc4WansxfUyPzZVXur3GSFuXvBSsk3ZDoU91rLCKG7IPsMuPPAscCBwC/BiY09Pk74PZ8ehpwZz49Ie9/INCRb2dYE+v6KHBwPv23PXXl85v7cX9dDfxrYt2RZPfyGAm05tOtzaqrof/1ZJeXr3R/5dv+M+CDwJM7WX4+8BNAwGnAL6reXyXr+nDP8wHn9dSVzz8HjO6n/XUm8OO9fQ30dV0NfS8EHmzS/joK+GA+PYLspmqN/ycre40N9SOIScCaiFgbEW8B84ApDX2mAN/Op+cDZ0lS3j4vIt6M7G53a/LtNaWuiFgUEf8vn32M7K56VSuzv3bmXGBhRLwaERuBhcDk3axTVV2XA3P76Ll3KSIeBnZ1r5IpwHci8xhwuKSjqHZ/7bauiHgkf15o3uurzP7amb15bfZ1Xc18fb0YEb/Mp18HngLGNnSr7DU21ANiLLCuMN/Fjjv33T4RsRV4DRhVct0q6yq6huwvhB4tkuqSHpN0cR/V1Ju6LskPZedL6rk17IDYX/mpuA7gwUJzVfurjJ3VXuX+6q3G11cA/1fSUknT+6Ge/yTp15J+IumkvG1A7C9JB5O9yd5daG7K/lJ2+vsU4BcNiyp7jVV2RznrG5KuBGrAfy40HxMR6yUdCzwoaXlEPNukkn4EzI2INyX9DdnR18ea9NxlTAPmR8S2Qlt/7q8BTdJHyQLijELzGfn+OgJYKOnp/C/sZvgl2e9rs6TzyW5LPL5Jz13GhcDPY/s7XFa+vyQdQhZKN0bE7/ty27sy1I8g1gPjCvNteVuyj6T9gcOADSXXrbIuJP058Hngooh4s6c9Itbn/64FFpP9VdGUuiJiQ6GWbwEfKrtulXUVTKPh8L/C/VXGzmqvcn+VIukDZL/DKZHf6he2218vA/fQd6dWdysifh8Rm/Pp+4Dhyu5X3+/7K7er11cl+0vScLJwuCMifpDoUt1rrIqBlYHyIDtCWkt2yqFnYOukhj7Xsv0g9V359ElsP0i9lr4bpC5T1ylkg3LjG9pbgQPz6dHAavposK5kXUcVpj8OPBZ/HBD7TV5faz49sll15f1OIBswVDP2V+E52tn5oOsFbD+A+HjV+6tkXUeTjat9uKH9PcCIwvQjwOQm1nVkz++P7I32t/m+K/UaqKqufPlhZOMU72nW/sp/9u8A/2sXfSp7jfXZzh2oD7IR/mfI3mw/n7fNJPurHKAF+H7+n+Vx4NjCup/P11sFnNfkuu4Hfgcsyx+defuHgeX5f5DlwDVNruvLwIr8+RcBJxTW/et8P64BPtHMuvL5W4FZDetVvb/mAi8Cb5Od470G+DTw6Xy5gNvyupcDtSbtr93V9S1gY+H1Vc/bj8331a/z3/Pnm1zXdYXX12MUAiz1GmhWXXmfq8k+uFJcr+r9dQbZGMcThd/V+c16jflSG2ZmljTUxyDMzGwPOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCbADIr2L64/6uw6zIAWFmZkkOCLNekHSlpMfza/9/Q9IwSZvz+1GsUHbvjjF534n5BQKfkHSPpNa8/T9Kuj+/IN0vJf2HfPOH5BdAfFrSHflVhc36jQPCrCRJJwJTgdMjYiKwDbiC7BIL9Yg4CXgIuCVf5TvAP0bEB8i+4drTfgdwW0T8Cdk3vV/M208BbiS7F8mxwOmV/1Bmu+CruZqVdxbZxQmX5H/cHwS8DLwD3Jn3+d/ADyQdBhweEQ/l7d8Gvi9pBDA2Iu4BiIgtAPn2Ho+Irnx+Gdm1gX5W/Y9lluaAMCtPwLcj4ubtGqX/2tBvT69f82Zhehv+/2n9zKeYzMp7APiL/Lr/SBqZ36BoP+Av8j5/CfwsIl4DNkr6SN7+V8BDkd0VrKvnxkXK7ol+cFN/CrOS/BeKWUkRsVLSF8juHrYf2ZU/rwX+AEzKl71MNk4BcBVwex4Aa4FP5O1/BXxD0sx8G5c28ccwK81XczXbS5I2R8Qh/V2HWV/zKSYzM0vyEYSZmSX5CMLMzJIcEGZmluSAMDOzJAeEmZklOSDMzCzp/wMFwhbF/rNUBQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEGCAYAAACQO2mwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAalElEQVR4nO3dfZQV9Z3n8fdHQBsUtXnwITTa7S5GJUZMriyrzozZjIrkKM4YBVezmnFlsqNGTTZncM2MDsme8Y89J7OekDGYw5mYNRCiienM0SUaQXciRC6OyoMiD9HQGGOL+MDEB8Dv/lG/1uq2C+6VrttN83md04eqX/1+9367+nI/XfXrW6WIwMzMrDcH9HcBZmY2cDkkzMyskEPCzMwKOSTMzKyQQ8LMzAoN7e8C+tKYMWOitbW1v8swM9unrFy58pWIGNvbtkEVEq2trVSr1f4uw8xsnyLphaJtPt1kZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhUoPCUlTJa2TtEHS7F62Hyvpl5KelrRUUktu2xWS1qevK8qu1czMuis1JCQNAeYC5wEnAZdKOqlHt/8F3BURnwTmAH+fxo4CbgH+AzAZuEVSc5n1mplZd2V/TmIysCEiNgFIWghMB9bm+pwEfCUtLwHuS8vnAg9GxKtp7IPAVGBBGYX+3c/XsPbFN8p4aDOz0p30sUO55fyJff64ZZ9uGgdszq13pLa8p4A/T8t/BoyUNLrGsUiaJakqqdrZ2dlnhZuZ2cD4xPV/B74t6UrgUWALsKvWwRExD5gHUKlUPvIdlMpIYDOzfV3ZIbEFGJ9bb0lt74uIF0lHEpIOAS6KiNckbQHO6jF2aZnFmplZd2WfbloBTJDUJulAYCbQnu8gaYykrjpuAuan5cXAOZKa04T1OanNzMwapNSQiIidwLVkb+7PAIsiYo2kOZIuSN3OAtZJeg44EvifaeyrwDfIgmYFMKdrEtvMzBpDER/5NP6AU6lUwleBNTOrj6SVEVHpbZs/cW1mZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlao9JCQNFXSOkkbJM3uZfsxkpZI+ldJT0ualtpbJb0l6cn0dUfZtZqZWXdDy3xwSUOAucDZQAewQlJ7RKzNdfs6sCgi/lHSScD9QGvatjEiJpVZo5mZFSv7SGIysCEiNkXEu8BCYHqPPgEcmpYPA14suSYzM6tR2SExDticW+9IbXm3ApdL6iA7irgut60tnYZ6RNIf9fYEkmZJqkqqdnZ29mHpZmY2ECauLwX+KSJagGnADyQdAPwOOCYiTgW+AvxQ0qE9B0fEvIioRERl7NixDS3czGywKzsktgDjc+stqS3vKmARQEQsA5qAMRHxTkRsTe0rgY3A8SXXa2ZmOWWHxApggqQ2SQcCM4H2Hn1+C3wWQNKJZCHRKWlsmvhG0nHABGBTyfWamVlOqX/dFBE7JV0LLAaGAPMjYo2kOUA1ItqBrwJ3SrqRbBL7yogISX8MzJG0A3gP+FJEvFpmvWZm1p0ior9r6DOVSiWq1Wp/l2Fmtk+RtDIiKr1tGwgT12ZmNkA5JMzMrJBDwszMCjkkzMyskEPCzMwKOSTMzKyQQ8LMzAo5JMzMrJBDwszMCjkkzMyskEPCzMwKOSTMzKyQQ8LMzAo5JMzMrJBDwszMCjkkzMyskEPCzMwKOSTMzKyQQ8LMzAo5JMzMrJBDwszMCjkkzMyskEPCzMwKOSTMzKxQ6SEhaaqkdZI2SJrdy/ZjJC2R9K+SnpY0LbftpjRunaRzy67VzMy6G1rmg0saAswFzgY6gBWS2iNiba7b14FFEfGPkk4C7gda0/JMYCLwMeAhScdHxK4yazYzsw+UfSQxGdgQEZsi4l1gITC9R58ADk3LhwEvpuXpwMKIeCcifgNsSI9nZmYNUnZIjAM259Y7UlvercDlkjrIjiKuq2MskmZJqkqqdnZ29lXdZmbGwJi4vhT4p4hoAaYBP5BUc10RMS8iKhFRGTt2bGlFmpntj0qdkwC2AONz6y2pLe8qYCpARCyT1ASMqXGsmZmVqOwjiRXABEltkg4km4hu79Hnt8BnASSdCDQBnanfTEkHSWoDJgCPl1yvmZnllHokERE7JV0LLAaGAPMjYo2kOUA1ItqBrwJ3SrqRbBL7yogIYI2kRcBaYCdwjf+yycyssZS9Hw8OlUolqtVqf5dhZrZPkbQyIiq9bRsIE9dmZjZAOSTMzKyQQ8LMzAo5JMzMrJBDwszMCjkkzMyskEPCzMwKOSTMzKxQXSEh6VhJf5qWh0saWU5ZZmY2ENQcEpKuBu4BvpuaWoD7yijKzMwGhnqOJK4BzgDeAIiI9cARZRRlZmYDQz0X+HsnIt6VBICkoWQX5DMz26ft2LGDjo4O3n777f4upVRNTU20tLQwbNiwmsfUExKPSPofwHBJZwN/Bfy8zhrNzAacjo4ORo4cSWtrK12/CA82EcHWrVvp6Oigra2t5nH1nG76a7L7PKwC/pLsVqNfr6tKM7MB6O2332b06NGDNiAAJDF69Oi6j5ZqOpKQNARYExEnAHd+hPrMzAa0wRwQXT7K91jTkUS62c86ScfU/QxmZrZbr732Gt/5znfqHjdt2jRee+21Eir6QD2nm5rJ7hb3S0ntXV9lFWZmtr8oComdO3fudtz999/P4YcfXlZZQH0T139TWhVmZvux2bNns3HjRiZNmsSwYcNoamqiubmZZ599lueee44LL7yQzZs38/bbb3P99dcza9YsAFpbW6lWq2zfvp3zzjuPM888k8cee4xx48bxs5/9jOHDh+91bTWHREQ8IulI4LTU9HhEvLzXFZiZDSB/9/M1rH3xjT59zJM+dii3nD+xcPttt93G6tWrefLJJ1m6dCmf+9znWL169ft/hTR//nxGjRrFW2+9xWmnncZFF13E6NGjuz3G+vXrWbBgAXfeeSeXXHIJ9957L5dffvle117PJ64vAR4HLgYuAX4t6fN7XYGZmXUzefLkbn+mevvtt3PKKacwZcoUNm/ezPr16z80pq2tjUmTJgHw6U9/mueff75PaqnndNPNwGldRw+SxgIPkV2qw8xsUNjdb/yNcvDBB7+/vHTpUh566CGWLVvGiBEjOOuss3r9M9aDDjro/eUhQ4bw1ltv9Ukt9UxcH9Dj9NLWOsebmVkvRo4cyZtvvtnrttdff53m5mZGjBjBs88+y/LlyxtaWz1HEv9X0mJgQVqfATzQ9yWZme1fRo8ezRlnnMEnPvEJhg8fzpFHHvn+tqlTp3LHHXdw4okn8vGPf5wpU6Y0tDZF1H75JUl/DpyZVv9fRPy0lKo+okqlEtVqtb/LMLN9zDPPPMOJJ57Y32U0RG/fq6SVEVHprX/NRxKS2oD7I+InaX24pNaIeH4P46YC/xsYAnwvIm7rsf1bwGfS6gjgiIg4PG3bRXYZEIDfRsQFtdZrZmZ7r57TTT8GTs+t70ptp/Xe/f3LecwFzgY6gBWS2iNibVefiLgx1/864NTcQ7wVEZPqqNHMzPpQPRPPQyPi3a6VtHzgHsZMBjZExKbUfyEwfTf9L+WDOQ8zM+tn9YREp6T3T/dImg68socx44DNufWO1PYhko4F2oCHc81NkqqSlku6sGDcrNSn2tnZWcv3YWZmNarndNOXgLslfRsQ2Zv/f+nDWmYC96SLCXY5NiK2SDoOeFjSqojYmB8UEfOAeZBNXPdhPWZm+716LsuxEZgi6ZC0vr2GYVuA8bn1ltTWm5lkt0jNP+eW9O8mSUvJ5is2fniomZmVoZ7Lclwv6VDg34B/kPSEpHP2MGwFMEFSm6QDyYLgQ1eOlXQC2VVml+XamiUdlJbHkN1fe23PsWZm+5tDDjmkYc9Vz5zEX0TEG8A5wGjgC8BtuxsQETuBa4HFwDPAoohYI2lOfn6DLDwWRvcPbZwIVCU9BSwBbsv/VZSZmZWvnjmJrlsaTQPuSm/2e7zNUUTcT3ar03zb3/ZYv7WXcY8BJ9dRn5nZPmn27NmMHz+ea67JzrjfeuutDB06lCVLlrBt2zZ27NjBN7/5TaZP390fh5ajnpBYKekXZH+BdJOkkcB75ZRlZtZPHpgNL63ac796HHUynFd84mXGjBnccMMN74fEokWLWLx4MV/+8pc59NBDeeWVV5gyZQoXXHBBw2+zWk9IXAVMAjZFxB8kjQa+2LVR0sSIWNPXBZqZDXannnoqL7/8Mi+++CKdnZ00Nzdz1FFHceONN/Loo49ywAEHsGXLFn7/+99z1FFHNbS2ev666T3gidz6VrIrwXb5AfCpvivNzKwf7OY3/jJdfPHF3HPPPbz00kvMmDGDu+++m87OTlauXMmwYcNobW3t9RLhZavnSGJPGnsMZGY2iMyYMYOrr76aV155hUceeYRFixZxxBFHMGzYMJYsWcILL7zQL3X1ZUj4g2xmZh/RxIkTefPNNxk3bhxHH300l112Geeffz4nn3wylUqFE044oV/q6suQMDOzvbBq1QcT5mPGjGHZsmW99tu+vZbPMveNvryz3Lt77mJmZvuSvQqJ9ElpACKisbdLMjOz0u3tkcQv+qQKMzMbkPY4JyHp9qJNwOF9W46ZWf+IiIZ/UK3R6rlddZdaJq6/CHwVeKeXbZfW/YxmZgNMU1MTW7duZfTo0YM2KCKCrVu30tTUVNe4WkJiBbA6XUupG0m31vVsZmYDUEtLCx0dHQz2G5c1NTXR0tJS15haQuLzQK8f84uItrqezcxsABo2bBhtbX47600tE9eHRMQfSq/EzMwGnFpC4r6uBUn3lliLmZkNMLWERH4W57iyCjEzs4GnlpCIgmUzMxvkapm4PkXSG2RHFMPTMmk9IuLQ0qozM7N+tceQiIghjSjEzMwGnr68wJ+ZmQ0yDgkzMyvkkDAzs0IOCTMzK+SQMDOzQqWHhKSpktZJ2iBpdi/bvyXpyfT1nKTXctuukLQ+fV1Rdq1mZtZdqfe4ljQEmAucDXQAKyS1R8Tarj4RcWOu/3XAqWl5FHALUCH7EN/KNHZbmTWbmdkHyj6SmAxsiIhNEfEusBCYvpv+lwIL0vK5wIMR8WoKhgeBqaVWa2Zm3ZQdEuOAzbn1jtT2IZKOBdqAh+sZK2mWpKqk6mC/FryZWaMNpInrmcA9EbGrnkERMS8iKhFRGTt2bEmlmZntn8oOiS3A+Nx6S2rrzUw+ONVU71gzMytB2SGxApggqU3SgWRB0N6zk6QTgGZgWa55MXCOpGZJzcA5qc3MzBqk1L9uioidkq4le3MfAsyPiDWS5gDViOgKjJnAwoiI3NhXJX2DLGgA5kTEq2XWa2Zm3Sn3vrzPq1QqUa1W+7sMM7N9iqSVEVHpbdtAmrg2M7MBxiFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhUoPCUlTJa2TtEHS7II+l0haK2mNpB/m2ndJejJ9tZddq5mZdTe0zAeXNASYC5wNdAArJLVHxNpcnwnATcAZEbFN0hG5h3grIiaVWaOZmRUr+0hiMrAhIjZFxLvAQmB6jz5XA3MjYhtARLxcck1mZlajskNiHLA5t96R2vKOB46X9CtJyyVNzW1rklRN7Rf29gSSZqU+1c7Ozr6t3sxsP1fq6aYaDQUmAGcBLcCjkk6OiNeAYyNii6TjgIclrYqIjfnBETEPmAdQqVSisaWbmQ1uZR9JbAHG59ZbUlteB9AeETsi4jfAc2ShQURsSf9uApYCp5Zcr5mZ5ZQdEiuACZLaJB0IzAR6/pXSfWRHEUgaQ3b6aZOkZkkH5drPANZiZmYNU+rppojYKelaYDEwBJgfEWskzQGqEdGetp0jaS2wC/haRGyVdDrwXUnvkYXZbfm/ijIzs/IpYvCcxq9UKlGtVvu7DDOzfYqklRFR6W2bP3FtZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhZmaFHBJmZlbIIWFmZoUcEmZmVsghYWZmhRwSZmZWqPSQkDRV0jpJGyTNLuhziaS1ktZI+mGu/QpJ69PXFWXXamZm3Q0t88ElDQHmAmcDHcAKSe0RsTbXZwJwE3BGRGyTdERqHwXcAlSAAFamsdvKrNnMzD5QakgAk4ENEbEJQNJCYDqwNtfnamBu15t/RLyc2s8FHoyIV9PYB4GpwIJSKn1gNry0qpSHNjMr3VEnw3m39fnDln26aRywObfekdryjgeOl/QrScslTa1jLJJmSapKqnZ2dvZh6WZmVvaRRC2GAhOAs4AW4FFJJ9c6OCLmAfMAKpVKfOQqSkhgM7N9XdlHEluA8bn1ltSW1wG0R8SOiPgN8BxZaNQy1szMSlR2SKwAJkhqk3QgMBNo79HnPrKjCCSNITv9tAlYDJwjqVlSM3BOajMzswYp9XRTROyUdC3Zm/sQYH5ErJE0B6hGRDsfhMFaYBfwtYjYCiDpG2RBAzCnaxLbzMwaQxEf/TT+QFOpVKJarfZ3GWZm+xRJKyOi0ts2f+LazMwKOSTMzKyQQ8LMzAo5JMzMrNCgmriW1Am8sBcPMQZ4pY/K6Uuuqz6uqz6uqz6Dsa5jI2JsbxsGVUjsLUnVohn+/uS66uO66uO66rO/1eXTTWZmVsghYWZmhRwS3c3r7wIKuK76uK76uK767Fd1eU7CzMwK+UjCzMwKOSTMzKzQfhESkqZKWidpg6TZvWw/SNKP0vZfS2rNbbspta+TdG6D6/qKpLWSnpb0S0nH5rbtkvRk+up5+fWy67pSUmfu+f9rbtsVktanrysaXNe3cjU9J+m13LYy99d8SS9LWl2wXZJuT3U/LelTuW1l7q891XVZqmeVpMcknZLb9nxqf1JSn141s4a6zpL0eu7n9be5bbt9DZRc19dyNa1Or6lRaVuZ+2u8pCXpvWCNpOt76VPeaywiBvUX2SXKNwLHAQcCTwEn9ejzV8AdaXkm8KO0fFLqfxDQlh5nSAPr+gwwIi3/t6660vr2ftxfVwLf7mXsKLJ7gYwCmtNyc6Pq6tH/OrJL05e6v9Jj/zHwKWB1wfZpwAOAgCnAr8veXzXWdXrX8wHnddWV1p8HxvTT/joL+Oe9fQ30dV09+p4PPNyg/XU08Km0PJLsxmw9/0+W9hrbH44kJgMbImJTRLwLLASm9+gzHfh+Wr4H+KwkpfaFEfFOZHfN25AeryF1RcSSiPhDWl1Odne+stWyv4qcCzwYEa9GxDbgQWDqHsaUVdelwII+eu7diohHgd3d62Q6cFdklgOHSzqacvfXHuuKiMfS80LjXl+17K8ie/Pa7Ou6Gvn6+l1EPJGW3wSeAcb16Fbaa2x/CIlxwObcegcf3sHv94mIncDrwOgax5ZZV95VZL8pdGmSVJW0XNKFfVRTPXVdlA5r75HUdZvZAbG/0mm5NuDhXHNZ+6sWRbWXub/q1fP1FcAvJK2UNKsf6vmPkp6S9ICkialtQOwvSSPI3mjvzTU3ZH8pOxV+KvDrHptKe42Vemc66xuSLgcqwJ/kmo+NiC2SjgMelrQqIjY2qKSfAwsi4h1Jf0l2FPafGvTctZgJ3BMRu3Jt/bm/BjRJnyELiTNzzWem/XUE8KCkZ9Nv2o3wBNnPa7ukaWS3OJ7QoOeuxfnAr6L7nTJL31+SDiELphsi4o2+fOzd2R+OJLYA43PrLamt1z6ShgKHAVtrHFtmXUj6U+Bm4IKIeKerPSK2pH83AUvJfrtoSF0RsTVXy/eAT9c6tsy6cmbS41RAifurFkW1l7m/aiLpk2Q/w+mRbhsM3fbXy8BP6bvTrHsUEW9ExPa0fD8wTNIYBsD+Snb3+iplf0kaRhYQd0fET3rpUt5rrIyJloH0RXa0tIns9EPXZNfEHn2uofvE9aK0PJHuE9eb6LuJ61rqOpVsom5Cj/Zm4KC0PAZYTx9N4NVY19G55T8DlscHk2S/SfU1p+VRjaor9TuBbBJRjdhfuedopXgi9nN0n1R8vOz9VWNdx5DNs53eo/1gYGRu+TFgagPrOqrr50f2ZvvbtO9qeg2UVVfafhjZvMXBjdpf6Xu/C/iH3fQp7TXWZzt3IH+Rzfw/R/aGe3Nqm0P22zlAE/Dj9B/mceC43Nib07h1wHkNrush4PfAk+mrPbWfDqxK/0lWAVc1uK6/B9ak518CnJAb+xdpP24AvtjIutL6rcBtPcaVvb8WAL8DdpCd870K+BLwpbRdwNxU9yqg0qD9tae6vgdsy72+qqn9uLSvnko/55sbXNe1udfXcnIh1ttroFF1pT5Xkv0xS35c2fvrTLI5j6dzP6tpjXqN+bIcZmZWaH+YkzAzs4/IIWFmZoUcEmZmVsghYWZmhRwSZmZWyCFhNkCkq5/+c3/XYZbnkDAzs0IOCbM6Sbpc0uPp3gHflTRE0vZ0P4s1yu79MTb1nZQuKvi0pJ9Kak7t/17SQ+kidk9I+nfp4Q9JF018VtLd6WrEZv3GIWFWB0knAjOAMyJiErALuIzscgzViJgIPALckobcBfx1RHyS7JOwXe13A3Mj4hSyT4T/LrWfCtxAdi+T44AzSv+mzHbDV4E1q89nyS5ouCL9kj8ceBl4D/hR6vN/gJ9IOgw4PCIeSe3fB34saSQwLiJ+ChARbwOkx3s8IjrS+pNk1xL6l/K/LbPeOSTM6iPg+xFxU7dG6W969Puo17t5J7e8C/8ftX7m001m9fkl8Pl03wAkjUo3OToA+Hzq85+Bf4mI14Ftkv4otX8BeCSyu4t1dN38SNk91kc09Lswq5F/SzGrQ0SslfR1sruQHUB2xdBrgH8DJqdtL5PNWwBcAdyRQmAT8MXU/gXgu5LmpMe4uIHfhlnNfBVYsz4gaXtEHNLfdZj1NZ9uMjOzQj6SMDOzQj6SMDOzQg4JMzMr5JAwM7NCDgkzMyvkkDAzs0L/H4+W+6c9WeDOAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAap0lEQVR4nO3df5RV5X3v8fdHRPEH6vDDaBgiY4MRiAnEI/VKbE1SEbGKrY1itUtzvdr0qrma3qxik1ZD7K2397bpyiqpYsOtNkZCTMwiWXq5GkFXE4kcUhRBESQaBmMcEPzRKoL53j/2M7oZHmbO4OwzA3xea53F3s9+nn2+s+dwPrP3PmdvRQRmZmZdHdDfBZiZ2cDkgDAzsywHhJmZZTkgzMwsywFhZmZZB/Z3AX1lxIgRMWbMmP4uw8xsr7J8+fJNETEyt2yfCYgxY8ZQr9f7uwwzs72KpOd3t8yHmMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIqDQhJ0yStkbRO0qzM8uMk/UjSE5KWSGotLbtM0tr0uKzKOs3MbFeVBYSkQcAc4GxgPHCxpPFduv1v4M6I+AgwG/jrNHYYcCPwm8Bk4EZJLVXVamZmu6ryexCTgXURsR5A0nxgBrC61Gc88Pk0vRj4fpo+C3ggIl5OYx8ApgF3V1Hol3+witUvvFrFqs3MKjf+/Udw47kT+ny9VR5iGgVsKM23p7ayx4HfT9O/BwyVNLzBsUi6SlJdUr2jo6PPCjczs/7/JvV/B/5B0uXAI8BG4O1GB0fEXGAuQK1W2+M7H1WRvGZme7sqA2IjMLo035ra3hERL5D2ICQdDlwQEVslbQTO6DJ2SYW1mplZF1UeYloGjJXUJukgYCawsNxB0ghJnTXcAMxL04uAqZJa0snpqanNzMyapLKAiIgdwDUUb+xPAQsiYpWk2ZLOS93OANZIegZ4H/BXaezLwFcoQmYZMLvzhLWZmTWHIvb40P2AUqvVwldzNTPrHUnLI6KWW+ZvUpuZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpblgDAzsywHhJmZZTkgzMwsywFhZmZZDggzM8uqNCAkTZO0RtI6SbMyyz8gabGkf5P0hKTpqX2MpDckrUiPW6us08zMdnVgVSuWNAiYA5wJtAPLJC2MiNWlbl8CFkTEP0oaD9wHjEnLno2IiVXVZ2Zm3atyD2IysC4i1kfEW8B8YEaXPgEckaaPBF6osB4zM+uFKgNiFLChNN+e2spuAi6V1E6x93BtaVlbOvT0sKTTc08g6SpJdUn1jo6OPizdzMz6+yT1xcA/R0QrMB34F0kHAL8EPhARk4DPA9+SdETXwRExNyJqEVEbOXJkUws3M9vXVRkQG4HRpfnW1FZ2BbAAICIeBYYAIyJiW0RsTu3LgWeBEyqs1czMuqgyIJYBYyW1SToImAks7NLnF8CnACSNowiIDkkj00luJB0PjAXWV1irmZl1UdmnmCJih6RrgEXAIGBeRKySNBuoR8RC4E+B2yVdT3HC+vKICEm/BcyWtB34NfDZiHi5qlrNzGxXioj+rqFP1Gq1qNfr/V2GmdleRdLyiKjllvX3SWozMxugHBBmZpblgDAzsywHhJmZZTkgzMwsywFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZVqUBIWmapDWS1kmalVn+AUmLJf2bpCckTS8tuyGNWyPprCrrNDOzXR1Y1YolDQLmAGcC7cAySQsjYnWp25eABRHxj5LGA/cBY9L0TGAC8H7gQUknRMTbVdVrZmY7q3IPYjKwLiLWR8RbwHxgRpc+ARyRpo8EXkjTM4D5EbEtIn4OrEvrMzOzJqkyIEYBG0rz7amt7CbgUkntFHsP1/ZiLJKuklSXVO/o6Oirus3MjP4/SX0x8M8R0QpMB/5FUsM1RcTciKhFRG3kyJGVFWlmtj+q7BwEsBEYXZpvTW1lVwDTACLiUUlDgBENjjUzswpVuQexDBgrqU3SQRQnnRd26fML4FMAksYBQ4CO1G+mpIMltQFjgccqrNXMzLqobA8iInZIugZYBAwC5kXEKkmzgXpELAT+FLhd0vUUJ6wvj4gAVklaAKwGdgBX+xNMZmbNpeL9eO9Xq9WiXq/3dxlmZnsVScsjopZb1t8nqc3MbIByQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ19k1rSFIorrx6XxgiIiDi+utLMzKw/NXqpjW8A1wPLAV/ywsxsP9BoQLwSEfdXWomZmQ0ojQbEYkn/C/gesK2zMSJ+VklVZmZNsn37dtrb23nzzTf7u5RKDRkyhNbWVgYPHtzwmEYD4jfTv+ULOgXwyYafycxsAGpvb2fo0KGMGTMGSf1dTiUigs2bN9Pe3k5bW1vD4xoKiIj4xB5XZmY2gL355pv7dDgASGL48OH09tbMDX3MVdKRkv6u8/7Pkv5W0pF7VKmZ2QCzL4dDpz35GRv9HsQ84DXgwvR4Ffg/vX42MzPbydatW/n617/e63HTp09n69atFVT0rkYD4jci4saIWJ8eXwb8HQgzs/dodwGxY8eObsfdd999HHXUUVWVBTQeEG9I+njnTPri3BvVlGRmtv+YNWsWzz77LBMnTuSUU07h9NNP57zzzmP8+PEAnH/++Zx88slMmDCBuXPnvjNuzJgxbNq0ieeee45x48Zx5ZVXMmHCBKZOncobb/TN23Ojn2L6E+COdN5BwMvA5X1SgZnZAPHlH6xi9Quv9uk6x7//CG48d8Jul99yyy08+eSTrFixgiVLlnDOOefw5JNPvvNpo3nz5jFs2DDeeOMNTjnlFC644AKGDx++0zrWrl3L3Xffze23386FF17Id7/7XS699NL3XHujn2JaAXxU0hFpvm+3oJmZATB58uSdPor6ta99jXvvvReADRs2sHbt2l0Coq2tjYkTJwJw8skn89xzz/VJLd0GhKRLI+Kbkj7fpR2AiPi7PqnCzGwA6O4v/WY57LDD3plesmQJDz74II8++iiHHnooZ5xxRvYLfQcffPA704MGDWraIabOSof2ybOZmdlOhg4dymuvvZZd9sorr9DS0sKhhx7K008/zdKlS5taW7cBERG3pX+/3JxyzMz2L8OHD2fKlCl8+MMf5pBDDuF973vfO8umTZvGrbfeyrhx4/jQhz7Eqaee2tTaFBE9d5L+BriZ4pNL/xf4CHB9RHyz2vIaV6vVol6v93cZZraXeeqppxg3blx/l9EUuZ9V0vKIqOX6N/ox16npxPTvAs8BHwS+0NMgSdMkrZG0TtKszPKvSlqRHs9I2lpa9nZp2cIG6zQzsz7S6MdcO/udA3wnIl7p6WvbkgYBc4AzgXZgmaSFEbG6s09EXF/qfy0wqbSKNyJiYoP1mZlZH2t0D+KHkp4GTgZ+JGkk0NO1cScD69I3r98C5gMzuul/MXB3g/WYmVnFGgqIiJgFnAbUImI78O90/2YPMArYUJpvT227kHQc0AY8VGoeki4MuFTS+bsZd1XnBQR7e5VCMzPrXk/fg/hkRDwk6fdLbeUu3+ujOmYC90RE+Xamx0XERknHAw9JWhkRz5YHRcRcYC4UJ6n7qBYzM6PncxC/TfFX/bmZZUH3AbERGF2ab01tOTOBq3daecTG9O96SUsozk88u+tQMzOrQk/fg7gx/fuZPVj3MmCspDaKYJgJ/GHXTpJOBFqAR0ttLcB/RMQ2SSOAKcDf7EENZmb7lMMPP5zXX3+9Kc/V6A2D/oeko0rzLZJu7m5MROwArgEWAU8BCyJilaTZks4rdZ0JzI+dv5AxDqhLehxYDNxS/vSTmZlVr9GPuZ4dEX/eORMRWyRNB77U3aCIuA+4r0vbX3aZvykz7ifASQ3WZma215o1axajR4/m6quLo+w33XQTBx54IIsXL2bLli1s376dm2++mRkzevpcUN9rNCAGSTo4IrYBSDoEOLiHMWZme5f7Z8GLK/t2ncecBGffstvFF110Edddd907AbFgwQIWLVrE5z73OY444gg2bdrEqaeeynnnndf0W6M2GhB3UXz/ofM2o58B7qimJDOz/cekSZN46aWXeOGFF+jo6KClpYVjjjmG66+/nkceeYQDDjiAjRs38qtf/YpjjjmmqbU1ej+I/5nOB/xOavpKRCyqriwzs37QzV/6Vfr0pz/NPffcw4svvshFF13EXXfdRUdHB8uXL2fw4MGMGTMme5nvqjW6BwHFieYdEfGgpEMlDY2I/DVqzcysYRdddBFXXnklmzZt4uGHH2bBggUcffTRDB48mMWLF/P888/3S12NforpSuAe4LbUNAr4flVFmZntTyZMmMBrr73GqFGjOPbYY7nkkkuo1+ucdNJJ3HnnnZx44on9UlejexBXU1xb6acAEbFW0tGVVWVmtp9ZufLdk+MjRozg0UcfzfZr1ncgoPGL9W1LF9wDQNKBFN+kNjOzfVSjAfGwpD8HDpF0JvAd4AfVlWVmZv2t0YD4M6ADWAn8McWX37r9kpyZme3dejwHkW78syoiTgRur74kM7Pmioimfwmt2Rq5vXRXPe5BpEtwr5H0gT0pysxsIBsyZAibN2/eozfQvUVEsHnzZoYMGdKrcY1+iqkFWCXpMYqbBXU+6Xm7H2JmNvC1trbS3t7Ovn7TsSFDhtDa2tqrMY0GxF/0vhwzs4Fv8ODBtLW19XcZA1JPd5QbAnwW+CDFCepvpMt4m5nZPq6ncxB3ADWKcDgb+NvKKzIzswGhp0NM4yPiJABJ3wAeq74kMzMbCHrag9jeOeFDS2Zm+5ee9iA+KunVNC2Kb1K/mqYjIo6otDozM+s33QZERAxqViFmZjawNHqpDTMz2884IMzMLMsBYWZmWQ4IMzPLckCYmVlWpQEhaZqkNZLWSZqVWf5VSSvS4xlJW0vLLpO0Nj0uq7JOMzPbVaMX6+u1dB+JOcCZQDuwTNLCiFjd2Sciri/1vxaYlKaHATdSXOYjgOVp7Jaq6jUzs51VuQcxGVgXEevT/aznAzO66X8xcHeaPgt4ICJeTqHwADCtwlrNzKyLKgNiFLChNN+e2nYh6TigDXioN2MlXSWpLqm+r1/L3cys2QbKSeqZwD3p7nUNi4i5EVGLiNrIkSMrKs3MbP9UZUBsBEaX5ltTW85M3j281NuxZmZWgSoDYhkwVlKbpIMoQmBh106STqS4pemjpeZFwFRJLZJagKmpzczMmqSyTzFFxA5J11C8sQ8C5kXEKkmzgXpEdIbFTGB+lO4YHhEvS/oKRcgAzI6Il6uq1czMdqXS+/JerVarRb1e7+8yzMz2KpKWR0Qtt2ygnKQ2M7MBxgFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpblgDAzsywHhJmZZVUaEJKmSVojaZ2kWbvpc6Gk1ZJWSfpWqf1tSSvSY2GVdZqZ2a4OrGrFkgYBc4AzgXZgmaSFEbG61GcscAMwJSK2SDq6tIo3ImJiVfWZmVn3qtyDmAysi4j1EfEWMB+Y0aXPlcCciNgCEBEvVViPmZn1QpUBMQrYUJpvT21lJwAnSPqxpKWSppWWDZFUT+3n555A0lWpT72jo6Nvqzcz289VdoipF88/FjgDaAUekXRSRGwFjouIjZKOBx6StDIini0Pjoi5wFyAWq0WzS3dzGzfVuUexEZgdGm+NbWVtQMLI2J7RPwceIYiMIiIjenf9cASYFKFtZqZWRdVBsQyYKykNkkHATOBrp9G+j7F3gOSRlAcclovqUXSwaX2KcBqzMysaSo7xBQROyRdAywCBgHzImKVpNlAPSIWpmVTJa0G3ga+EBGbJZ0G3Cbp1xQhdkv5009mZlY9Rewbh+5rtVrU6/X+LsPMbK8iaXlE1HLL/E1qMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpblgDAzsywHhJmZZTkgzMwsywFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWZUGhKRpktZIWidp1m76XChptaRVkr5Var9M0tr0uKzKOs3MbFcHVrViSYOAOcCZQDuwTNLCiFhd6jMWuAGYEhFbJB2d2ocBNwI1IIDlaeyWquo1M7OdVRYQwGRgXUSsB5A0H5gBrC71uRKY0/nGHxEvpfazgAci4uU09gFgGnB3JZXePwteXFnJqs3MKnfMSXD2LX2+2ioPMY0CNpTm21Nb2QnACZJ+LGmppGm9GIukqyTVJdU7Ojr6sHQzM6tyD6LR5x8LnAG0Ao9IOqnRwRExF5gLUKvVYo+rqCB5zcz2dlXuQWwERpfmW1NbWTuwMCK2R8TPgWcoAqORsWZmVqEqA2IZMFZSm6SDgJnAwi59vk+x94CkERSHnNYDi4CpkloktQBTU5uZmTVJZYeYImKHpGso3tgHAfMiYpWk2UA9IhbybhCsBt4GvhARmwEkfYUiZABmd56wNjOz5lDEnh+6H0hqtVrU6/X+LsPMbK8iaXlE1HLL/E1qMzPLckCYmVmWA8LMzLIcEGZmlrXPnKSW1AE8/x5WMQLY1Efl9CXX1Tuuq3dcV+/si3UdFxEjcwv2mYB4ryTVd3cmvz+5rt5xXb3junpnf6vLh5jMzCzLAWFmZlkOiHfN7e8CdsN19Y7r6h3X1Tv7VV0+B2FmZlnegzAzsywHhJmZZe3zASFpmqQ1ktZJmpVZfrCkb6flP5U0prTshtS+RtJZTa7r85JWS3pC0o8kHVda9rakFenR9RLqVdd1uaSO0vP/l9KyyyStTY/LmlzXV0s1PSNpa2lZldtrnqSXJD25m+WS9LVU9xOSPlZaVuX26qmuS1I9KyX9RNJHS8ueS+0rJPXpFTAbqOsMSa+Ufl9/WVrW7Wug4rq+UKrpyfSaGpaWVbm9RktanN4LVkn6b5k+1b3GImKffVBcZvxZ4HjgIOBxYHyXPv8VuDVNzwS+nabHp/4HA21pPYOaWNcngEPT9J901pXmX+/H7XU58A+ZscMo7uUxDGhJ0y3NqqtL/2spLi9f6fZK6/4t4GPAk7tZPh24HxBwKvDTqrdXg3Wd1vl8wNmddaX554AR/bS9zgB++F5fA31dV5e+5wIPNWl7HQt8LE0PpbipWtf/k5W9xvb1PYjJwLqIWB8RbwHzgRld+swA7kjT9wCfkqTUPj8itkVxt7t1aX1NqSsiFkfEf6TZpRR31ataI9trd84CHoiIlyNiC/AAMK2HMVXVdTFwdx89d7ci4hGgu3uVzADujMJS4ChJx1Lt9uqxroj4SXpeaN7rq5HttTvv5bXZ13U18/X1y4j4WZp+DXgKGNWlW2WvsX09IEYBG0rz7ey6cd/pExE7gFeA4Q2OrbKusiso/kLoNERSXdJSSef3UU29qeuCtCt7j6TOW8MOiO2VDsW1AQ+VmqvaXo3YXe1Vbq/e6vr6CuD/SVou6ap+qOc/SXpc0v2SJqS2AbG9JB1K8Sb73VJzU7aXisPfk4CfdllU2WussjvKWd+QdClQA3671HxcRGyUdDzwkKSVEfFsk0r6AXB3RGyT9McUe1+fbNJzN2ImcE9EvF1q68/tNaBJ+gRFQHy81PzxtL2OBh6Q9HT6C7sZfkbx+3pd0nSK2xKPbdJzN+Jc4Mex8x0uK99ekg6nCKXrIuLVvlx3d/b1PYiNwOjSfGtqy/aRdCBwJLC5wbFV1oWk3wG+CJwXEds62yNiY/p3PbCE4q+KptQVEZtLtfwTcHKjY6usq2QmXXb/K9xejdhd7VVur4ZI+gjF73BGpFv9wk7b6yXgXvru0GqPIuLViHg9Td8HDFZxv/p+315Jd6+vSraXpMEU4XBXRHwv06W611gVJ1YGyoNiD2k9xSGHzhNbE7r0uZqdT1IvSNMT2Pkk9Xr67iR1I3VNojgpN7ZLewtwcJoeAaylj07WNVjXsaXp3wOWxrsnxH6e6mtJ08OaVVfqdyLFCUM1Y3uVnmMMuz/peg47n0B8rOrt1WBdH6A4r3Zal/bDgKGl6Z8A05pY1zGdvz+KN9pfpG3X0GugqrrS8iMpzlMc1qztlX72O4G/76ZPZa+xPtu4A/VBcYb/GYo32y+mttkUf5UDDAG+k/6zPAYcXxr7xTRuDXB2k+t6EPgVsCI9Fqb204CV6T/ISuCKJtf118Cq9PyLgRNLY/9z2o7rgM80s640fxNwS5dxVW+vu4FfAtspjvFeAXwW+GxaLmBOqnslUGvS9uqprn8CtpReX/XUfnzaVo+n3/MXm1zXNaXX11JKAZZ7DTSrrtTncooPrpTHVb29Pk5xjuOJ0u9qerNeY77UhpmZZe3r5yDMzGwPOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCbABIVzH9YX/XYVbmgDAzsywHhFkvSLpU0mPp2v+3SRok6fV0P4pVKu7dMTL1nZguEPiEpHsltaT2D0p6MF2Q7meSfiOt/vB0AcSnJd2Vrips1m8cEGYNkjQOuAiYEhETgbeBSygusVCPiAnAw8CNacidwJ9FxEcovuHa2X4XMCciPkrxTe9fpvZJwHUU9yI5HphS+Q9l1g1fzdWscZ+iuDjhsvTH/SHAS8CvgW+nPt8EvifpSOCoiHg4td8BfEfSUGBURNwLEBFvAqT1PRYR7Wl+BcW1gf61+h/LLM8BYdY4AXdExA07NUp/0aXfnl6/Zltp+m38/9P6mQ8xmTXuR8AfpOv+I2lYukHRAcAfpD5/CPxrRLwCbJF0emr/I+DhKO4K1t554yIV90Q/tKk/hVmD/BeKWYMiYrWkL1HcPewAiit/Xg38OzA5LXuJ4jwFwGXArSkA1gOfSe1/BNwmaXZax6eb+GOYNcxXczV7jyS9HhGH93cdZn3Nh5jMzCzLexBmZpblPQgzM8tyQJiZWZYDwszMshwQZmaW5YAwM7Os/w930SM6plNcBQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZ6UlEQVR4nO3df5BV5Z3n8fcniLQoaPPDH6HRbmcxItFAcsO46sw6E38gKcWMieDoFM46stlVo8lsajFmR4dkd/xjqzKVConBFDVJ1kAYMyZkyiyLEbQmQuSSQRGUH6KGRg0tgsqOKJDv/nGf1kPz0NyWPreh+byqbnHOc55z7rdPX+6nz3nuPUcRgZmZWVcf6usCzMzs8OSAMDOzLAeEmZllOSDMzCzLAWFmZlnH9HUBvWXEiBHR2tra12WYmR1RVq5c+VpEjMwt6zcB0draSrVa7esyzMyOKJJeOtAyn2IyM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLKjUgJE2StE7SRkkzM8vPkPRLSU9LWiqppbBsuqQN6TG9zDrNzGx/pQWEpAHAbOAK4BzgOknndOn2v4AfRMR5wCzg79K6w4C7gT8EJgJ3S2ouq1YzM9tfmd+DmAhsjIhNAJLmA1OAtYU+5wBfStNLgJ+m6cuBxRHxelp3MTAJmFdGoX/78zWsffnNMjZtZla6cz48lLuvHNfr2y3zFNMoYHNhvj21FT0F/Fma/gwwRNLwOtdF0gxJVUnVjo6OXivczMz6/pvU/xX4lqQbgceBLcDeeleOiDnAHIBKpfKB73xURvKamR3pygyILcDownxLantPRLxMOoKQdAJwTUTskLQFuLjLuktLrNXMzLoo8xTTCmCMpDZJxwLTgIXFDpJGSOqs4U5gbppeBFwmqTkNTl+W2szMrEFKC4iI2APcSu2N/VlgQUSskTRL0lWp28XAOknrgVOA/5HWfR34GrWQWQHM6hywNjOzxlDEBz51f1ipVCrhq7mamfWMpJURUckt8zepzcwsywFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpblgDAzsywHhJmZZZUaEJImSVonaaOkmZnlp0taIulfJT0taXJqb5X0tqRV6XFfmXWamdn+jilrw5IGALOBS4F2YIWkhRGxttDtq8CCiPiOpHOAh4HWtOz5iBhfVn1mZta9Mo8gJgIbI2JTRLwLzAemdOkTwNA0fSLwcon1mJlZD5QZEKOAzYX59tRWdA9wg6R2akcPtxWWtaVTT49J+qPcE0iaIakqqdrR0dGLpZuZWV8PUl8H/ENEtACTgR9K+hDwCnB6REwAvgT8SNLQritHxJyIqEREZeTIkQ0t3MysvyszILYAowvzLamt6CZgAUBELAOagBER8U5EbEvtK4HngbNKrNXMzLooMyBWAGMktUk6FpgGLOzS57fApwAkjaUWEB2SRqZBbiSdCYwBNpVYq5mZdVHap5giYo+kW4FFwABgbkSskTQLqEbEQuCvgfslfZHagPWNERGS/hiYJWk38Hvg8xHxelm1mpnZ/hQRfV1Dr6hUKlGtVvu6DDOzI4qklRFRyS3r60FqMzM7TDkgzMwsywFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpblgDAzs6xSA0LSJEnrJG2UNDOz/HRJSyT9q6SnJU0uLLszrbdO0uVl1mlmZvs7pqwNSxoAzAYuBdqBFZIWRsTaQrevAgsi4juSzgEeBlrT9DRgHPBh4BFJZ0XE3rLqNTOzfZV5BDER2BgRmyLiXWA+MKVLnwCGpukTgZfT9BRgfkS8ExEvABvT9szMrEHKDIhRwObCfHtqK7oHuEFSO7Wjh9t6sC6SZkiqSqp2dHT0Vt1mZkbfD1JfB/xDRLQAk4EfSqq7poiYExGViKiMHDmytCLNzI5GpY1BAFuA0YX5ltRWdBMwCSAilklqAkbUua6ZmZWozCOIFcAYSW2SjqU26LywS5/fAp8CkDQWaAI6Ur9pkgZJagPGAE+WWKuZmXVR2hFEROyRdCuwCBgAzI2INZJmAdWIWAj8NXC/pC9SG7C+MSICWCNpAbAW2APc4k8wmZk1lmrvx0e+SqUS1Wq1r8swMzuiSFoZEZXcsr4epDYzs8OUA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpZV5uW+zcwOe7t376a9vZ1du3b1dSmlampqoqWlhYEDB9a9jgPCzI5q7e3tDBkyhNbWViT1dTmliAi2bdtGe3s7bW1tda/nU0xmdlTbtWsXw4cP77fhACCJ4cOH9/goyQFhZke9/hwOnT7Iz+iAMDPrQzt27ODb3/52j9ebPHkyO3bsKKGi93UbEJLekvRm5vGWpDdLrczM7ChwoIDYs2dPt+s9/PDDnHTSSWWVBRxkkDoihpT67GZmR7mZM2fy/PPPM378eAYOHEhTUxPNzc0899xzrF+/nquvvprNmzeza9cubr/9dmbMmAFAa2sr1WqVnTt3csUVV3DRRRfxxBNPMGrUKH72s59x3HHHHXJt3QaEpGHdLY+I1w+5AjOzw8Tf/nwNa1/u3ZMj53x4KHdfOe6Ay++9916eeeYZVq1axdKlS/n0pz/NM888896njebOncuwYcN4++23+eQnP8k111zD8OHD99nGhg0bmDdvHvfffz/XXnstP/nJT7jhhhsOufaDfcx1JRBAbnQjgDMPuQIzM3vPxIkT9/ko6je/+U0eeughADZv3syGDRv2C4i2tjbGjx8PwCc+8QlefPHFXqnlYKeY6v/ArJnZEa67v/Qb5fjjj39veunSpTzyyCMsW7aMwYMHc/HFF2c/qjpo0KD3pgcMGMDbb7/dK7XU/UU5Sc3AGKCpsy0iHu+VKszMjlJDhgzhrbfeyi574403aG5uZvDgwTz33HMsX768obXVFRCS/gq4HWgBVgHnA8uAPy2vNDOz/m/48OFceOGFfPSjH+W4447jlFNOeW/ZpEmTuO+++xg7diwf+chHOP/88xtamyLi4J2k1cAngeURMV7S2cD/jIg/K7vAelUqlahWq31dhpkdYZ599lnGjh3b12U0RO5nlbQyIiq5/vV+UW5XROxKGxsUEc8BHznYSpImSVonaaOkmZnl35C0Kj3WS9pRWLa3sGxhnXWamVkvqXcMol3SScBPgcWStgMvdbeCpAHAbOBSoB1YIWlhRKzt7BMRXyz0vw2YUNjE2xExvs76zMysl9UVEBHxmTR5j6QlwInA/znIahOBjRGxCUDSfGAKsPYA/a8D7q6nHjMzK19dp5gknS9pCEBEPAYsZd+/9nNGAZsL8+2pLbf9M4A24NFCc5OkqqTlkq4+wHozUp9qR0dHPT+KmZnVqd4xiO8AOwvzO1Nbb5kGPBgRewttZ6SBkz8H/l7SH3RdKSLmREQlIiojR47sxXLMzKzegFAUPu4UEb/n4KentgCjC/MtqS1nGjCv2BARW9K/m6jviMXMzHpRvQGxSdIXJA1Mj9uBTQdZZwUwRlKbpGOphcB+n0ZKH5ltpva9is62ZkmD0vQI4EIOPHZhZnbUOOGEExr2XPUGxOeBC6gdAbQDfwjM6G6FiNgD3AosAp4FFkTEGkmzJF1V6DoNmB/7fiFjLFCV9BSwBLi3+OknMzMrX72fYtpK7Y28RyLiYeDhLm1/02X+nsx6TwDn9vT5zMyONDNnzmT06NHccsstANxzzz0cc8wxLFmyhO3bt7N7926+/vWvM2XKlIbXVu+lNs6iNih9SkR8VNJ5wFUR8fVSqzMza6RfzIRXV/fuNk89F66494CLp06dyh133PFeQCxYsIBFixbxhS98gaFDh/Laa69x/vnnc9VVVzX81qj1nmK6H7gT2A0QEU/zAY4ozMxsXxMmTGDr1q28/PLLPPXUUzQ3N3Pqqafyla98hfPOO49LLrmELVu28Lvf/a7htdX7TerBEfFkl/Tq/n54ZmZHmm7+0i/T5z73OR588EFeffVVpk6dygMPPEBHRwcrV65k4MCBtLa2Zi/zXbZ6A+K19D2EAJD0WeCV0qoyMzuKTJ06lZtvvpnXXnuNxx57jAULFnDyySczcOBAlixZwksvdXtlo9LUGxC3AHOAsyVtAV4Ari+tKjOzo8i4ceN46623GDVqFKeddhrXX389V155Jeeeey6VSoWzzz67T+qq91NMm4BLJB1Pbdzi36iNQfRNrJmZ9TOrV78/OD5ixAiWLVuW7bdz585sexm6HaSWNFTSnZK+JelSasEwHdgIXNuIAs3MrG8c7Ajih8B2at9yvhm4CxDwmYhYVXJtZmbWhw4WEGdGxLkAkr5HbWD69M6bB5mZWf91sO9B7O6cSFdabXc4mFl/U8+tl490H+RnPNgRxMckvZmmBRyX5lV7vhja42c0MzuMNDU1sW3bNoYPH97wbyo3SkSwbds2mpqaerRetwEREQMOqSozs8NcS0sL7e3t9PebjjU1NdHS0tKjder9HoSZWb80cOBA2tra+rqMw1K912IyM7OjjAPCzMyyHBBmZpblgDAzsywHhJmZZTkgzMwsywFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaWVWpASJokaZ2kjZJmZpZ/Q9Kq9FgvaUdh2XRJG9Jjepl1mpnZ/kq7WJ+kAcBs4FKgHVghaWFErO3sExFfLPS/DZiQpocBdwMVIICVad3tZdVrZmb7KvMIYiKwMSI2RcS7wHxgSjf9rwPmpenLgcUR8XoKhcXApBJrNTOzLsoMiFHA5sJ8e2rbj6QzgDbg0Z6sK2mGpKqkan+/lruZWaMdLoPU04AH021N6xYRcyKiEhGVkSNHllSamdnRqcyA2AKMLsy3pLacabx/eqmn65qZWQnKDIgVwBhJbZKOpRYCC7t2knQ20AwsKzQvAi6T1CypGbgstZmZWYOU9immiNgj6VZqb+wDgLkRsUbSLKAaEZ1hMQ2YHxFRWPd1SV+jFjIAsyLi9bJqNTOz/anwvnxEq1QqUa1W+7oMM7MjiqSVEVHJLTtcBqnNzOww44AwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZlgPCzMyyHBBmZpblgDAzsywHhJmZZTkgzMwsywFhZmZZDggzM8tyQJiZWZYDwszMskoNCEmTJK2TtFHSzAP0uVbSWklrJP2o0L5X0qr0WFhmnWZmtr9jytqwpAHAbOBSoB1YIWlhRKwt9BkD3AlcGBHbJZ1c2MTbETG+rPrMzKx7ZR5BTAQ2RsSmiHgXmA9M6dLnZmB2RGwHiIitJdZjZmY9UGZAjAI2F+bbU1vRWcBZkn4labmkSYVlTZKqqf3q3BNImpH6VDs6Onq3ejOzo1xpp5h68PxjgIuBFuBxSedGxA7gjIjYIulM4FFJqyPi+eLKETEHmANQqVSisaWbmfVvZR5BbAFGF+ZbUltRO7AwInZHxAvAemqBQURsSf9uApYCE0qs1czMuigzIFYAYyS1SToWmAZ0/TTST6kdPSBpBLVTTpskNUsaVGi/EFiLmZk1TGmnmCJij6RbgUXAAGBuRKyRNAuoRsTCtOwySWuBvcCXI2KbpAuA70r6PbUQu7f46SczMyufIvrHqftKpRLVarWvyzAzO6JIWhkRldwyf5PazMyyHBBmZpblgDAzsywHhJmZZTkgzMwsywFhZmZZDggzM8tyQJiZWZYDwszMshwQZmaW5YAwM7MsB4SZmWU5IMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMzCzLAWFmZlkOCDMzy3JAmJlZVqkBIWmSpHWSNkqaeYA+10paK2mNpB8V2qdL2pAe08us08zM9ndMWRuWNACYDVwKtAMrJC2MiLWFPmOAO4ELI2K7pJNT+zDgbqACBLAyrbu9rHrNzGxfpQUEMBHYGBGbACTNB6YAawt9bgZmd77xR8TW1H45sDgiXk/rLgYmAfNKqfQXM+HV1aVs2sysdKeeC1fc2+ubLfMU0yhgc2G+PbUVnQWcJelXkpZLmtSDdZE0Q1JVUrWjo6MXSzczszKPIOp9/jHAxUAL8Likc+tdOSLmAHMAKpVKfOAqSkheM7MjXZlHEFuA0YX5ltRW1A4sjIjdEfECsJ5aYNSzrpmZlajMgFgBjJHUJulYYBqwsEufn1I7ekDSCGqnnDYBi4DLJDVLagYuS21mZtYgpZ1iiog9km6l9sY+AJgbEWskzQKqEbGQ94NgLbAX+HJEbAOQ9DVqIQMwq3PA2szMGkMRH/zU/eGkUqlEtVrt6zLMzI4oklZGRCW3zN+kNjOzLAeEmZllOSDMzCzLAWFmZln9ZpBaUgfw0iFsYgTwWi+V05tcV8+4rp5xXT3TH+s6IyJG5hb0m4A4VJKqBxrJ70uuq2dcV8+4rp452uryKSYzM8tyQJiZWZYD4n1z+rqAA3BdPeO6esZ19cxRVZfHIMzMLMtHEGZmluWAMDOzrH4fEJImSVonaaOkmZnlgyT9OC3/taTWwrI7U/s6SZc3uK4vSVor6WlJv5R0RmHZXkmr0qPrJdTLrutGSR2F5/+rwrLpkjakx/QG1/WNQk3rJe0oLCtzf82VtFXSMwdYLknfTHU/LenjhWVl7q+D1XV9qme1pCckfayw7MXUvkpSr14Bs466Lpb0RuH39TeFZd2+Bkqu68uFmp5Jr6lhaVmZ+2u0pCXpvWCNpNszfcp7jUVEv31Qu8z488CZwLHAU8A5Xfr8F+C+ND0N+HGaPif1HwS0pe0MaGBdfwIMTtP/ubOuNL+zD/fXjcC3MusOo3Yvj2FAc5publRdXfrfRu3y8qXur7TtPwY+DjxzgOWTgV8AAs4Hfl32/qqzrgs6nw+4orOuNP8iMKKP9tfFwD8f6mugt+vq0vdK4NEG7a/TgI+n6SHUbqrW9f9kaa+x/n4EMRHYGBGbIuJdYD4wpUufKcD30/SDwKckKbXPj4h3ona3u41pew2pKyKWRMS/pdnl1O6qV7Z69teBXA4sjojXI2I7sBiYdJB1yqrrOmBeLz13tyLicaC7e5VMAX4QNcuBkySdRrn766B1RcQT6Xmhca+vevbXgRzKa7O362rk6+uViPhNmn4LeBYY1aVbaa+x/h4Qo4DNhfl29t+57/WJiD3AG8DwOtcts66im6j9hdCpSVJV0nJJV/dSTT2p65p0KPugpM5bwx4W+yudimsDHi00l7W/6nGg2svcXz3V9fUVwP+VtFLSjD6o599LekrSLySNS22Hxf6SNJjam+xPCs0N2V+qnf6eAPy6y6LSXmOl3VHOeoekG4AK8B8KzWdExBZJZwKPSlodEc83qKSfA/Mi4h1J/4na0defNui56zENeDAi9hba+nJ/HdYk/Qm1gLio0HxR2l8nA4slPZf+wm6E31D7fe2UNJnabYnHNOi563El8KvY9w6Xpe8vSSdQC6U7IuLN3tx2d/r7EcQWYHRhviW1ZftIOgY4EdhW57pl1oWkS4C7gKsi4p3O9ojYkv7dBCyl9ldFQ+qKiG2FWr4HfKLedcusq2AaXQ7/S9xf9ThQ7WXur7pIOo/a73BKpFv9wj77ayvwEL13avWgIuLNiNiZph8GBqp2v/o+319Jd6+vUvaXpIHUwuGBiPinTJfyXmNlDKwcLg9qR0ibqJ1y6BzYGtelzy3sO0i9IE2PY99B6k303iB1PXVNoDYoN6ZLezMwKE2PADbQS4N1ddZ1WmH6M8DyeH9A7IVUX3OaHtaoulK/s6kNGKoR+6vwHK0ceND10+w7gPhk2furzrpOpzaudkGX9uOBIYXpJ4BJDazr1M7fH7U32t+mfVfXa6CsutLyE6mNUxzfqP2VfvYfAH/fTZ/SXmO9tnMP1we1Ef711N5s70pts6j9VQ7QBPxj+s/yJHBmYd270nrrgCsaXNcjwO+AVemxMLVfAKxO/0FWAzc1uK6/A9ak518CnF1Y9z+m/bgR+MtG1pXm7wHu7bJe2ftrHvAKsJvaOd6bgM8Dn0/LBcxOda8GKg3aXwer63vA9sLrq5raz0z76qn0e76rwXXdWnh9LacQYLnXQKPqSn1upPbBleJ6Ze+vi6iNcTxd+F1NbtRrzJfaMDOzrP4+BmFmZh+QA8LMzLIcEGZmluWAMDOzLAeEmZllOSDMDgPpKqb/3Nd1mBU5IMzMLMsBYdYDkm6Q9GS69v93JQ2QtDPdj2KNavfuGJn6jk8XCHxa0kOSmlP7v5P0SLog3W8k/UHa/AnpAojPSXogXVXYrM84IMzqJGksMBW4MCLGA3uB66ldYqEaEeOAx4C70yo/AP5bRJxH7Ruune0PALMj4mPUvun9SmqfANxB7V4kZwIXlv5DmXXDV3M1q9+nqF2ccEX64/44YCvwe+DHqc//Bv5J0onASRHxWGr/PvCPkoYAoyLiIYCI2AWQtvdkRLSn+VXUrg30L+X/WGZ5Dgiz+gn4fkTcuU+j9N+79Pug1695pzC9F///tD7mU0xm9fsl8Nl03X8kDUs3KPoQ8NnU58+Bf4mIN4Dtkv4otf8F8FjU7grW3nnjItXuiT64oT+FWZ38F4pZnSJiraSvUrt72IeoXfnzFuD/ARPTsq3UxikApgP3pQDYBPxlav8L4LuSZqVtfK6BP4ZZ3Xw1V7NDJGlnRJzQ13WY9TafYjIzsywfQZiZWZaPIMzMLMsBYWZmWQ4IMzPLckCYmVmWA8LMzLL+P8Bt1c8FhQAoAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "52HsGwKn5KC2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 375
        },
        "outputId": "8c046bcf-1126-4f6f-a02d-53af594b54ef"
      },
      "source": [
        "# Evaluate model\n",
        "test_eval = model.evaluate(x_test, y_test, verbose=1, return_dict=True)\n",
        "validation_eval = model.evaluate(x_validate, y_validate, verbose=1, return_dict=True)\n",
        "# model.save(\"model.h5\")"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 4/63 [>.............................] - ETA: 2:15 - loss: 1.2023 - accuracy: 0.6641 - f1_m: 0.6641 - precision_m: 0.6641 - recall_m: 0.6641"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-43-22bcf2539a46>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Evaluate model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtest_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mvalidation_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_validate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_validate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# model.save(\"model.h5\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict)\u001b[0m\n\u001b[1;32m   1387\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_r\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1388\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1389\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1390\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1391\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    860\u001b[0m       \u001b[0;31m# In this case we have not created variables on the first call. So we can\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    861\u001b[0m       \u001b[0;31m# run the first trace but we should fail if variables are created.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 862\u001b[0;31m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    863\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_variables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    864\u001b[0m         raise ValueError(\"Creating variables on a non-first call to a function\"\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2942\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2943\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2945\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1917\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1918\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1919\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1921\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    558\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 560\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    561\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    562\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "0tC4GrGl5KC2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "54b9dc1b-f828-416e-fb44-590cd4530832"
      },
      "source": [
        "for i, j in validation_eval.items():\n",
        "    print(f'{i}: {j}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss: 0.6498839855194092\n",
            "accuracy: 0.748129665851593\n",
            "f1_m: 0.7310635447502136\n",
            "precision_m: 0.860512375831604\n",
            "recall_m: 0.6394230723381042\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "BPNXGVMO5KC3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96a51b60-64fd-4e26-e93d-cf20f0407c4e"
      },
      "source": [
        "for i, j in test_eval.items():\n",
        "    print(f'{i}: {j}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss: 0.7060219049453735\n",
            "accuracy: 0.7413879036903381\n",
            "f1_m: 0.7320070266723633\n",
            "precision_m: 0.8558425903320312\n",
            "recall_m: 0.6424916386604309\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3DqyE7-W5KC3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3b6aa5d1-926d-491f-b823-58f1b584aefe"
      },
      "source": [
        "# Get predictions for plotting the confusion matrix\n",
        "predictions = model.predict(x_test, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "63/63 [==============================] - 12s 194ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "pFXzuzzh5KC3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "0312528e-d5e7-4043-8911-088bb8ce1caa"
      },
      "source": [
        "# Plot the confusion matrix\n",
        "predictions = np.argmax(predictions, axis=0)\n",
        "y_test = np.argmax(y_test, axis=0)\n",
        "\n",
        "sns.heatmap(confusion_matrix(y_test, predictions), annot=True, cmap = 'coolwarm', fmt='.0f')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD8CAYAAADUv3dIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAf1klEQVR4nO2de7xcVX32v885SbhfAyKQCIjRAmIRImD1RSoXA1KoFSuoVRRJ+76iWLAVP/gioFWwikWhYOQmeEFE2wZFwBuorUCiAiZcI9cgEAq88LaokJxf/5gdHI6Z2evM7DN7zT7P9/NZn7Nnz+xnPzOZ/M46a6/1bEUExhhjBsNI3QaMMWYq4aJrjDEDxEXXGGMGiIuuMcYMEBddY4wZIC66xhgzQFx0jTGmA5LOl7RC0pIOz0vSZyUtk3SzpF3LNF10jTGmMxcC87o8fwAwp2jzgbPLBF10jTGmAxHxI+CxLi85BLgoWlwHbCxpy26a06o0uCZe/WfXesmbmVQ+dOX8yrQ+MW9BZVpm4vzk8teoX41vT39Jcs05aOUdf02rh7qaBRExkS/B1sD9bY+XF/se7HTApBddY4zJlaLADvQ3rYuuMaZRaHrfneWJ8AAwu+3xrGJfR1x0jTGNYnSd0UGebiFwtKRLgD2AJyKi49ACZFB099h1E4456kWMjIhvffdBvnTZ/eUHDUArZ2/Wmhgv+8LHed6Be/P0ikf50cv/rGedqn1VrTcVtFIYmVZdT1fSV4G9gc0kLQc+AkwHiIhzgCuAA4FlwFPAO0v9VeauB0ZG4Ni/mcMHTvolb3vPIvbd63lsO3vd2rVy9matibP8i9/khoPe3fPxk+Ur188sV61UNF3JrYyIODwitoyI6RExKyLOi4hzioJLMWvhPRGxfUTsHBGLyzRLi66kP5L0wWIC8GeL7R2S3n0JO8zZkOUP/oZfP/xbVq4MvvejFbx6j5m1a+XszVoT57GfLOaZx57o+fjJ8pXrZ5arVioj05Tc6qBr0ZX0QeASQMANRRPwVUnH93vyzWfOYMV//u7Zx488+js2n7lW7Vo5e7NWffg7Vq9WKlX2dCeDsjHdI4GdIuKZ9p2STgeWAqeu6SBJ8ynmvm2/83E8f5v+xtGMMSaVunqwqZQV3TFgK+Decfu3LJ5bI+1z37otjnjk0ad53ma//623+cy1eOTR33V6eVeq1MrZm7Xqw9+xerVSGZ2R90LbMnfvB74v6TuSFhTtSuD7wDH9nvy2O59k9lbrsOUWazNtmth3r+fx7zc8WrtWzt6sVR/+jtWrlYpGlNzqoGtPNyKulPRiYHdaS9ugNfF3UUSs6vfkq8bg9HOWcfrJOzMyIr79vYe4+76natfK2Zu1Js4uF3+ama/ZnRmbbcJr776WO0/5HPdfcFntvnL9zHLVSkWjefd0Ndl3A3b2gplsnL3QHKrIXrhuj92Ta86e198w8O5u7YsjjDGmSuoaNkjFRdcY0yhyv5DmomuGniqHBDxUMfxoxEXXGGMGhocXjDFmgIyMuugaY8zAcE+3hJwj5HL1Zq36tKqMiaza21TQSiH3MV1HOw6ZN2vVpwXVxURW7W0qaKUyOn0kudVBrUU35wi5XL1Zqz4tqC4msmpvU0ErldyXAddadHOOkMvVm7Xq06qaXN9nrlqpaGQkudVBz2eVVHpbCmOMGTRN7ume3OkJSfMlLZa0+KF7L+8okHOEXK7erFWfVtXk+j5z1UplqIuupJs7tF8CW3Q6LiIWRMTciJjbLcA85wi5XL1Zqz6tqsn1feaqlcrItNHkVgdlU8a2AF4HPD5uv4D/6PfkOUfI5erNWvVpQXUxkVV7mwpaqeS+OKJrtKOk84ALIuIna3juKxHxlrITONrRDBPOXqiXKqId7zrioOSa88ILv5VXtGNEHNnludKCa4wxgyb3xRG1r0gzxpgq8TJgY4wZIC66xgwRuY7Deqw5nbpmJaTiomuMaRQe0zXGmEEiDy8YY8zA8JhuCTnndubqzVrN0KpSb6rk/KaQ+/CC83SHzJu1mqFVtd5UyPlNZaizFwAk/ZGkfSStP27/vH5PnnNuZ67erNUMrar1pkLObyq5Zy+UBd68D/g34L3AEkmHtD398X5PnnNuZ67erNUMrcnQq4qcP7MUhr2nexSwW0T8ObA38H8lHVM819FxarSjMcZUzshIequBsgtpIxHxXwARcY+kvYHLJG1Dl6IbEQuABdA98Cbn3M5cvVmrGVqToVcVOX9mKSjzKWNlpf5hSbusflAU4IOAzYCd+z15zrmduXqzVjO0JkOvKnL+zFKo8nY9kuZJul3SMknHr+H5F0j6oaRfFFnjB5ZplvV03w6sbN8RESuBt0v6fKnjEnLO7czVm7WaoVW13lTI+U1FFV0gkzQKnAXsBywHFklaGBG3tL3sw8ClEXG2pB2BK4Btu+p2y9OtAufpGtM/UyV7oYo83cc+9tfJNWfTD3++27WpVwInRcTriscfAoiIT7S95vPAXRFxWvH6T0fEn3Q7Z+2LI4wxpkqk9AtkkuYD7b/RFhTXpAC2BtpXciwH9hgncRJwtaT3AusB+5ad00XXGNMsJjAVrP2if48cDlwYEZ8ueroXS3ppRIx1OsBF15ghoMohgaYPVVS4DPgBYHbb41nFvnaOBOYBRMRPJa1Na6LBik6ieS9SNsaYCVLh4ohFwBxJ20maARwGLBz3mvuAfQAk7QCsDTzSTdQ9XWNMo9BoNbMXImKlpKOBq4BR4PyIWCrpFGBxRCwEjgO+IOlvgQCOiJLZCS66xphmUeFKs4i4gtY0sPZ9J7Zt3wK8aiKaLrrGmEaR+4q02otuzrmduXqzVjO0cvWWczZvEs7T7XLyjHM7c/VmrWZo5ewt12zeVIY9ZQxJu0t6RbG9o6RjU9YXp5Bzbmeu3qzVDK2cveWazZuMRtJbDZTl6X4E+CxwtqRPAGfSWnVxvKQT+j15zrmduXqzVjO0cvdWFbXk6Y6OJrc6KBvTPRTYBVgLeAiYFRFPSvoUcD3wD2s6qH1p3fY7H8fzt+l/XMgYY5LI/MaUZf3rlRGxKiKeAn4VEU8CRMRvgI7L3CJiQUTMjYi53QpuzrmduXqzVjO0cvdWFbXk6VYY7TgZlJ31aUmrR713W71T0kZ0Kbqp5Jzbmas3azVDK3dvVVGLLym91UDZ8MJeEfE7gHEBDtOBd/R78pxzO3P1Zq1maOXsLdds3mQynzLmPF1jphg5B95Ukaf7m4s/llxz1vmrDw+8u1v74ghjjKmUmqaCpeKia4xpFpnPXnDRNbWQ85+4Tafpn9dE7hxRBy66xphm4Z6uMcYMEPd0jTFmgNS0vDeV2otujtF2uXubClpVxgvm+h5z9parVhKZ93Qd7Thk3qaCFlQXL5jze8zVW65a6SdVequBCRddSRdVdfJco+1y9jYVtKC6eMGc32Ou3nLVSmbIox0XjmuXA3+x+nG/J8852i5Xb1NBq0pyfo+5estVK5khz16YBdwCnEvrTpcC5gKf7naQox2NMbWRefZCmbu5wM+AE4AnIuIa4DcRcW1EXNvpIEc7WquJ0YL+jtWrlczIaHqrga5FNyLGIuIzwDuBEySdSYUzHnKOtsvV21TQqpKc32Ou3nLVSmZkJL3VQFIBjYjlwJskvR54sqqT5xptl7O3qaAF1cUL5vwec/WWq1Yymd+C3dGOphacvWDWRBXRjr+9+oLkmrP2/u90tKMxxvRF5j1dF11jTKMILwM2xpgBkvkyYBddUwseh20GVY7Nt7i9fwkXXWOMGRzhMV1jjBkg7ukaY8wAcU+3OznndubqzVrN0MrZW1VaVeYip5L77AXn6Q6ZN2s1Qytnb1VqVZWLPCGGOdpxPJJeLelYSftXcfKccztz9WatZmjl7K1KrapykSdCaCS5lSFpnqTbJS2TdHyH1/ylpFskLZX0lTLNsjzdG9q2jwLOBDYAPtLJwETIObczV2/WaoZWzt5yzUZOpqI8XUmjwFnAAcCOwOGSdhz3mjnAh4BXRcROwPvL7JWV+ult2/OB/SLiZGB/4K1dzM6XtFjS4ofuvbzMgzHGVEaFPd3dgWURcVdEPA1cAhwy7jVHAWdFxOMAEbGiTLTsrCOSNpE0k1Y4ziOF8H8DKzsd5Dxda1lreL3lmo2czATydNs7iEVrX+2xNdB+BXF5sa+dFwMvlvTvkq6TNK/UXsnzG9EKMV8MbCppSwBJ69O6i0Rf5Jzbmas3azVDK2dvuWYjpxJSemvrIBZtokslpwFzgL2Bw4EvSNq47IDO5iO27fDUGPCGCZr7A3LO7czVm7WaoZWztyq1qspFnhDVzUp4AJjd9nhWsa+d5cD1EfEMcLekO2gV4UUd7TlP1xjTK1VnL7z+mdv7/gv6iZ9/L7nmbLTrvh3PJ2kacAewD61iuwh4S0QsbXvNPODwiHiHpM2AXwC7RETHPw1qXxxhjDFVkjIVLEknYqWko4GrgFHg/IhYKukUYHFELCye21/SLcAq4O+6FVxw0TXGNI0KFz1ExBXAFeP2ndi2HcCxRUvCRdeYSWIq3JKoal+vr0BjrKa7/KbiomuMaRYOvDHGmMFR1ZjuZOGia4xpFNH/EoJJpfaim2McXe7erNUMrapjD3N9n1X/vywj956uox2HzJu1mqEF1cYe5vo+q/7Mkqgo8GayqLXo5hpHl7M3azVDC6qNPcz1fVb9maUwptHkVgdl0Y57SNqw2F5H0smSLpd0mqSN+j15znF0uXqzVjO0qibX91nHZ1Zlnu5kUHbW84HVi67PoBWAc1qx74JOBzna0RhTF4GSWx2UXUgbiYjVEY5zI2LXYvsnkm7sdFCR1LMAumcv5BxHl6s3azVDq2pyfZ91fGbDfiFtiaR3Fts3SZoLIOnFwDP9njznOLpcvVmrGVpVk+v7rOMzm0i0Yx2U9XTfDZwh6cPAfwI/lXQ/rWDfvi+75hpHl7M3azVDC6qNPcz1fVb9maVQ1wWyVJKiHYuLadvRKtLLI+Lh1BM42tFMVaZC9kLV/OTy1/Td/bzvzluTa84L5uww8O5u0uKIiHgSuGmSvRhjTN/kPqZb+4o0Y4ypEi8DNsaYAeKerjFTlKkyDpsb7ukaY8wAGas33aAUF11jTKMIF11jjBkcHl4oIefczly9WasZWjl7y1UrhdyLrvN0h8ybtZqhlbO3XLVSyT3wpiza8X2SZk/WyXPO7czVm7WaoZWzt1y1Uhnqogt8FLhe0o8l/R9Jm1d58pxzO3P1Zq1maOXsLVetVMZiJLnVQdlZ7wJm0Sq+uwG3SLpS0jskbdDpIOfpGmPqYth7uhERYxFxdUQcCWwF/DMwj1ZB7nTQgoiYGxFzn79N5xvu5Zzbmas3azVDK2dvuWqlMuxF9zmuIuKZiFgYEYcD2/R78pxzO3P1Zq1maOXsLVetVCKU3OqgbMrYmzs9ERF9h2LmnNuZqzdrNUMrZ2+5aqUylvmUsaQ83X5wnq4xJpUq8nR/fsejyTVn1xfPzDNP1xhjhoXcF0e46BpjGkVdY7WpuOgaYxqFe7rGGDNA3NM1xpgBMla3gRJcdI0xjaKu5b2p1F50c46Qy9WbtZqhlbO3XLVSqHJ4QdI84AxgFDg3Ik7t8Lo3ApcBr4iIxd00He04ZN6s1QytnL3lqpVKVcuAJY0CZwEHADsCh0vacQ2v2wA4Brg+xV9ZtOMMSW+XtG/x+C2SzpT0HknTU07QjZwj5HL1Zq1maOXsLVetVMYivZWwO7AsIu6KiKeBS4BD1vC6jwKnAb9N8VfW070AeD1wjKSLgTfRquavAM5NOUE3co6Qy9WbtZqhlbO3XLVSmUhPtz0RsWjz26S2BtrHQpYX+55F0q7A7Ij4dqq/sjHdnSPiZZKmAQ8AW0XEKklfAm7qdFBhfD7A9jsfR7ekMWOMqZKJjOlGxAJgQS/nkTQCnA4cMZHjynq6I5JmABsA6wIbFfvXAjoOLzja0VrWGl5vuWqlsiqU3Ep4AGi/c86sYt9qNgBeClwj6R5gT2ChpLndRMuK7nnAbcCNwAnA1yV9AVhEa3yjL3KOkMvVm7WaoZWzt1y1Uqkw2nERMEfSdkXn8zBg4e/PE09ExGYRsW1EbAtcBxxcNnuh6/BCRHxG0teK7V9LugjYF/hCRNxQ5riMnCPkcvVmrWZo5ewtV61UqgpOjIiVko4GrqI1Zez8iFgq6RRgcUQs7K6wZhztaIzJhiqiHa++6enkmrP/H89wtKMxxvRDwlSwWnHRNcY0irExB94YY8zAyP12PS66xphGMcmXqfrGRdcY0yicp2uMMQPEF9KMMWaAeHihhJxzO3P1Zq1maOXsLVetFBKW99aK83SHzJu1mqGVs7dctVKJSG91UFp0Jb1Q0gcknSHpdEl/I2nDKk6ec25nrt6s1QytnL3lqpXKUBddSe8DzgHWppWhuxat1J3rJO3d78lzzu3M1Zu1mqGVs7dctVIZCyW3Oijr6R4FHBARH6MVdLNTRJwAzAM+0+mg9mDgh+69vDq3xhhTwlD3dAtWX2xbC1gfICLuw3m61rKWv2MD1Epl1Vh6q4OyonsusKjI0P0prZu0IWlz4LF+T55zbmeu3qzVDK2cveWqlUqFebqTQmm0o6SdgB2AJRFx20RPUBbtuOdum3LMUds/m7V50aX3TfQUk6KVszdrNUMrZ291aVUR7XjRtSQPHLz9NYMPanCerjEmG6oouhdek150j9h78EW39sURxhhTJV6RZowxA8RF1xhjBkhdsxJScdE1xjSKMRddY4wZHB5eMMaYAeKiW0LOEXK5erNWM7Ry9parVgq5h5g72nHIvFmrGVo5e8tVK5WISG51UGvRzTlCLldv1mqGVs7ectVKZdWq9FYHtRbdnCPkcvVmrWZo5ewtV61UmpAyNmEc7WiMqYuxSG91UBZivpGkUyXdJukxSY9KurXYt3Gn4xztaC1rDa+3XLVSGfae7qXA48DeEbFpRMwE/rTYd2m/J885Qi5Xb9ZqhlbO3nLVSiXGIrnVQdmUsW0j4rT2HRHxEHCapHf1e/JVY3D6Ocs4/eSdn419u/u+p2rXytmbtZqhlbO3XLUmcs6c6RrtKOlq4HvAFyPi4WLfFsARwH4RsW/ZCRztaIxJpYpox09cuiq55nzoL0cHHu1YNrzwZmAmcG0xpvsYcA2wKfCmSfZmjDETJvcx3a7DCxHxOPDBoj0HSe8ELpgkX8YY0xO5LwPuZ8rYyZW5MMaYihiLSG510LWnK+nmTk8BW1Rvxxhj+iMqvJAmaR5wBjAKnBsRp457/ljg3cBK4BHgXRFxbzfNstkLWwCvozVF7DnnAv4j3boxxgyGVenX0boiaZTWHdD3A5bTujP6woi4pe1lvwDmRsRTkv438Ela18I6UlZ0vwWsHxE3rsHQNRPwb4wxA6HCIJvdgWURcReApEuAQ4Bni25E/LDt9dcBbysTLbuQdmSX595SJm6MMYNmImseJM0H5rftWhARC4rtrYH2HMrlwB5d5I4EvlN2TufpDqE3azVDK2dvuWqlMJGVZkWBXVD6whIkvQ2YC7ym7LXO0x0yb9ZqhlbO3nLVSqXCeboPALPbHs8q9j0HSfsCJwAHR0RpsITzdIfMm7WaoZWzt1y1Uhkbi+RWwiJgjqTtJM0ADgMWtr9A0suBz9MquCtS/DlPd8i8WasZWjl7y1UrlbFVkdy6ERErgaOBq4BbgUsjYqmkUyQdXLzsH4H1ga9LulHSwg5yzzIpY7rtg9Pb73wc3eIdjTGmSqpc9BARVwBXjNt3Ytt2af7MeHru6UrqeJXOebrWstbwestVK5WhvkeapF07tN2AXfo9ec65nbl6s1YztHL2lqtWKhWO6U4KZcMLi4Braa1AG0/HO0ekknNuZ67erNUMrZy95aqVSu6BN2V5ukuAN0TEnWt47v6ImL2Gw56D83SNMalUkaf73n96MrnmfO79Gw48T7esp3sSnYcg3lutFWOM6Z+6bsOTStky4Mu6PL1JxV6MMaZvci+6ztM1xjSK3G/B7jxdY0yjyL2n6zxdY0yjqGv+bSrO0zXGNIpVmd+DvfY83Zwj5HL1Zq1maOXsLVetFHIfXnC045B5s1YztHL2lqtWKjEWya0OHO04ZN6s1QytnL3lqpVK7ncDdrTjkHmzVjO0cvaWq1Yqufd0a79djzHGVEnusxfKUsY2lPQJSRdLesu45/65y3HzJS2WtPihey/vqJ9zhFyu3qzVDK2cveWqlcqqlWPJrQ7KhhcuoDUn9xvAYZK+IWn1J7hnp4NS83RzjpDL1Zu1mqGVs7dctVLJPU+3bHhh+4h4Y7H9r5JOAH7QdquKvsg5Qi5Xb9ZqhlbO3nLVSiXG8p6nWxbteCuwU0SMte07Avg7Wosmtik7gaMdjTGpVBHt+OYP3Jtcc772qW0GHu1YNrxwOfDa9h0RcSFwHPD0JHkyxpieGerhhYj4+w77r5T08cmxZIwxvTNW0wWyVBztaIxpFGMxltzqwNGOxphGkXv2gqMdjTGNYtiLrqMdjTFDRe4r0mqPdjTGmCoZy3yebu3ZCznndubqzVrN0MrZW65aKYytWjWp+v3iPN0h82atZmjl7C1XrVRyTxlznu6QebNWM7Ry9parViouul3IObczV2/WaoZWzt5y1Uol93m6ZdGOz5d0tqSzJM2UdJKkX0q6VNKWXY5LinY0xpiqGfae7oXALcD9wA+B3wAHAj8Gzul0UGq0Y865nbl6s1YztHL2lqtWKjE2ltzqoKzobhERn4uIU4GNI+K0iLg/Ij4HlCaMlZFzbmeu3qzVDK2cveWqlcrYqlXJrQ7Kpoy1F+WLxj032u/Jc87tzNWbtZqhlbO3XLVSGct8RVpZnu4pwCcj4r/G7X8RcGpEHFp2AufpGmNSqSJPd+9Df5pcc6657JV55elGxInjC26xfxnw7UlzZYwxPVLlhTRJ8yTdLmmZpOPX8Pxakr5WPH+9pG3LNB3taIxpFBFjya0bkkaBs4ADgB2BwyXtOO5lRwKPR8SLgM8Ap5X5c7SjMaZRjK2s7ALZ7sCyiLgLQNIlwCG0ZnSt5hDgpGL7MuBMSYpu47Ylt7J4GNiF1kyF9rYt8OuJ3BYj4bYZ8601/Fo5e7NWM7SqbMB8YHFbm9/23KHAuW2P/wo4c9zxS4BZbY9/BWzW7Zxlwwurox3vHdfuAa4pOXaizLdWI7Sq1rOWtSaNaFtTULQFk31ORzsaY8yaeQCY3fZ4VrFvTa9ZLmkasBHQdSJyrdkLxhiTMYuAOZK2kzQDOAxYOO41C4F3FNuHAj+IYpyhE7Xn6bZRZbfeWvVpVa1nLWvVQkSslHQ0cBWtxWDnR8TSYv3C4ohYCJwHXCxpGfAYrcLcla6LI4wxxlSLhxeMMWaAuOgaY8wAqb3oli2zm6DW+ZJWSFpSga/Zkn4o6RZJSyUd04fW2pJukHRTodX3aj5Jo5J+IelbfercU2Qk3yhpcZ9aG0u6TNJtkm6V9MoedV5S+FndnpT0/j58/W3xuS+R9FVJa/ehdUyhs7QXT2v6jkraVNJ3Jd1Z/NykD603Fd7GJM3t09c/Fv+WN0v6F0kb96H10ULnRklXS9oq1VvjqHli8iitycQvBGYANwE79qG3F7ArsKQCb1sCuxbbGwB39OqN1gq+9Yvt6cD1wJ59+jsW+ArwrT517qFkMvcEtL4IvLvYnkErDrSK78hDwDY9Hr81cDewTvH4UuCIHrVeSmsy/Lq0LkJ/D3jRBDX+4DsKfBI4vtg+HjitD60dgJfQmkc/t09f+wPTiu3T+vS1Ydv2+4BzqvjODWOru6f77DK7iHgaWL3Mrici4ke0riD2TUQ8GBE/L7b/P3Arrf/AvWhF/D44aHrRer6CKWkW8Hrg3F41qkbSRrT+s50HEBFPR8T/q0B6H+BXEXFvHxrTgHWKeZTrAr/uUWcH4PqIeCoiVgLXAn8xEYEO39FDaP3Covj5571qRcStEXH7RDx10bq6eJ8A19Gap9qr1pNtD9ejj+//sFN30d2a1l0pVrOcHgvbZFIkB72cVg+1V41RSTcCK4DvRkTPWsA/AX8PVBF9H8DVkn4mqZ9VQ9sBjwAXFMMe50parwJ/hwFf7fXgiHgA+BRwH/Ag8EREXN2j3BLgfxW3rlqX1l1UZpcck8IWEfFgsf0QeeaavAv4Tj8Ckv5B0v3AW4ETK3E1hNRddLNH0vrAN4D3j/ttPSEiYlVE7EKrt7C7pJf26OcgYEVE/KxXL+N4dUTsSitJ6T2S9upRZxqtPynPjoiXA/9N60/lnikmpB8MfL0PjU1o9SS3A7YC1pP0tl60IuJWWn9mXw1cCdwIVHr7gWj9/Z1VL1DSCcBK4Mv96ETECRExu9A5ugpvw0jdRTdlmV1tSJpOq+B+OSK+WYVm8Sf3D4F5PUq8CjhY0j20hmNeK+lLffh5oPi5AvgXWkM+vbAcWN7Wg7+MVhHuhwOAn0fEw31o7AvcHRGPRMQzwDeBP+lVLCLOi4jdImIv4HFaY/398rCKG70WP1dUoFkJko4ADgLeWvxCqIIvA2+sSGvoqLvopiyzqwVJojU+eWtEnN6n1uarr/xKWgfYD7itF62I+FBEzIqIbWl9Xj+IiJ56bpLWk7TB6m1aF056mvkREQ8B90t6SbFrH54bgdcLh9PH0ELBfcCektYt/k33oTU+3xOSnlf8fAGt8dyv9OkPnruU9B3Av1Wg2TeS5tEaxjo4Ivq6x46kOW0PD6HH738jqPtKHq1xsTtozWI4oU+tr9Iat3uGVs/ryD60Xk3rz7ybaf0ZeSNwYI9aLwN+UWgtAU6s6LPbmz5mL9CaNXJT0ZZW8PnvQise72bgX4FN+tBaj1ZwyEYVfE4n0/pPvgS4GFirD60f0/plchOwTw/H/8F3FJgJfB+4k9aMiE370HpDsf07WtGsV/WhtYzWNZfV3/+kGQcdtL5RfP43A5cDW/f77zqszcuAjTFmgNQ9vGCMMVMKF11jjBkgLrrGGDNAXHSNMWaAuOgaY8wAcdE1xpgB4qJrjDED5H8ALAiqoVtnjiMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-6ys5M0w8SSm"
      },
      "source": [
        "# Model Parameters\n",
        "input_shape = (75, 100, 3)\n",
        "\n",
        "# Number of classes\n",
        "num_classes = 7\n",
        "\n",
        "# Optimizer\n",
        "optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n",
        "\n",
        "# Epoch is number of times a node is trained\n",
        "epochs = 9\n",
        "batch_size = 20\n",
        "\n",
        "# Callback functions (To follow the path of gradient descent)\n",
        "learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=5, verbose=0, factor=0.5, min_lr=0.00001)\n",
        "early_stopping_monitor = EarlyStopping(patience=20, monitor='val_accuracy')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B3COod77Db5c",
        "outputId": "e99a24fe-b6a1-428d-9c8c-64e007fa7e2d"
      },
      "source": [
        "# dataaugment.flow(x_train,y_train, batch_size=batch_size),\n",
        "# callbacks=[learning_rate_reduction,early_stopping_monitor]\n",
        "history = model.fit(\n",
        "    x = x_train,\n",
        "    y = y_train, \n",
        "    batch_size=batch_size,\n",
        "    verbose = 0, \n",
        "    steps_per_epoch=x_train.shape[0] // batch_size,\n",
        "    epochs=10,\n",
        "    validation_data = (x_validate, y_validate)\n",
        ")\n",
        "\n",
        "loss, accuracy = model.evaluate(x_test, y_test, verbose=0)\n",
        "predictions = model.predict(x_test)\n",
        "loss_v, accuracy_v = model.evaluate(x_validate, y_validate, verbose=0)\n",
        "loss_t, accuracy_t = model.evaluate(x_train, y_train, verbose=0)\n",
        "\n",
        "print(\"Training: accuracy = %f\" % (accuracy_t))\n",
        "print(\"Validation: accuracy = %f\" % (accuracy_v))\n",
        "print(\"Test: accuracy = %f\" % (accuracy))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7l6D3elvzeOc",
        "outputId": "aa5f62ba-4636-4e80-c7ac-f7d814fbbb25"
      },
      "source": [
        "predictions = np.argmax(predictions, axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NVVDXdOHxYcY"
      },
      "source": [
        "y_test = np.argmax(y_test, axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IAGYMa0HxUsy",
        "outputId": "f3c2f649-b11b-4c68-efdc-1b9c627d14c6"
      },
      "source": [
        "\n",
        "sns.heatmap(confusion_matrix(y_test, predictions), annot=True, cmap = 'coolwarm', fmt='.0f')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jL1zPJ27xXmE",
        "outputId": "18204e51-7a3b-4c0f-df3f-bd99c34f6d58"
      },
      "source": [
        "len(predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qj27aH_-z1f4",
        "outputId": "db34107b-edb9-436e-aa12-b978a68a1074"
      },
      "source": [
        "target.value_counts()\n",
        "len(x_train)\n",
        "len(x_validate)"
      ]
    }
  ]
}