siamese_network_prototype.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n",
    "# https://www.cs.cmu.edu/~rsalakhu/papers/oneshot1.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from fastai.vision import *\n",
    "from fastai.metrics import accuracy_thresh\n",
    "from fastai.basic_data import *\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch import nn\n",
    "from fastai.callbacks.hooks import num_features_model, model_sizes\n",
    "from fastai.layers import BCEWithLogitsFlat\n",
    "from fastai.basic_train import Learner\n",
    "from skimage.util import montage\n",
    "import pandas as pd\n",
    "from torch import optim\n",
    "import re\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import fastai\n",
    "# from fastprogress import force_console_behavior\n",
    "# import fastprogress\n",
    "# fastprogress.fastprogress.NO_BAR = True\n",
    "# master_bar, progress_bar = force_console_behavior()\n",
    "# fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Posing the problem as a classification task is probably not ideal. We are asking our NN to learn to recognize a whale out of 5004 possible candidates based on what it has learned about the whales. That is a tall order.\n",
    "\n",
    "Instead, here we will try to pose the problem as a verification task. When presented with two images of whale flukes, we will ask the network - are the images of the same whale or of different whales? In particular, we will try to teach our network to learn features that can be useful in determining the similarity between whale images (hence the name of this approach - feature learning).\n",
    "\n",
    "This seems like a much easier task, at least in theory. Either way, no need to start with a relatively big CNN like resnet50. Let's see what mileage we can get out of resnet18."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new architecture calls for a new validation set, this time our validation set will consist of all whales that have exactly two images\n",
    "df = pd.read_csv('data/train.csv')\n",
    "im_count = df[df.Id != 'new_whale'].Id.value_counts()\n",
    "im_count.name = 'sighting_count'\n",
    "df = df.join(im_count, on='Id')\n",
    "val_fns = set(df[df.sighting_count == 2].Image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2570"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_fns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn2label = {row[1].Image: row[1].Id for row in df.iterrows()}\n",
    "path2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)\n",
    "\n",
    "name = f'res18-siamese'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "SZ = 224\n",
    "BS = 64\n",
    "NUM_WORKERS = 12\n",
    "SEED=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_block api creates categories based on classes it sees in the train set and\n",
    "# our val set contains whales whose ids do not appear in the train set\n",
    "classes = df.Id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (\n",
    "    ImageItemList\n",
    "        .from_df(df[df.Id != 'new_whale'], f'data/train-{SZ}', cols=['Image'])\n",
    "        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n",
    "        .label_from_func(lambda path: fn2label[path2fn(path)], classes=classes)\n",
    "        .add_test(ImageItemList.from_folder(f'data/test-{SZ}'))\n",
    "        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n",
    "#         .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n",
    "#         .normalize(imagenet_stats)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I am still using the ImageItemList even though I will create my own datasets. Why? Because I want to reuse the functionality that is already there (creating datasets from files, augmentations, resizing, etc).\n",
    "\n",
    "I realize the code is neither clean nor elegant but for the time being I am happy with this approach."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_even(num): return num % 2 == 0\n",
    "\n",
    "class TwoImDataset(Dataset):\n",
    "    def __init__(self, ds):\n",
    "        self.ds = ds\n",
    "        self.whale_ids = ds.y.items\n",
    "    def __len__(self):\n",
    "        return 2 * len(self.ds)\n",
    "    def __getitem__(self, idx):\n",
    "        if is_even(idx):\n",
    "            return self.sample_same(idx // 2)\n",
    "        else: return self.sample_different((idx-1) // 2)\n",
    "    def sample_same(self, idx):\n",
    "        whale_id = self.whale_ids[idx]        \n",
    "        candidates = list(np.where(self.whale_ids == whale_id)[0])\n",
    "        candidates.remove(idx) # dropping our current whale - we don't want to compare against an identical image!\n",
    "        \n",
    "        if len(candidates) == 0: # oops, there is only a single whale with this id in the dataset\n",
    "            return self.sample_different(idx)\n",
    "        \n",
    "        np.random.shuffle(candidates)\n",
    "        return self.construct_example(self.ds[idx][0], self.ds[candidates[0]][0], 1)\n",
    "    def sample_different(self, idx):\n",
    "        whale_id = self.whale_ids[idx]\n",
    "        candidates = list(np.where(self.whale_ids != whale_id)[0])\n",
    "        np.random.shuffle(candidates)\n",
    "        return self.construct_example(self.ds[idx][0], self.ds[candidates[0]][0], 0)\n",
    "    \n",
    "    def construct_example(self, im_A, im_B, class_idx):\n",
    "        return [im_A, im_B], class_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(\n",
    "    TwoImDataset(data.train),\n",
    "    batch_size=BS,\n",
    "    shuffle=True,\n",
    "    num_workers=NUM_WORKERS\n",
    ")\n",
    "valid_dl = DataLoader(\n",
    "    TwoImDataset(data.valid),\n",
    "    batch_size=BS,\n",
    "    shuffle=False,\n",
    "    num_workers=NUM_WORKERS\n",
    ")\n",
    "\n",
    "data_bunch = ImageDataBunch(train_dl, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_batch(batch):\n",
    "    stat_tensors = [torch.tensor(l).cuda() for l in imagenet_stats]\n",
    "    return [normalize(batch[0][0], *stat_tensors), normalize(batch[0][1], *stat_tensors)], batch[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_bunch.add_tfm(normalize_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functional import seq\n",
    "\n",
    "class SiameseNetwork(nn.Module):\n",
    "    def __init__(self, arch=models.resnet18):\n",
    "        super().__init__() \n",
    "        self.cnn = create_body(arch)\n",
    "        self.head = nn.Linear(num_features_model(self.cnn), 1)\n",
    "        \n",
    "    def forward(self, im_A, im_B):\n",
    "        # dl - distance layer\n",
    "        x1, x2 = seq(im_A, im_B).map(self.cnn).map(self.process_features)\n",
    "        dl = self.calculate_distance(x1, x2)\n",
    "        out = self.head(dl)\n",
    "        return out\n",
    "    \n",
    "    def process_features(self, x): return x.reshape(*x.shape[:2], -1).max(-1)[0]\n",
    "    def calculate_distance(self, x1, x2): return (x1 - x2).abs_()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below I include two slightly different siamese networks. I leave the code commented out and choose to use the one above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from functional import seq\n",
    "\n",
    "# def cnn_activations_count(model):\n",
    "#     _, ch, h, w = model_sizes(create_body(models.resnet18), (SZ, SZ))[-1]\n",
    "#     return ch * h * w\n",
    "\n",
    "# class SiameseNetwork(nn.Module):\n",
    "#     def __init__(self, lin_ftrs=2048, arch=models.resnet18):\n",
    "#         super().__init__() \n",
    "#         self.cnn = create_body(arch)\n",
    "#         self.fc1 = nn.Linear(cnn_activations_count(self.cnn), lin_ftrs)\n",
    "#         self.fc2 = nn.Linear(lin_ftrs, 1)\n",
    "        \n",
    "#     def forward(self, im_A, im_B):\n",
    "#         x1, x2 = seq(im_A, im_B).map(self.cnn).map(self.process_features).map(self.fc1)\n",
    "#         dl = self.calculate_distance(x1.sigmoid(), x2.sigmoid())\n",
    "#         out = self.fc2(dl)\n",
    "#         return out\n",
    "    \n",
    "#     def calculate_distance(self, x1, x2): return (x1 - x2).abs_()\n",
    "#     def process_features(self, x): return x.reshape(x.shape[0], -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from functional import seq\n",
    "\n",
    "# def cnn_activations_count(model):\n",
    "#     _, ch, h, w = model_sizes(create_body(models.resnet18), (SZ, SZ))[-1]\n",
    "#     return ch * h * w\n",
    "\n",
    "# class SiameseNetwork(nn.Module):\n",
    "#     def __init__(self, lin_ftrs=2048, pool_to=3, arch=models.resnet18, pooling_layer=nn.AdaptiveMaxPool2d):\n",
    "#         super().__init__() \n",
    "#         self.cnn = create_body(arch)\n",
    "#         self.pool = pooling_layer(pool_to)\n",
    "#         self.fc1 = nn.Linear(num_features_model(self.cnn) * pool_to**2, lin_ftrs)\n",
    "#         self.fc2 = nn.Linear(lin_ftrs, 1)\n",
    "        \n",
    "#     def forward(self, im_A, im_B):\n",
    "#         x1, x2 = seq(im_A, im_B).map(self.cnn).map(self.pool).map(self.process_features).map(self.fc1)\n",
    "#         dl = self.calculate_distance(x1.sigmoid(), x2.sigmoid())\n",
    "#         out = self.fc2(dl)\n",
    "#         return out\n",
    "    \n",
    "#     def calculate_distance(self, x1, x2): return (x1 - x2).abs_()\n",
    "#     def process_features(self, x): return x.reshape(x.shape[0], -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data_bunch, SiameseNetwork(), loss_func=BCEWithLogitsFlat(), metrics=[lambda preds, targs: accuracy_thresh(preds.squeeze(), targs, sigmoid=False)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.split([learn.model.cnn[:6], learn.model.cnn[6:], learn.model.head])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze_to(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl8XXWd//HXJ2uzNUuTtGm6pBstpUChAUQRWhBFmJ8siorLgMzIjOuoD2ec+fkbdfTnuDsuqPxQAR1HGBFQYUSQtazStHSlLV3Tplv2Pc12P78/7m1IQ5KmTU7OTe77+XjcB/fe8733vM+l937yPd9zvsfcHREREYCksAOIiEj8UFEQEZE+KgoiItJHRUFERPqoKIiISB8VBRER6aOiICIifVQURESkj4qCiIj0SQk7wMkqLCz0srKysGOIiEwoa9eurXX3ohO1m3BFoaysjIqKirBjiIhMKGZWOZJ22n0kIiJ9VBRERKSPioKIiPRRURARkT4qCiIi0kdFQURE+qgoiIhIHxUFEZE49NzOWtbvbxz39aooiIjEof/9wCY++LO/sKe2bVzXq6IgIhJnunsjVDV00NLZw0d+tZb2rp5xW7eKgohInDnQ0EFvxLl6+Uy2H2nhX+7fhLuPy7oDKwpmdoeZVZvZ5mHarDSz9Wa2xcyeDiqLiMhEUlnfDsD7L5jLZ95yGr9ff5BfvjCiqYtGLcgJ8e4CbgV+OdhCM8sDfgxc4e77zKw4wCwiIhNGZV10HKFsWiblc/NZv7+Rrzz0CstKp7JibkGg6w6sp+Duq4H6YZq8D7jf3ffF2lcHlUVEZCKprGsnIzWZopx0kpKM775nOaX5GTyzozbwdYc5dfZpQKqZPQXkAN9396F6FbcAtwDMmTNn3AKKiIShsq6dOQWZmBkAuRmpPPiJi5g6JTXwdYc50JwCrACuAt4G/KuZnTZYQ3e/3d3L3b28qOiE14gQEZnQKuvamDMt87jnxqMgQLhFoQr4k7u3uXstsBo4O8Q8IiKhi0ScffXtlA0oCuMlzKLwe+DNZpZiZpnABcDWEPOIiISuuqWTzp4Ic6ZlhbL+wMYUzOxuYCVQaGZVwBeBVAB3v83dt5rZn4CNQAT4mbsPefiqiEgiOHbk0dyCcHoKgRUFd79hBG2+BXwrqAwiIhNNZV30HIWykHoKOqNZRCSOVNa3kZJkzMybEsr6VRREROJIZV07pfkZpCSH8/OsoiAiEkeOnaMQFhUFEZE4UlnXFtp4AqgoiIjEjcb2LpqP9jA3pHMUQEVBRCRu7I0deaTdRyIi8to5Ctp9JCIi+9RTEBGRYyrr25k+NZ2MtOTQMqgoiIjEicq6NuYWhLfrCFQURETiRmVde6hHHoGKgohIXGjv6qG6pVNFQUQkEbV29vDKwea+x/vqY4PMIR55BCoKIiKh+N6fX+XKHzzD5x/YREdXb9/sqGFNmX1MmNdoFhFJWM/vqiMvM5X/+ss+XtpTz7lz8oHwpsw+Rj0FEZFx1ny0m22Hm7nxwjJ+efP5NLR3898V+8nNSCU3c3yuxTwU9RRERMbZusoGIg7nlRVw0aJC/vSpN/OF328mLzMt7GgqCiIi461ibwPJScY5c/IAKMxO58fvXxFyqqjAdh+Z2R1mVm1mg1532cxWmlmTma2P3b4QVBYRkXjy0t56zpg5laz0+Pu7PMgxhbuAK07Q5hl3Xx67fTnALCIicaGzp5cN+xs5r6wg7CiDCqwouPtqoD6o9xcRmYg2H2imsyfCeWX5YUcZVNhHH11oZhvM7GEzO2OoRmZ2i5lVmFlFTU3NeOYTERlTa/ZG/1YuT7SewgisA+a6+9nAD4HfDdXQ3W9393J3Ly8qKhq3gCIiY61ibz3zC7MozE4PO8qgQisK7t7s7q2x+38EUs2sMKw8IiJBi0ScNXsb4nY8AUIsCmY2w8wsdv/8WJa6sPKIiARtZ00rTR3dlMfpeAIEeJ6Cmd0NrAQKzawK+CKQCuDutwHvAj5iZj1AB/Bed/eg8oiIhO2lPdHxhPPnxW9PIbCi4O43nGD5rcCtQa1fRCTeVOytpygnPdTLbZ5I2EcfiYgkjDV7Gzi/rIDYnvO4pKIgIjIODjR2cKCxI67HE0BzH4mIBKKrJ8Ltq3ex5WAz+xva+66XEM9HHoGKgohIIO5bV8W3H32V+UVZzCnI5JzZ+ZxeMpUzZk4NO9qwVBRERALwm4r9LCrO5tFPXxzXYwgDaUxBRGSM7axu4eV9jby7fPaEKgigoiAiMubuXVtFcpJxzTmlYUc5aSoKIiJjqKc3wv3rDrBqcTFFOfE5v9FwVBRERMbQ6h011LR0cn35rLCjnBIVBRGRMXRvRRXTstK4dElx2FFOiYqCiMgYqW/r4rGtR7jmnFJSkyfmz+vETC0iEod+9/IBunt9wu46AhUFEZExc+/aKs4szWXJjPg+QW04KgoiImOgsq6NrYeaue7ciXcYan8qCiIiY2BtZQMAFy6YFnKS0VFREBEZA2srG8hJT2FRcU7YUUZFRUFEZAysrWxg+Zw8kpMm1rQWAwVWFMzsDjOrNrPNJ2h3npn1mtm7gsoiIhKklqPdbD/Swoq58X2thJEIsqdwF3DFcA3MLBn4BvBIgDlERAK1fn8j7qgoDMfdVwP1J2j2CeA+oDqoHCIiQVtb2YAZLJ+dF3aUUQttTMHMSoFrgdvCyiAiMhbWVjaweHoOOVNSw44yamEONH8P+Jy7956ooZndYmYVZlZRU1MzDtFEREYmEnHW72ucFLuOINwrr5UD98QuQFEIXGlmPe7+u4EN3f124HaA8vJyH9eUIiLD2FHdSktnj4rCaLn7vGP3zewu4KHBCoKISDw7dtKaisIJmNndwEqg0MyqgC8CqQDurnEEEZkU1lY2UJidxpyCzLCjjInAioK733ASbW8KKoeISJDW7WvgnDn5E+5azEPRGc0iIqeorrWTPbVtk2bXEagoiIicsnX7GoHJM54AKgoiIqdsbWUDqcnGmaW5YUcZMyoKIiKnaF1lA2fMzGVKanLYUcaMioKIyCmob+ti/f5Gzp9XEHaUMaWiICJyCu6t2E9Xb4R3njtxr8c8GBUFEZGTFIk4v35pH+eV5bN4xsS+qM5AKgoiIifp2Z21VNa184E3zA07yphTURAROUm/erGSgqw0rlg2I+woY05FQUTkJBxq6uDxbdVcXz6L9JTJc9TRMSoKIiIn4Z6X9hNx5/3nT75dR6CiICIyYt29Ee5Zs4+LFxUxZ9rkmABvIBUFEZERenxrNUeaO3n/BXPCjhIYFQURkRH6TcV+SnKncOmS4rCjBEZFQURkhI7NiJqSPHl/OifvlomIjLH6ti4KstLCjhEoFQURkRHo6Y3Q1NFNfqaKgohIwmto7wZgWraKwikxszvMrNrMNg+x/Goz22hm682swswuCiqLiMhoNbR3AainMAp3AVcMs/xx4Gx3Xw7cDPwswCwiIqNS3xYtChpTOEXuvhqoH2Z5q7t77GEW4EO1FREJm4rCODCza81sG/A/RHsLIiJxSUVhHLj7A+6+BLgG+MpQ7czslti4Q0VNTc34BRQRiWmIFYW8zNSQkwQrLo4+iu1qWmBmhUMsv93dy929vKioaJzTiYhAfXsXOekpk3Jm1P5CKwpmttDMLHb/XCANqAsrj4jIcOrbusif5LuOAFJG0sjMFgBV7t5pZiuBs4BfunvjMK+5G1gJFJpZFfBFIBXA3W8D3gn8tZl1Ax3Ae/oNPIuIxBUVhePdB5Sb2ULg58AfgF8DVw71Ane/Ybg3dPdvAN8Y4fpFRELV0N5Fcc6UsGMEbqS7jyLu3gNcC3zP3T8NlAQXS0QkvtS3dk36E9dg5EWh28xuAG4EHoo9N7mH4EVE+qlv76Iga/L/7I20KHwIuBD4qrvvMbN5wK+CiyUiEj86uno52h3RmMIx7v4K8EkAM8sHctz960EGExGJF3VtnQBMS4CiMKKegpk9ZWZTzawA2ADcaWbfDTaaiEh8aGiLzpCqMYXX5Lp7M3AdcKe7rwDeElwsEZH4Ud+eGFNcwMiLQoqZlQDv5rWBZhGRhNCQIPMewciLwpeBR4Bd7r7GzOYDO4KLJSISP+oSqCiMdKD5XuDefo93Ez0jWURk0mto6yI5yZg6RYekAmBms8zsgdiV1I6Y2X1mNivocCIi8aC+vYv8zFSSkizsKIEb6e6jO4lObTETKAUejD0nIjLpJcrZzDDyolDk7ne6e0/sdhegOaxFJCHUtyfGZHgw8qJQa2YfMLPk2O0DaJprEUkQDW1dCXHiGoy8KNxM9HDUw8Ah4F1Ep74QEZn0EmXabBhhUXD3fe7+Dncvcvdid7+G6IlsIiKTWiTiNLR3UaAxhRP6zJilEBGJU81Hu4l4YpyjAKMrCpP/2CwRSXj1CXTiGoyuKOjSmSIy6R0rChpTAMysxcyaB7m1ED1nYbjX3hE72W3zEMvfb2YbY7fnzezsUWyHiEggjhWFRDn6aNhpLtw9ZxTvfRdwK/DLIZbvAS5x9wYzeztwO3DBKNYnIjLmGtoTq6cwormPToW7rzazsmGWP9/v4YuAps0QkbjTNxmejj4aV38DPBx2CBGRgRraupiSmkRGWnLYUcZFYD2FkTKzVUSLwkXDtLkFuAVgzpw545RMRATq27qZlpUedoxxE2pPwczOAn4GXO3uQ06b4e63u3u5u5cXFWnKJREZP/VtneRnTf4ps48JrSiY2RzgfuCD7v5qWDlERIZT396dMDOkQoC7j8zsbmAlUGhmVcAXgVQAd78N+AIwDfixmQH0uHt5UHlERE5FQ1sX86Zlhh1j3AR59NENJ1j+t8DfBrV+EZGx0JBAk+FB/Bx9JCISdzp7emnp7EmYw1FBRUFEZEiN7d0AFGSrKIiIJLz6BDtxDVQURESGlGiT4YGKgojIkBJtMjxQURARGVKiTYYHKgoiIkOqa40WhbwMndEsIpLwGtq7yMtMJSU5cX4qE2dLRURO0p7aNqbnTAk7xrhSURARGUR7Vw9/2VPPRYsKw44yrlQUREQG8dzOOrp6Ily2pDjsKONKRUFEZBBPbDtCdnoK5WUFYUcZVyoKIiIDuDtPbKvmzYsKSUtJrJ/JxNpaEZER2HKwmSPNnVyaYLuOQEVBROR1ntxWDcDKxSoKIiIJ7/Ft1Zw9O4+inMS5NvMxKgoiIv3UtnayoaqRSxOwlwAqCiIix3lqew3ucNnpKgpjyszuMLNqM9s8xPIlZvaCmXWa2WeDyiEicjKe3FZNcU46Z8ycGnaUUATZU7gLuGKY5fXAJ4FvB5hBRGTEunsjrH61hkuXFGNmYccJRWBFwd1XE/3hH2p5tbuvAbqDyiAicjLW7K2npbOHVQl4KOoxE2JMwcxuMbMKM6uoqakJO46ITFLP7KglJcm4aGFizXfU34QoCu5+u7uXu3t5UVFR2HFEZJLasL+RJSU5ZKWnhB0lNBOiKIiIBC0ScTZVNXHWrLywo4RKRUFEBNhT10ZLZw/LE7woBNZHMrO7gZVAoZlVAV8EUgHc/TYzmwFUAFOBiJl9Cljq7s1BZRIRGcrGqkYAzpqdG3KScAVWFNz9hhMsPwzMCmr9IiInY8P+JjJSk1lYlB12lFBp95GICNGewrLSqQl1PebBJPbWi4gQPWlty8HmhB9kBhUFERG2H26hsyfCWbMSezwBVBRERNhY1QTA8tnqKagoiEjC21jVSF5mKnMKMsOOEjoVBRFJeBuqmjizNDdhJ8HrT0VBRBJaR1cvrx5p4WwNMgMqCiKS4F451ERvxDXIHKOiICIJbcP+6CDz2RpkBlQURCTBbaxqZMbUKUyfOiXsKHFBRUFEEtqGqibtOupHRUFEElZTRzd7atu066gfFQURSVibYietqafwGhUFEUlYGw9Ep8s+s1RF4RgVBRFJWJuqmpg7LZO8zLSwo8QNFQURSVgbdfnN11FREJGEVNvayYHGDs7SrqPjqCiISELadCA6yHymBpmPE1hRMLM7zKzazDYPsdzM7AdmttPMNprZuUFlEREZaOP+JsxgmXoKxwmyp3AXcMUwy98OLIrdbgF+EmAWEZHjbDrQyIKibLLTA7tU/YQUWFFw99VA/TBNrgZ+6VEvAnlmVhJUHhGR/jZWNWk8YRBhjimUAvv7Pa6KPfc6ZnaLmVWYWUVNTc24hBORyetw01GqWzo1njCIMIvCYFez8MEauvvt7l7u7uVFRUUBxxKRyW5jVfSkNR2O+nphFoUqYHa/x7OAgyFlEZEEsulAE8lJxtKSqWFHiTthFoU/AH8dOwrpDUCTux8KMY+IJIgNVU0sKs4mIy057ChxJ7BhdzO7G1gJFJpZFfBFIBXA3W8D/ghcCewE2oEPBZVlvFW3HKW+rYvF03N0zVeROOPubKpq5K1LZ4QdJS4FVhTc/YYTLHfgY0GtfzzVtnbyqxcr2bC/kS0Hm6lu6QTgDfML+PyVSzWYJRJHqho6aGjv1vdyCAl3gK678+T2an705C52HGnhQ2+ax4cvnj/sscq7a1r51Yv7yM1I5ZpzZjJ3WhYAnT29/OL5vfzw8Z20dfWwqDiHixYVcsbMXCIR5ydP7+J/3fosVy+fyU1vLKPlaA+Hmjo41HSU7PQU3nH2TIrH4WpPkYjT2NFNXWsndW1dlE3LYkaurjIlienYmcyaLntwFv2DfeIoLy/3ioqKk35db8R5ePMhfvTkLrYeaqY0L4PFM3J4Yls107LS+PilC3nfBXNIT3ltH+OWg038+Kld/HHTIVKTkuiORHCHFXPzWbW4iHvXVlFZ186qxUV8/qqlLCzOPm6dzUe7ue2pXfz82T109kRelynJ4OLTinjXilm8cUEheRmpJCWdeHdTQ1sXeZmpw+6acnee3VnLD5/YScXeeiL9/jcnJxlvO2M6N71xHueV5WsXlySUrz28lTuf3cumf3vrcd/3yc7M1rp7+QnbJUpRuOelffzz/ZuYX5TFRy5ZwDXnlJKanMT6/Y184+FtvLC7juQkIyM1mSmpyaSnJHGgsYOc9BQ+eOFcbr5oHl09EX6//iAPvFzFq0daWViczf+56nRWLi4edt2HmjpYV9lI8dT0vmvB7m9o5761VTzw8gEONR0Foj/WBVlpTMtK47yyAq47t5Tls/MwMyIR5/Ft1dy+ehdr9jYwpyCTK5bN4IplM1g+K4+kJMPd6e51nt9Vyw8e38G6fY2U5E7hmnNKmZ6TTkF2OnkZqTy3q5Z7XtpPU0c3p5dM5YyZU0lJMlKSjZSkJDp7euno6qWju5feiFM2LYvFM3JYMmMqi6ZnMyX15L5IDW1dPLOzlhd21ZGabJTkZjAzbwqz8jNZPjuP5BEUQpFT0Rtxvvvn7SwoyuavzppJWkoS7/vpi7R29vCHj18UdrxxpaIwQEdXL09ur+ZtZ8x43Y/Qsb+qX9xdx9HuCB3dvRzt6mVBcTYfeMNccjNSX9f+YNNRinPSSU0e3QFcvRHnxd11bD/cQl1bJ3WtXRxuPsoLu+ro7IkwvzCLtyydzuNbj7Crpo3SvAyuO7eUjVVNPL+rlu5eJzs9BXfnaE+E3liXoDQvg4+uWsC7Vswa9K+hjq5efr/+AHev2U9tSyfdvRF6Ik5Pb4T01GQy05LJiP3476lt6+vpmMGs/AzmF2azoCibOQUZFGSnk5+ZSn5mGr0R79tFdrCxgzV7G9hQ1Yg7TJ0S3UXXfLSnL8ei4mw+9ZbTePuyGSPqJYmcjP9es4/P3bcJgBlTp/ChN5Vx65M7ecfZM/nqtWeGnG58qShMcM1Hu3l40yHuX3eAv+ypZ2nJVP7ukvlcdWYJKbFC1NTRzRPbjrCuspG0lKRYLyeJWfmZXHlmCWkpY3PEcW/E2VvXxvbDLWw/3MLu2jZ217Syu6aNju7eIV+XnpLE0plTWXlaMRefVshZs6K9gtbOHg41drD5YBM/enIXO6tbWTIjh09ffhqXnz5dxUHGRFtnD6u+/RSz8jP4xKWL+Okzu3l+Vx0A33jnmbznvDkhJxxfKgqTSGtnD1lpyXG37z8Scerbu2hs76KhvZu61i6SDGbmZVCSO4WCrLQTZu6NOA9tPMj3HtvBnto2lszI4aOrFnLVmSXH9ejaOnto6+qhKDs97j4HiU/fe+xVvvfYDu77yBtZMTcfgM0HmvjzK0f42zfPI2dK6gneYXJRUZAJpac3wh82HOTHT0V7DvMLs/irs2dSWdfG5gNN7K5twx1y0lNYUJzNwuJsyufmc9VZJQn35ZYTO9J8lJXfeopLlxTzo/drVn5QUZAJKhJxHtlymB8+sZNXDjVTkjuFM2bmsqx0KnkZqeyqaWNndSs7qlupbe1kSmoSV55ZwrvLZ3PBvAL1IgSAz/12I/e/XMVjn7mk7xDyRDfSopBw5ylIfEtKMt5+ZglXLJtBa2fPkL0Ad2f9/kZ+U1HFgxsOcv+6AyyZkcPHVi3kygG7niSxbD3UzG/W7ufmN81TQTgF6inIhNfR1cv/bDrEbU9Hdz0tKMriY6sW8o6zZ/YNysvk19bZw9Ov1vDjp3ayr66d1f+0irzMtLBjxQ3tPpKE0xtx/rT5MD98YgfbDrcwKz+DD795Pu8un62JzyaxF3bV8fNnd7N6Ry1dPRHyM1P50jvO4Orlg16eJWGpKEjCikScx7Ye4band7FuXyP5manc+MYyrjtnFnOmZYYdT8ZQdfNRLvvO02SkJXPlmSW87YwZnFeWrx7iIFQURIA1e+u57aldPL6tGoAlM3J46xkzWLW4iJLcDPIyU487Q7s34rR39ZCekjxm53kMxd052h0hNdn0I3aKPnXPy/xx02Ee+fTFzCvU+MFwNNAsApxXVsB5NxWwv76dR7Yc5tEtR7j1iR384PEdfW3SU5LISEumvauXrtiZ2/mZqXx05UI+eOHcEU/rcezM+AdePkBHV2/sOeiJOG2dPbT2u7V39tDe3Ys7ZKYlc8G8At60sJCLFhUOOeW6u1PV0MGGqkbW72tk/f5G9tS2saw0lzcvGvq1zUe7eeyVI/xx0yGqGjrITk8he0oK2ekpnDUrl2vOKaU4Z+JNkPji7jp+t/4gn7h0oQrCGFJPQRJObWsnFXsbaGjvoqG9i6b2bjq6e8lISyYzNYXMtGSe2VnL6ldrmDF1Cp+8bBHXl88ackqT7t4ID208yO2r97D1UDP5makUZqdz7Lc5yYyc2I9w9pRUstKSyUxLISs9mYy0ZA43HeXZnbXsrmkDokViQVE284uymFeYRUNbF1sPtbD1cDMtsSlC0lKSWDZzKmWFWazf39j32rzMVEpyMyjKSac4Jz0679SOWrp6I8zMncLSmbm0d0ULU1NHN5V17SQnGasWF3F9+WwuXVI86qlbxkN3b4SrfvAMbZ29PPaZSzRmNALafSQySi/uruObf9rGun2NpKckcdr0nNjEgDl09kSorGtjb107O6tbqW/rYlFxNh++eD5XL595SrNvHmjs4LmdtbxysJndtW3sqm7lQGMHWWnJLCmZyukl0UkJz5qVy5IZU4/bvXWwsYNnd9Ty8v5GalqOUtPSSU1LJ8nJxluXzuCqs0r6Jk7sb1dNK/dWVHH/uiqqWzqZXZDBx1Yu5LpzZwW++2w0frp6N1/941Z++tflXL50ethxJgQVBZEx4O48/WoNz+6oZfuRFrYeaqG2NXoRpaKcdMqmZTJ3WhZXnjmDlacVj/m8TUe7e0lLTgp8Pqie3ghPbKvmR0/uZENVE6V5GfzdJfNZPjuPgqw0CrLSyEiNj6lWDjcd5bLvPMUF86fx8xvL4yLTRBAXRcHMrgC+DyQDP3P3rw9YPhe4AygC6oEPuHvVcO+poiBhq2/rIj0liaxhLsw0Ubk7q3fU8v3HXmXdvsbjlmWnp7BqSTHvOHsmF59WGMq1CKoa2vnIr9ax/UgLj336Eh1NdhJCLwpmlgy8ClwOVAFrgBvc/ZV+be4FHnL3X5jZpcCH3P2Dw72vioJI8NydLQebOdR0lIa2Lurbu9hT08ajrxymob2bqVNSeMvS6ZxfVsA5c/JZWJwd+FnkT79awz/c8zK9vc5337Ncu41OUjwcfXQ+sNPdd8cC3QNcDbzSr81S4NOx+08Cvwswj4iMkJmxrDSXZaXHX7Ly//Yu47mdtTy44RBPbDvC/esOANFexIq5+bxl6XQuP336mF7uNRJxfvjETr73+Kssnp7DTz6wQkcbBSjIolAK7O/3uAq4YECbDcA7ie5iuhbIMbNp7l4XYC4ROUWpyUmsXFzMysXFuDuVde2s29fAun0NPLezjn/93Wb+9XebOXt2Hlcum8G1547ucNfu3gif+c0GHtxwkOvOKeWr156pI40CFmRRGKwvOXBf1WeBW83sJmA1cADoGfgiM7sFuAVgzpzEujCGSLwyM8oKsygrzOK6c2fh7uyqaeWRLUd4dMthvvbwNr75yHZWLS7m3eWzWHWSh7se7e7l479ex2Nbq/ncFUv4+0vma1B5HAQ5pnAh8CV3f1vs8b8AuPvXhmifDWxz91nDva/GFEQmhp3Vrfx2bRX3rauipqWTktwp3PjGMm44bw65mcNfA6Ots4db/rOC53fV8ZWrl/GBN8wdp9STVzwMNKcQHWi+jGgPYA3wPnff0q9NIVDv7hEz+yrQ6+5fGO59VRREJpae3ghPbq/hzuf28PyuOjLTkrl+xSwuPq2IeYVZzC7IJDU5ia7YuR87qlv56TO72VjVxLevP4trzxn270QZodAHmt29x8w+DjxC9JDUO9x9i5l9Gahw9z8AK4GvmZkT3X30saDyiEg4UpKTuHzpdC5fOp0tB5v4+bN7+PVL+/jFC5UAJCcZxTnp1LR00hOJ/pE6JTWJH73vXK5YNiPM6AlJJ6+JyLhr6uhmZ3Ure2vb2FPbxoHGDkpyp7BoejaLinNYUJStAeUxFnpPQURkKLkZqayYm8+KuflhR5EB4ndyExERGXcqCiIi0kdFQURE+qgoiIhIHxUFERHpo6IgIiJ9VBRERKSPioKIiPSZcGc0m1kTsGOQRblA0zDPDVx+7PFgbQqB2lOMOFiOkSw/Uf6Bjwe7r/zxkR9OfRtOlH+4NsPlHfh4Mubvfz8e8g+Xs/+FCVNtAAAH0klEQVTj8foNmuvuRSd8tbtPqBtw+0if7//cwOXHHg/WhujcTGOab7T5h9uegdui/OHmH802nCj/yWxDouUfi39DY5l/uJzDfO6BfwdOdJuIu48ePInnHxxm+YMjaHMqTvQep5p/4OPB7iv/5M8/XJvh8g58PBnzj3T9wxnL/AOfi5ffoGFNuN1H48HMKnwEE0fFK+UP30TfBuUPV5j5J2JPYTzcHnaAUVL+8E30bVD+cIWWXz0FERHpo56CiIj0mfRFwczuMLNqM9t8Cq9dYWabzGynmf3A+l013Mw+YWbbzWyLmX1zbFMfl2HM85vZl8zsgJmtj92uHPvkfRkC+fxjyz9rZh67rGsgAvr8v2JmG2Of/aNmNnPsk/dlCCL/t8xsW2wbHjCzvLFPflyOILbh+th3N2JmY77vfjSZh3i/G81sR+x2Y7/nh/2OnJLRHLo0EW7AxcC5wOZTeO1LwIWAAQ8Db489vwp4DEiPPS6eYPm/BHx2on7+sWWziV7qtRIonEj5gan92nwSuG2C5X8rkBK7/w3gGxPt3xBwOrAYeAooj5fMsTxlA54rAHbH/psfu58/3PaN5jbpewruvhqo7/+cmS0wsz+Z2Voze8bMlgx8nZmVEP3yvuDRT/+XwDWxxR8Bvu7unbF1VE+w/OMmwPz/AfwTEOigWBD53b25X9MsAtyGgPI/6u49saYvArOCyh/gNmx19+3xlnkIbwP+7O717t4A/Bm4Iqjv+KQvCkO4HfiEu68APgv8eJA2pUBVv8dVsecATgPebGZ/MbOnzey8QNO+3mjzA3w81v2/w8zG+5qIo8pvZu8ADrj7hqCDDmHUn7+ZfdXM9gPvB74QYNbBjMW/n2NuJvoX6ngby20YLyPJPJhSYH+/x8e2I5DtS7hrNJtZNvBG4N5+u9/SB2s6yHPH/qJLIdqNewNwHvAbM5sfq9aBGqP8PwG+Env8FeA7RL/cgRttfjPLBD5PdBfGuBujzx93/zzweTP7F+DjwBfHOOqgxip/7L0+D/QA/zWWGU9kLLdhvAyX2cw+BPxD7LmFwB/NrAvY4+7XMvR2BLJ9CVcUiPaOGt19ef8nzSwZWBt7+AeiP5z9u8WzgIOx+1XA/bEi8JKZRYjOVVITZPCYUed39yP9XvdT4KEgAw8w2vwLgHnAhtiXaxawzszOd/fDAWeHsfn309+vgf9hnIoCY5Q/Ntj5V8Bl4/HH0ABj/f9gPAyaGcDd7wTuBDCzp4Cb3H1vvyZVwMp+j2cRHXuoIojtG+sBlni8AWX0G/ABngeuj9034OwhXreGaG/g2CDOlbHn/x74cuz+aUS7djaB8pf0a/Np4J6J9PkPaLOXAAeaA/r8F/Vr8wngtxMs/xXAK0BRkLnH498QAQ00n2pmhh5o3kN070R+7H7BSLbvlHKP1//UsG7A3cAhoJtoZf0bon9p/gnYEPvH/YUhXlsObAZ2Abfy2sl+acCvYsvWAZdOsPz/CWwCNhL9i6pkIuUf0GYvwR59FMTnf1/s+Y1E56kpnWD5dxL9Q2h97BbY0VMBbsO1sffqBI4Aj8RDZgYpCrHnb4597juBD53Md+RkbzqjWURE+iTq0UciIjIIFQUREemjoiAiIn1UFEREpI+KgoiI9FFRkEnBzFrHeX0/M7OlY/RevRadMXWzmT14ollHzSzPzD46FusWGUiHpMqkYGat7p49hu+X4q9N+hao/tnN7BfAq+7+1WHalwEPufuy8cgniUU9BZm0zKzIzO4zszWx25tiz59vZs+b2cux/y6OPX+Tmd1rZg8Cj5rZSjN7ysx+a9HrB/zXsfnqY8+Xx+63xia422BmL5rZ9NjzC2KP15jZl0fYm3mB1yb+yzazx81snUXnzL861ubrwIJY7+Jbsbb/GFvPRjP7tzH8GCXBqCjIZPZ94D/c/TzgncDPYs9vAy5293OIzlD67/1ecyFwo7tfGnt8DvApYCkwH3jTIOvJAl5097OB1cCH+63/+7H1n3BOmtjcPZcRPcsc4ChwrbufS/QaHt+JFaV/Bna5+3J3/0czeyuwCDgfWA6sMLOLT7Q+kcEk4oR4kjjeAiztNyvlVDPLAXKBX5jZIqKzSqb2e82f3b3/PPgvuXsVgJmtJzqfzbMD1tPFa5MKrgUuj92/kNfmt/818O0hcmb0e++1ROfLh+h8Nv8e+4GPEO1BTB/k9W+N3V6OPc4mWiRWD7E+kSGpKMhklgRc6O4d/Z80sx8CT7r7tbH980/1W9w24D06+93vZfDvTLe/Njg3VJvhdLj7cjPLJVpcPgb8gOi1FoqAFe7ebWZ7gSmDvN6Ar7n7/zvJ9Yq8jnYfyWT2KNFrFQBgZsemLc4FDsTu3xTg+l8kutsK4L0nauzuTUQvz/lZM0slmrM6VhBWAXNjTVuAnH4vfQS4OTZnP2ZWambFY7QNkmBUFGSyyDSzqn63zxD9gS2PDb6+QnTKc4BvAl8zs+eA5AAzfQr4jJm9BJQATSd6gbu/THQWzfcSvXhNuZlVEO01bIu1qQOeix3C+i13f5To7qkXzGwT8FuOLxoiI6ZDUkUCErtKXIe7u5m9F7jB3a8+0etEwqQxBZHgrABujR0x1Mg4XfJUZDTUUxARkT4aUxARkT4qCiIi0kdFQURE+qgoiIhIHxUFERHpo6IgIiJ9/j+RAs6Sj5jeEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 05:11 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th><lambda></th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.525171</th>\n",
       "    <th>0.736362</th>\n",
       "    <th>0.532685</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.409036</th>\n",
       "    <th>0.425513</th>\n",
       "    <th>0.759533</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.369458</th>\n",
       "    <th>0.314608</th>\n",
       "    <th>0.868093</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>0.328593</th>\n",
       "    <th>0.296097</th>\n",
       "    <th>0.857588</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(4, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'{name}-stage-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_lr = 5e-4\n",
    "lrs = [max_lr/100, max_lr/10, max_lr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 15:46 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th><lambda></th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.299970</th>\n",
       "    <th>0.285136</th>\n",
       "    <th>0.863424</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.286753</th>\n",
       "    <th>0.260144</th>\n",
       "    <th>0.887743</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.277695</th>\n",
       "    <th>0.269493</th>\n",
       "    <th>0.872763</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>0.259490</th>\n",
       "    <th>0.234493</th>\n",
       "    <th>0.895720</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>0.229194</th>\n",
       "    <th>0.224973</th>\n",
       "    <th>0.912257</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>6</th>\n",
       "    <th>0.217003</th>\n",
       "    <th>0.232760</th>\n",
       "    <th>0.897082</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>7</th>\n",
       "    <th>0.202161</th>\n",
       "    <th>0.215272</th>\n",
       "    <th>0.907977</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>8</th>\n",
       "    <th>0.203944</th>\n",
       "    <th>0.228468</th>\n",
       "    <th>0.894163</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>9</th>\n",
       "    <th>0.201418</th>\n",
       "    <th>0.222140</th>\n",
       "    <th>0.896498</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>10</th>\n",
       "    <th>0.198599</th>\n",
       "    <th>0.217933</th>\n",
       "    <th>0.899416</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, lrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'{name}-stage-2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzsnXd4VcXWh99J75BAqAFClRoghCDSiwiiYEEpchUb115QP7EjNsSGXlGuegE7ioqigCAKUpQSeu8BAkgJNaQn8/2xT83ZJ/2ksd7nOc/de/bM3nNyZa8za9b6LaW1RhAEQRDyw6u8JyAIgiBUfMRYCIIgCAUixkIQBEEoEDEWgiAIQoGIsRAEQRAKRIyFIAiCUCBiLARBEIQCEWMhCIIgFIgYC0EQBKFAfMp7AqVFzZo1dXR0dHlPQxAEoVKxbt26U1rryIL6VRljER0dTUJCQnlPQxAEoVKhlDpYmH7ihhIEQRAKRIyFIAiCUCBiLARBEIQCqTJ7FoIgVB2ysrJISkoiPT29vKdSZQgICCAqKgpfX99ijRdjIQhChSMpKYnQ0FCio6NRSpX3dCo9WmuSk5NJSkqicePGxbqHuKEEQahwpKenU6NGDTEUpYRSiho1apRopSbGQhCECokYitKlpH/PS95YXMzI5u1Fu9h4+Gx5T0UQBKHCcskbi7SsHN77Yy+bk8RYCIJgkJycTIcOHejQoQN16tShfv36tvPMzMxC3eP2229n165dHp5p2XHJb3BbF2Zal+s0BEGoQNSoUYONGzcCMGHCBEJCQnj88ced+mit0Vrj5WX+m3vGjBken2dZcsmvLKx+PC3WQhCEAti7dy9t27blnnvuITY2lmPHjjF27Fji4uJo06YNEydOtPXt3r07GzduJDs7m+rVqzN+/Hjat29P165dOXHiRDl+i+IhK4vynoAgCPny4s/b2H70fKnes3W9MF64tk2xxm7fvp0ZM2Ywbdo0ACZNmkRERATZ2dn06dOHYcOG0bp1a6cx586do1evXkyaNIlx48Yxffp0xo8fX+LvUZZc8isLK7KuEAShMDRt2pTOnTvbzr/++mtiY2OJjY1lx44dbN++3WVMYGAggwYNAqBTp04kJiaW1XRLDVlZWJYW4oUShIpJcVcAniI4ONh2vGfPHt59913WrFlD9erVGT16tGkug5+fn+3Y29ub7OzsMplraXLJryyUxRE1eeHOUrnfr1uPkXQmtVTuJQhCxeb8+fOEhoYSFhbGsWPHWLhwYXlPyWNc8isL66ZFelZuqdzuni/WA9C2fhi/PNijVO4pCELFJDY2ltatW9O2bVuaNGlCt27dyntKHuOSNxalmSSak2v3ZW09cp6cXI23l2yhC0JlZsKECbbjZs2a2UJqwYim/Pzzz03HrVixwnZ89qw9j2vEiBGMGDGi9CfqYcQN5XDs+LIvDlk5zquTFXtPleh+giAIFQUxFg5LizEz1pToXt+sPex0LrkbgiBUFTxqLJRSA5VSu5RSe5VSLkHFSql7lFJblFIblVIrlFKtLe1XKqXWWa6tU0r19eQ8rSzfU/yVwF97T/HC3G1ObRfSK1/EgyAIghkeMxZKKW9gKjAIaA2MtBoDB77SWrfTWncAJgNvW9pPAddqrdsBtwHmTsHSmGcJxq7an0z0+Hms2HOKUZ+sdrn+4NcbSEg8XYInCIIgVAw8ubKIB/ZqrfdrrTOBWcBQxw5aa8e0zGAsuXFa6w1a66OW9m1AgFLK3xOTLMkG94iPVgEw+n+uhsLKsGl/iztKEIRKjyeNRX3A0YmfZGlzQil1v1JqH8bK4iGT+9wIbNBaZ3hikqoEa4sbOrp8HQBqhjjbteSLmew7mcKZi4VTqxQEQahoeNJYmL2FXX5ia62naq2bAk8CzzrdQKk2wOvAv00foNRYpVSCUirh5MmTxZtkCVYWgX7epu2f3tHZ6Tzu5cX0e+tPBkxZxs+bjspKQxAqOL1793ZJsJsyZQr33Xef2zEhISEAHD16lGHDhrm9b0JCQr7PnjJlCqmp9sTeq6++2in0trzwpLFIAho4nEcBR930BcNNdZ31RCkVBcwBbtVa7zMboLX+SGsdp7WOi4yMLIUpF431h8z/D2xSM4T7+zR1aT95IYMHv97AD+uPeHpqgiCUgJEjRzJr1iyntlmzZjFy5MgCx9arV4/vvvuu2M/Oayzmz59P9erVi32/0sKTxmIt0Fwp1Vgp5QeMAOY6dlBKNXc4HQzssbRXB+YBT2mtV3pwji4ri4k/byc9K6dQY3ccc1XCTJw0mEA/bx7q15yxPZuYjjt0WuRABKEiM2zYMH755RcyMgzvd2JiIkePHqVDhw7069eP2NhY2rVrx08//eQyNjExkbZt2wKQlpbGiBEjiImJYfjw4aSlpdn63XvvvTZp8xdeeAGA9957j6NHj9KnTx/69OkDQHR0NKdOGZGab7/9Nm3btqVt27ZMmTLF9rxWrVpx991306ZNGwYMGOD0nNLCYxncWutspdQDwELAG5iutd6mlJoIJGit5wIPKKX6A1nAGYzIJ4AHgGbAc0qp5yxtA7TWHheBn77yAHWq+TO2p+vKoCC6NathO/b38eaamLp8tGy/S793f9/Do1e2KNE8BeGSYcF4+GdL6d6zTjsYNMnt5Ro1ahAfH8+vv/7K0KFDmTVrFsOHDycwMJA5c+YQFhbGqVOnuPzyyxkyZIjb+tYffvghQUFBbN68mc2bNxMbG2u79sorrxAREUFOTg79+vVj8+bNPPTQQ7z99tssWbKEmjVrOt1r3bp1zJgxg9WrV6O1pkuXLvTq1Yvw8HD27NnD119/zccff8zNN9/M999/z+jRo0vnb2XBo3kWWuv5WusWWuumWutXLG3PWwwFWuuHtdZttNYdtNZ9tNbbLO0va62DLe3Wj0cMhdkGd0YhdaKa1QpxOp95e7zTuZ/PJZ/zKAiVFkdXlNUFpbXm6aefJiYmhv79+3PkyBGOHz/u9h7Lli2zvbRjYmKIiYmxXfv222+JjY2lY8eObNu2zVTa3JEVK1Zw/fXXExwcTEhICDfccAPLly8HoHHjxnTo0AHwnAS6aEOZ/CDIKqTsR7aDvMdH/+qEr7ezcWhZJ6xEcxMEgXxXAJ7kuuuuY9y4caxfv560tDRiY2OZOXMmJ0+eZN26dfj6+hIdHW0qSe6I2arjwIEDvPnmm6xdu5bw8HDGjBlT4H3yC4zx97dHYHp7e3vEDXXJ//Q1Wzzm1Xhyh6NSbUSwn2mfcW7cTSJjLggVm5CQEHr37s0dd9xh29g+d+4ctWrVwtfXlyVLlnDw4MF879GzZ0++/PJLALZu3crmzZsBQ9o8ODiYatWqcfz4cRYsWGAbExoayoULF0zv9eOPP5KamsrFixeZM2cOPXqUnbK1GAsTq5+Wad/gfmL2Jv7z+x4XA7I56Sz/nLf/EqgeZG4sujatYdq+ogTSIoIglA0jR45k06ZNNpXYW265hYSEBOLi4vjyyy9p2bJlvuPvvfdeUlJSiImJYfLkycTHG67q9u3b07FjR9q0acMdd9zhJG0+duxYBg0aZNvgthIbG8uYMWOIj4+nS5cu3HXXXXTs2LGUv7F7VFWJ+Y+Li9MFxS+bkZurafL0fKe2EZ0bMOlGw7cYPX6erX3p472Jrhns1N6yTiiTboyhQwP3oW0bDp2hVlgA3Sb9YWtrH1WNnx7oXuT5CsKlwI4dO2jVqlV5T6PKYfZ3VUqt01rHFTRWVhYmfqhj58x9h2ahslqTr6EA6NgwnPrVA53a4htHFH6SgiAI5cwlbyzM+HO3eTb4ruMX+HmTc17hXT0aF+sZHy8/wK9bjxVrrCAIQllzyRsLd/HRZkxZvIcHv94AQKu6YVzZujY3xTUoYJSdxEmDeaBPM9v5vV+uL/xEBeESo6q4yCsKJf17XvLGIj/c/XHjXv6N82lZBPiaa0Plx2MD7NFR8m9BEMwJCAggOTlZDEYpobUmOTmZgICAYt/jks+zcMfPm47SrVlN02unUgz12KZ5kvIKg+NKxl+S9gTBlKioKJKSkiiuQKjgSkBAAFFRUcUeL8bCgYFt6vDrtn8Ao3DRU4PyD4vbeuRciZ7nTrVWEC51fH19ady4ePuBgmeQn7YOvHJ9W6fz1xbszLf/jbHm9SwKy9nUrBKNFwRBKCvEWDjg41W0P8eTA/NfeQiCIFQVxFg44OOtuKu769L3wb7NTHqDj3fJ/3yF2cDbcew8P25wroGRlplD9Ph53P1Z0RMRBUEQiooYCwd8vJUtQ9uR266IdmmbNjrWpa04pFlqZ6Rn5TBpwU5SM7NJy8zh8dmb2HrkHKv2JzPo3eU88s1Gp3G/7TCULn/bfpy/9op0iCAInkU2uB3w9fLC28s17yLA15s7uzfmfysO2NoGtq1bKs9MSc8myM+H//yxh2l/7mP1gWTaR1Xnu3VJfLcuyalvZnauTfbccUUy6pPVJE4aXCrzEQRBMENWFg54mRgKAG+lePrqVsy4vbPp9aLy8wPdubVrIwBSMrIBmLrEqBy74dBZZv6VaDruXJp9Q7ywNTcEQRBKAzEWebiuQ3388uxFeHspvL0UvVtE8nC/5qx5pl+JntEuqho9mxs1w6/9zwp2/nOe/q1qFzjOsdzr/32/2elaRnbhSsEKgiAUB48aC6XUQKXULqXUXqXUeJPr9yiltiilNiqlViilWjtce8oybpdS6ipPztORQD9vdr080KnN6ppSSvHolS2oFVr8LEgrIQGGB/BiZg4Dpyxn8Q731bas5GcQUtKzSzwnQRAEd3jMWCilvIGpwCCgNTDS0RhY+Epr3U5r3QGYDLxtGdsaGAG0AQYCH1juVybk1Ysy28coKSH+Rd8uSst073pydFEJgiCUNp5cWcQDe7XW+7XWmcAsYKhjB621o+Z3MGDdtR0KzNJaZ2itDwB7Lfcrc5646jKP3Lc4xiLdZGUxoLXhvlq6S2QRBEHwHJ40FvWBww7nSZY2J5RS9yul9mGsLB4qytiy4Ao3le5KitUNVRQcK/j1b1WLID9vxlmECSf+kn+xd0EQhJLgSWNh5rtxyUDTWk/VWjcFngSeLcpYpdRYpVSCUirBU4Jj/j6e8X45riwKu8rYePis7TgzR9O8dijVA83LuZYVogoqCJcGnjQWSYBjsYco4KibvmC4qa4rylit9Uda6zitdVxkZGQJp2uOv69n/kQBvt58eEssbeqF2cJnrTx/Td6tHYO3f9ttO07PyiHAx4vAYsiklyaNn5pP0zxlaQVBqHp40lisBZorpRorpfwwNqznOnZQSjV3OB0M7LEczwVGKKX8lVKNgebAGg/O1S2elBEf1K4uTSNdZc7XJp52Or/boRpfbq7xSz4jK4cAX2+PGbPCkJVjbLjn5MrqQhCqOh5702its4EHgIXADuBbrfU2pdREpdQQS7cHlFLblFIbgXHAbZax24Bvge3Ar8D9WutySSTwlBvKSligqwuqee1QFo/rCcBLQ9vwzGD7SuNiZjanL2ayKekcK/eecirAtPfEhSI9+/DpVKLHz2NLUvGk1tMc8j4kz0MQqjYelfvQWs8H5udpe97h+OF8xr4CvOK52dmZNjqW5rVDTa/5ebhAUV5JD4Cr2tSmWa1QDrx2tUsY7+erDhLXKAKAbMsv+mcHt+LleTvo//ayIsl+LNl1AoBZaw/RLqpdkeee7rDhvi7xDFe4KRYlCELlR7ShMNd5+vupvizddZJqgb4efXb3ZpG2hLwnrrqMHs1r0qZeNcC8PvjkX3cxqktDp7aM7MJJf6Rn5ZCZk0tYgPGdlu02ggKK60ayVgwE2HX8ghgLQajCiNyHG+pWC2RkfMOCO5aQN4bF2I5v6hRFTFR1035NIu1quF+tPuR07WxqZt7uptzwwV/ETFhkO1+8w1hZZBfTWLy1aJftuLw32gVB8CxiLMqZ8GA/WtYxXGBB+YTQXt/BNc3k3REdABjVpVGBz7mYkc32Y/YcyBV77LLmZq6wwhDgUBY2Sza5BaFKI8aiAvDJbXG8O6JDvvkWw+MbuLSFWhL7GtcMZniccT3X5KV9IT2LNi8stJ3n5GoOnU4t6bSZt/mY7fi5H7dyKiWjxPcUBKFiIsaiAhAVHsRQk5WDI2bihd4OZWAbW9xUZpIgJy84v8RT0rNpGBFU6PldSM8qVPLdqI9XFfqegiBULsRYVCLyypj7OAgcZlk2uR/6egMbDp1x6nfg1EWn8y9WHyQzx25UHPdD8rLvZArtJizi0zw1NvIaIIDdx1Py/wKCIFRaxFhUIl4c2sbp3FEN9w9LGOziHSe4/oO/+GPncVv9izs/da7T/cbCXZy5aFepzcwnmuq1+TsA+NnB5QQwY+UBs+6CIFRRxFhUIupXD3Q6d1xZXBtTz+naHTMTeCkfccHHZm8CYHC7ui5yI47ENzZyOtrVr+bUfuRsWuEmLQhClUCMRSVj6eO9bceOCYNjroh26fvl6kNEj5+X7/3Cg305m+p+T+LV+TsBWH/oDNHj5xE9fh5pmTnENQoHYOX4vux79WoA4qMjivJVBEGoRIixqGRE1wwmuoaxOR3sED3lrn54QXyxysjZ+Gbt4Xz7bXaQBDmdmslzP20DoEawH95einb1qxHsL7kWglBVEWNRCYkM9QdwqRVeEsb/sKXQfZfvtsvBW7WpAn29nbSiBEGoWoixqIRMHRXLS9e1pUERwl+/GXs5U0fFurRf39EI2fX2UmRk53Axn/0LK+kmRiHAz9upOJMgCFULMRblSU42pJ0puF8eaoUF8K/LXbO2h3aoZ9LbIMDXm8ExrhpYEy0RVqPiGzLo3eVOyXsAwX6urqVUE2Oxan8ym5LO8di3mwqcvyAIlQ8xFuXJT/fDp9dC2tmC+xaCl65ry4Rr7XLmjpvh1lXItNHOq4vQAF/Cgwxhwf0nnfMxHMc5YpUHcTRO1vDb79cnmWaRC4JQuRFjUZ7E3AQndsKXN0FGyRPawgJ8GdPNXijJugEe4u9DRLBRftVMYTckwMdt+GxmjmsOhtWo3Nm9scs1gHd/32PaLghC5UWMRXnSrD8Mmw5HEuCbWyArvVRvH+Lvw7In+vDrIz2c2h/t38Lp/PDpNOZsOGI7dwyjdUzYa1MvzGmcY+ElRzX1d3/fk2/uhiAIlQ8xFuVN6yEw9APYvxS+uwNysgocUlgCfL1oWCOIqHBnV9Id3aPzHZeYbBcZdKyV8d7IjjSvZS8D6yhLvuBhZ4P0wZK9xZmyIAgVFI8aC6XUQKXULqXUXqXUeJPr45RS25VSm5VSvyulGjlcm2wpubpDKfWeMqsEVFXoMBKufhN2zYMf74PcwhUzKgh3f7LQAF/WPduf7ROvMr2e7eB6SknPxtfbuE+dsACn1YRj/e+WdcJ4qJ+9pPoHS/eVaO6CIFQsPGYslFLewFRgENAaGKmUap2n2wYgTmsdA3wHTLaMvQLoBsQAbYHOQC9PzbVCEH839HsBtnwL8x+DQqi8loQaIf4E+ZlLon/290EAUjOzScvK4bEBl5E4aTDB/j5OVfUC8hQ8smZ1uyM3V/PduiSyTPZBBEGo2HhyZREP7NVa79daZwKzgKGOHbTWS7TWVp/HKiDKegkIAPwAf8AXOO7BuVYMeoyD7uMgYTr89nyxDcbQDvVcdKSKgvVlvuPYBcDI0rbiWEApwMfZWPRsEel0Hj1+HnuOX7Cd/7bjOI/P3sR7sgEuCJUOT9bgrg84akgkAV3y6X8nsABAa/23UmoJcAxQwPta6x15ByilxgJjARo29HwJ1DKh3/OQcQH+eg8CwqDnE0W+xbsjOhapv5+3l1PUU3zjCM6mZnLjh38B7mt8W91T+fHq/B3MuD0eMFxaAElnRIRQECobnlxZmL1JTH8qK6VGA3HAG5bzZkArjJVGfaCvUqqny820/khrHae1jouMjMx7uXKiFAyaDDEj4I+XYdU0jz/yx/u7OZ2nZ+Uy4iN7IaP64earlMJsI6U6ZHX7WIxLcWt+C4JQfnjSWCQBjrVAo4CjeTsppfoDzwBDtNbWijrXA6u01ila6xSMFcflHpxrxcLLC4ZOhZbXwK9PwoYvPPq41nlCYp+es4Wd/9jdR71bFN4Q77co0FpZfeC07XjRdsOTeORMyUu6CoJQtnjSWKwFmiulGiul/IARwFzHDkqpjsB/MQzFCYdLh4BeSikfpZQvxua2ixuqSuPtY+RgNO0Lcx+EbXPKbSqOK4gWtY3Q2XrVXMu8Qv7qt9aa3esPlU7GuiAIZYfHjIXWOht4AFiI8aL/Vmu9TSk1USk1xNLtDSAEmK2U2qiUshqT74B9wBZgE7BJa/2zp+ZaYfHxh+FfQIMu8P3dsHuRxx5lVg8DoJlDXgXA9DGd+XevJqx4sq/be704xLmiX5KsJASh0uPJDW601vOB+Xnannc47u9mXA7wb0/OrdLgFwyjvjE0pL79F4z+HqK7l9njFz3ivFUUFR7EU4Na5TvmYqZz9nb315dw4LWr3fQWBKEyIBnclYGAajB6DoRHw1fDIWldqT/ihtj61AlzdS0Vp6jSqv2nXdpGfrzK6TxHNrkFoVIhxqKyEFwD/vUjBNeEL26A49tK9fYxUdVZ9XQ/Zt/T1db2y4PFW8GYCQyu2n/aST+q6dPzuZBeetImgiB4FuWu9nJlIy4uTickJJT3NDzPmUSYPhByc+COX6FG0/KekSk5uZpnf9zK12uMsq1Xtq7Nb9uP06ZeGNuOGol9oQE+DO1Qj5eva1eeUxWESxql1DqtdVxB/WRlUdkIj4ZbfwKdA58NhXNJ5T0jU7y9FBOG2NVdfrOEzTZwEDW8kJ5tqwEuCELFRoxFZSTyMvjXHEg/bxiMlBMFjykH/H1cq+y5S/ATBKFiI8aislK3PdzyLZw/Cp9fX6zyrGXBiif7OJ2bbaILglDxEWNRmWl4OYz4Ek7thi+GGZpSFYy8tTR8TPSkRIVWECo+YiwqO037wrAZcHQDfD2y1KvtlQbD4xrgRS69vDZxxfrH+az5Mqfrn/6VWD4TEwSh0IixqAq0ugau+xASl8PsMaVaba/EnDvCqPRZLPN/hE/9XqfJ6WX0PDyNK7y22rq8PK/4Si7/nEvn8GnJEBcETyPGoqrQfjgMfgt2L4A59xihteVFTjbsnG8kEE5pS/u9U9mfW5f7Mh/i1TZzoUZzpoVOZ0Azu4vqzMXMIj9Ga83lr/1Oj8lLSnP2giCY4FG5D6GM6XwXZKTA4hcMmZBr34WyrEZ75iBs+NxQyb1wDELqQPdxLA8dyK0/GBFbI1QwXPchYdMH8HrILBZhyISdTs0k3KHIUl601vy5+yS9WkTahA2ldKsglB2ysqhqdH8EejwO6z+FRc96vDwrOVmw/Sf4/AZ4tz0sfwvqxMCIr+DRbdDvObwj7Bnd2bkaGnSGKx4ifOcsenttBMDLYgDeWLiTtYmuciHfJhxmzIy1/G/FAVvbrLXuczTOpWZx5KxRZCk9K4f0LPOVVnpWDsM+/IstSeeK/t0F4RJCjEVVpO+zEP9v+Pt9+HOyZ56RvA9+ewHebg3f3gond0Hv8fDIFiOkt+VgQ2YdCPa3L2BrhFhWD32ehshWTPL9mDBSyMzOZeuRc0xdso+bpv3t9KifNx3lye+3AM77G75e9v98z6U579O0n7iIbpP+YOPhs7R87lfaTVhITq7mj53HcVQt2Jx0joSDZ5jwc+nKpwhCVUPcUFURpWDgJMhMgaWvgn8IdL2/5PfNzoAdP8O6mcZmuvKGywZB7G3QrB94uSbhAQT729sf7d/COPDxh+s/pOZ/+/KC7+d8vaYtM02ioo6dS2Pqkr228yHt69mOrSsHgMe+3cgnt3V2GX/d1JUAZOVopv25jzcW7qJ9g+r8ZKkOuO9kCiAy6oJQEGIsqipeXnDte4bBWPg0+IVAp9uKd6+Tu2Ddp7Dpa0g7DdUbQd/noMMtEFa3wOGOK4sAXweDUq8jM71v5C41mw3bfwHa2i7l5GpSM7Pp+tofTvey5mTMTjjsVBt893HjpZ+WmcOCrcdM5/HGwl0AbDpsL7701A/GiuX4+QzTMYIgGIixqMp4+8ANn0DmRfj5YWOF0fbGwo3NSoNtPxp7H4f+Bi9fw7XU6TZo3NswRoXE0Vjkxaf3/7Ft8RqeyPqQX5jEWUIB2Hb0HL9u/celf5pl7+GJ7zY7tR86nUpOrqbV878Wel5VRURTEMoCMRZVHR8/uPlz+HIY/DAWfIPhsoHu+/+z1TAQm76BjHMQ0RSunAjtR0FI4WtxOxLka+6eArg2thG3LLiXX7ye5SXfGTyY9RAAQ95fadq/fnX32lJnUwsXftukZjAA//689OuCCEJVxaMb3EqpgUqpXUqpvUqp8SbXxymltiulNiulfldKNXK41lAptUgptcPSJ9qTc63S+AXByFlQp52xGX3AOYOajBRY/xl83BemdTNcTi0GwG2/wIProNvDxTYUAD7e7v8z8/f1ZqduyDtZN3Ct9yqu9lrlti8YbqaVe0+ZXruQnm3a7o5FFiVcgGLUeBKES4pCGQulVFOllL/luLdS6iGlVPUCxngDU4FBQGtgpFKqdZ5uG4A4rXUMRt1tx9Cdz4A3tNatgHigYkqrVhYCwmD0DxDRBL4aAYfXGhIhPz8Cb7WEuQ8a7qqrXoPHdsKNn0DjHqWWpxEe5Ms9vVxrb4RYXFTTcq5lU24TXvKdQQ3ch7FuPHyWWz5ZbTtvWz/Mdtz7zaWFmsv+Uxc5ecF5jyJXwwdL97oZIQhCoYofKaU2AnFANLAQmAtcprV2W1hZKdUVmKC1vspy/hSA1vo1N/07Au9rrbtZjMpHWutCl2q7ZIoflZQL/xjFk84eMmpi+ARCm+uh0xhoEF+2SXwWosfPA6CZSmKe3zMsye3APVmPAAXP5a2b2vPY7E2lNpfESYOZt/kYGs01MfUKHiAIlZzSLn6Uq7XOBq4HpmitHwUKCoOpDxx2OE+ytLnjTmCB5bgFcFYp9YNSaoNS6g3LSsUJpdRYpVSCUirh5MmThfwqlzihdYziSa2uhavfNFYR138IDbuUi6FwZK+O4q3sYQz0XstQL/M9i7wMaFPbtP22ro1M2wvD/V/hDkaMAAAgAElEQVSt54GvNhR7vCBURQprLLKUUiOB24BfLG2+BYwxe/OYLmOUUqMxVi5vWJp8gB7A40BnoAkwxuVmWn+ktY7TWsdFRhbfp37JEd4Ibv4U4u+GwHy9iWXCXQ41u9fWHUVSSDte9P2UWjjX6HjtBufyqyvH9yU0wJfIUH+n9rduas+EIW1Mn7VyfN9Cz8sappuelcO51CyycnJZd9A1u1wQLgUKayxuB7oCr2itDyilGgNfFDAmCWjgcB4FHM3bSSnVH3gGGKK1znAYu0Frvd+yovkRiC3kXIVKhpfD7nK/1nU5O2AK/mTxmu8nOP6+CHEIwR0e18AWGXVla+fVxQ2x9VFKseDhHjSNDLa1d44Op371QJrXCnHq7+dj/2fwl8Pm+TSL9tTw//5N+4mLeGbOFm788G+nZEBBuFQoVOis1no78BCAUiocCNVaTypg2FqgucWwHAFGAKMcO1j2Kf4LDNRan8gzNlwpFam1Pgn0BWRDoorTv1Vt7uvdjHWHzjA5ezgv+H7Om4228vheY0URGmD/zzXXYa/txSFt+Gq1oRO1feJVNqHBVnXDWPRoLy5mZuPv44WPJTckyGJ0agT78fdT/TiXlkXnVxYDMMph8/zoOaM2yCaLbtS3CUa989SMokVdCUJVoLDRUEuVUmFKqQhgEzBDKfV2fmMsK4IHMDbEdwDfaq23KaUmKqWGWLq9AYQAs5VSG5VScy1jczBcUL8rpbZguLQ+Lsb3EyoBfpbQ2u7NauDlpYhrFM4vAUM4HBbLsJNTqUsy4KyJmOtw7OvtxRd3dqF/q9oE5snp8PZShAX44u/jjbdlBWPN4E6+mImfj5eLG8tKZIi5Cu4mER0ULkEKm5RXTWt9Xil1FzBDa/2CUmpzQYO01vOB+Xnannc47p/P2N+AmELOT6jEjO3VhKycXEbENwRAKcXa5wbA6ebwYTde9/2Ip4JeJMjPbgjSspx/3XdvXpPuzWsWew4Trm3NhJ+3O7V9teYQD/Zr7tL38dmbGNYpqtjPEoTKSGH3LHyUUnWBm7FvcAtCqRAW4MtTV7dy1o0CiGgMAybS03sLS/smEt84wpY8N3+LqxRISTCTJDmVkuk2AVAQLjUKaywmYriT9mmt1yqlmgB7PDctQbAQdyc06Y3v4udQZw+y6ql+AHxwS/HjHW6IdY3gDnGjXzV7XVKxnyMIVYlCGQut9WytdYzW+l7L+X6tdSEV6QShBCgFQ9435M9/vJ9aIX4kThrM1e0KVrt1x5vD2gM4ubUuZpoXR5q32VzB1pHsnNwC+whCZaewG9xRSqk5SqkTSqnjSqnvlVLitBXKhuoN4KpX4eAKWFvyOAcvL8X+V69m24tX2dryyn/kxTESy5GfNh6h2TMLOJh8scTzEoSKTGHdUDMwJD7qYWRh/2xpE4SyoeNoaD7AqM6XXPLa215eyhZiC3B9R7tr6n+3xXFvb2cdqx8txZLy8p3FTbXj2IUSz0kQKjKFNRaRWusZWutsy2cmICnTQtmhlFHMyccPfrwXcs3dRsWldpg9fLZfq9rc09PZWDSpGczicT3p36oW4UF28YLle4wNcKmNIVR1CmssTimlRiulvC2f0WAJfheEsiKsLgx6Aw6vhr+nluqtVR5dLMdSsNbrzWqF0iQyhLSsHJbtPmkTQATnvA9BqIoU1ljcgRE2+w9wDBiGIQEiCGVLzM3Q8hr442Wj3Gspcn+fpkwbbURZ+Xh7mda4CPD1Jj0rl1unr3Fq1+ayZ4JQZShsNNQhrfUQrXWk1rqW1vo64AYPz00QXFEKrnkH/IJhzj2QU3rSG09c1ZKBbe1RVvtfGwxA3WoBtra8GeJWcnI1W4+cI3r8PCYt2FlqcxKEikJJKuWNK7VZCEJRCKkFg9+Co+th5RSPPipx0mD+tuR2AAT6Gv9kQvPkZWRk5zLzr0QApv1Z8g14QaholMRYSCFKofxoe4NRtGnpJKNueBkRaMnNuJBHTDA1I5vle6SmilB1KYmxECetUL5c/ZZRj+PHeyAnq0we6SJJYmHCz9s5ft7I1WjiIIsuCFWFfI2FUuqCUuq8yecCRs6FIJQfwTXgminwzxZY9maZPNLdnoUj+09Kgp5Q9cjXWGitQ7XWYSafUK11YRVrBcFztLoGYobD8jfh6EaPPy7Qz9VY+Pt4EeBbkkW6IFR85L9wofIz6HUIjjSio7Lzl+0oKVZXkyMt64SSniX6UELVRoyFUPkJDDeyu0/uMDa8PYhjpneTyGAe6NOMzBzZvhOqPh41FkqpgUqpXUqpvUqp8SbXxymltiulNiulfldKNcpzPUwpdUQp9b4n5ylUAVoMMPSjVk6BJM9V4O3R3K5y8+KQNjx+1WXsOHbepV9mtqw0hKqFx4yFUsobmAoMAloDI5VSrfN02wDEaa1jgO+AyXmuvwT86ak5ClWMq16F0HqGOyorzeOPC/Iztu3ioyNsbSMt1f5EhVaoanhyZREP7LXUvsgEZgFDHTtorZdorVMtp6sAm+y5UqoTUBtY5ME5ClWJgGow9H1I3mPIgXgIq4yUtR7Gte2NrG8fL8XWI0Z97ie/N686/MP6JL5cfdBjcxMET+FJY1EfOOxwnmRpc8edwAIApZQX8BbwhMdmJ1RNmvYxquv9PRUO/u3RR1mNRYil1kWO1gyOMQzH+kNnTceM+3YTz8zZyh87j3t0boJQ2njSWJhleJvuBFpUbOOANyxN9wHztdaHzfo7jBurlEpQSiWcPCnZs4KFKydC9YaGlHmm59xBVjdUsOV/tYaxPZoA0NQkMc9RxnxBKdcQFwRP40ljkQQ0cDiPAo7m7aSU6g88AwzRWlvjErsCDyilEoE3gVuVUi5hLlrrj7TWcVrruMhIKa8hWPAPges+gDMHYPGEUr9992Y1AXv1vBwHfXIvi1TtPpPEPKt2FEBEsF+pz0sQPIknE+vWAs2VUo2BI8AIYJRjB6VUR+C/wECt9Qlru9b6Foc+YzA2wV2iqQTBLdHdocu9sPpDQ9K8Sa9Su/VbN7dn74kUm/RHppsa3Nk5ufh423+PvbVot+04ItiPv/cl07VpjVKblyB4Eo+tLLTW2cADwEJgB/Ct1nqbUmqiUmqIpdsbQAgwWym1USk111PzES5B+j0PEU3hpwcgo/TKntYKDeCKpjVt592a1TTtdzHDuZpfioP44GsLdjLy41WsO3i61OYlCJ7Eo3kWWuv5WusWWuumWutXLG3Pa63nWo77a61ra607WD5DTO4xU2v9gCfnKVRR/ILgug/hfBIsetZjj6kZ4m/avsUSGQXuy65+seqQR+YkCKWNZHALVZuGXaDrA7BuJuxd7LHHfPvvrvzxmLOra/T/VtuOs93UXZ2z4YjH5iQIpYkYC6Hq0+cZiGxpuKO2/egROfP4xhE0iQwBIDzI1+X66v3ibhIqN2IshKqPbwDc8BEob5h9G7zTBn6fCGc8kxw38/Z4l7ZlDoWRrm0v6v5C5UOMhXBpULc9PLIZRn4D9TrCinfg3fbwxY2w45dSreVtDakF2H38AulZOcxcmQjAsif68NZN7UvtWW7JzYEDy2Hr98axIJQQqUkhXDp4ecNlA43P2cOw/jPY8Dl8c4uhKRX7L4i9FapFFXyvfKgWaHdDDXhnGVe1qW0Lr60W5Iufj/NvtE2Hz9K+QXW390tOySDIz8e0loYTOdlwcIXhatv5C1y0rGaiZ8ANH0NY3eJ9IUFAVhbCpUr1BtD3GXhkKwz/Emq3hj8nw5R28NVw2L2w2L/Ia4T4ExNVzXa+cJtd2sPfx/WfXEG1uzu9vJhh0/5yaoseP4+7Pl1r7L/s/R3mPgRvtYDPhsLmb4w8k5tmwpD/wJF1MK077LFv8K87eJr0LFlxCIVHVhbCpY23j1Ftr9U1cCbRWG2s/xx2/wrVGhgrjY7/KvKv8g4NqrM56ZxLu5mx+Od8utv7LNll5KpuO2rIoO/65wJ1gr3o7bWRK/esgTc3QdoZ8AuBFgOh9VBo1t8IG7bSoAv629tQX94I3R4hMeZRbvzwb0Zf3pCXr2tXpO8lXLqIsRAEK+HRRiJf76dg5zxYNwOWvGIUVLpsEMTdDk36glfBC/IAk1rdcY3CUcpVMq16oHvpj9tnrAXAjyzYvZAtn79PHa8EZvqlcl4HQvMhhoFo2hd8A03vcSaoMZcffpKvo+YQu3IK5/9aQH3uY/c/Eab9BcEMMRaCkBdvX2hznfFJ3gfrP4UNXxj7ANUbQafbjNVGSC23tzBT0XxneAeXNj8fL7LcyIWQlU5/r3Vc7b2a/l7r4Ks0BngF8VtuJ+bldGFFbjt233BdgV/nVEoGGfhxe/Johvo244nMD5jv/xRf6CcxZNgEoWDEWAhCftRoaqjY9nkGdvxsJPf9PhGWvAotB0PcHRDd02W14SgaaKVBhN01dF/vpmxKOsu+Exc5k5pp75SVZiQPbv8Jdv3KJ34XOKuD+TUnnvm58azMbUdWEf/ZTrdEYp1Pz+Iz3Yml6lXe932P+4+/APOPwoCXwMc8C71SkbwPfn8Rjm+DRldAkz7QuBcEi/5WaSDGQhAKg48/tBtmfE7tMYzGxi+Nl3pEE+g0BjrcAsGGTlRGnrKqbeqFOZ3/38CWAFz59p9kpKUYEUzbfzI21rMuQmAEtL2eW1fV5a/cNmS7+aeamZ2Ln48XGw6dISo8iMhQ15f+12sMSRGr4sghXZthmRN40mcWd675LxxeBcNmGIaxMpJ62ghOWPsx+ARAw66w7Sdj/wkFdWOgSW/DeDTsauTdCEVGudOsqWzExcXphATP1V4WBBey0mHHXEiYDof+Bm8/aHUtxN3BjKR6vPjLDmKiqrE56RzfjL2cLk0cfuFmpMCehaycO524rAT8dToE1TTGtx4K0d15c/F+3l+yN98pbHz+Sl78eTtzNhyhbf0wfnmwh0ufYR/+RcLBM6bjd43W+P9yP+RmwzVTIOamEv1JypTsDFjzMSybbAhFxt4KvZ+G0NpGGPHRDbB/KexfAofXQG6WxZhcbhiOJr2hTkyh9qCqMkqpdVrruIL6ycpCEIqLbwDE3Gx8TuwwVhubvoat3zOmZgu6drmRZlfezUWvMKoF+RovtN0LYdscw9WUnU4br3Dm0oubbnsAGl5hRGdZKMhQgKFka9WX2nrkvGmfK5rWcGss0ptchf89K+D7u+CHu+DAnzBosnM0VUVDa2MVtvgFI4KtWX+48iUj/NmKtw806Gx8ej1hGOeDfxmGY/9SYywYK7gmvezGI7xR2X+fSoIYC0EoDWq1gkGvQ78XYPuPqIQZtNz0Gmx9m2qth0JmipEPkZMBIXWMX8GthxI77Qy5eNGrZjy1HAxFYXMgDp9OK7BPWp57XdehHj9uNOqQZeTkGDknY+bB0ldh+duQtNbI0ajVqvDfv6xISoCFzxius1qtYfQP0KxfweP8Q6DFAOMDcOEf2P+n3Xhsm2O0RzSxG47GPSHQfbLkpYYYC0EoTfyCoMMo4/PPViP8dvO3Rh5E3B2Gi6lBF5vrI5d5AJy5mMXpi5nUrx5IaIAvX612li5/7YZ2PPXDFpfHjfx4ldN5SkY2z87ZwiP9WxBd0yjtmtdYNK4Zwg0d6/PDhiP833ebaR9VnUevbGGEDUd3hx/Gwkd94OrJRtSXSbhvmXPmoLF5vfV7CK4F175rzM2rgKx2d4TWgfbDjY/WcHKX3WW1+RtI+B8oL0Mapkkfo7Z7VOeqEQhQTMRYCIKnqNMWBr9luHVQ+frGk1MyGPWJIWm+5PHeTi/4W7o0pF39au6GOtH2hYUA/LjxKImTBgOQmulsLJIvZtDrskh+2HCEpbtOsnTXScNYgJGvcc9K+OFumPug8ev7mncgwHmDvsxIPwfL34JV04yXd88noNvD4B9aes9QCmq1ND6X32NkxScl2FcdK96B5W+CbxA06masOpr2MVY2FcGQlhFiLATB0+Tz63dkfEO+XnOIWWsP29r6vLkUP4dyrFe3q0vbQhoLM/7am0z96oEcOWu4rC5m5Djd34XQ2vCvObDibSNE+Oh6I1qqnmueiMfIyTL2gJa+BqnJ0H4k9H0OqtX3/LO9faFRV+PT52nDYCWuNIzHviWw6BmjX3AtS5RVb8N4hFVtNWGPGgul1EDgXcAb+ERrPSnP9XHAXUA2cBK4Q2t9UCnVAfgQCANygFe01t94cq6CUB7c1aMxX685xNxNR53arcKDoy9vSJfGRqb15BtjiAzzp05YAM1rhdDsmQUF3n/nP+dtciLbXryKP3efZFDbOnywdJ9TP2u98IPJF6kW6Ev1ID/jV3yjbvDdnfC/K2HAyxA/1rO/prU2ggB+ew5O7YboHsZzy9JQ5SWgGrS82vgAnEuyuKyWGgZky7dGe80WhtuqdhvjU6uN4e6qIqsPj4XOKqW8gd3AlUASsBYYqbXe7tCnD7Baa52qlLoX6K21Hq6UagForfUepVQ9YB3QSmt91t3zJHRWqIwkp2TQ6WX3Ffy2T7yKID/z33TR4+fle+/ESYOd+ljdUmAo3Q6dutKp/7ODW/HyvB0ufbmYDD/dZ+hltbwGhr4PgeH5PrtYHNtklL89sAxqNDMinC4bVLFftrm5cGK7YTQOLIfjW+G8Q/XDwAi78bAakFotwS+4/Oach4oQOhsP7NVa77dMaBYwFLAZC631Eof+q4DRlvbdDn2OKqVOAJGAW2MhCJWRsEDXqnqOuDMUjoyMb8h9vZvSY/ISp3ZHQxEV7qwbVd2kmp/VUACcuJBOrVBL8lpwDRg5C1Z9AL+9ANN6wLDp0CCeH9YncUXTmtSpVoJEt/NH4feXjLDjwHAY9Iahw+Wd/9+mQuDlZexN1WkLVzxotKWeNgzI8e2G8Ti+zRCnzLpoGaQgorHFgLQ19j5qt4HwxhU658OTxqI+cNjhPAnokk//OwGXdbVSKh7wA/a5jBCESo6vtxf1qgVw9Jx75Vl31AkL4J/z6bx2Q8HKsYPa1nE6NxM6dCT+ld/ZMXGgvYaGUtD1fmhwOXx3O0wfSFbvZ3lsQXM0Xs4rkcKSkQIr34W//gM6x3jZ9nis8oerBkUYUWXR3e1tublwNtEwHFYjcmK7UXgLi3fHN8gIV85rRIIqhuCjJ42F2drR1OellBoNxAG98rTXBT4HbtNau6itKaXGAmMBGjZsWNL5CkK58MZN7bnFEglVFJY+0ZvsXPs/qRljOnP7zLWmffNqFQb4FBxyumTXCa5ul0eaPaoT3LOcnJ8exHfJi8z0jWFc1r1Fm3hujiGV8sfLkHIc2twA/V8wVH+rKl5eRg5HRBMjS99KZiqc3GExINsMI7LjF4tUiYXQus5urNptjP0RH/dqxZ7Ak8YiCWjgcB4FHM3bSSnVH3gG6KW1znBoDwPmAc9qrVflHQegtf4I+AiMPYvSm7ogVHzyrg76tHSvghsR7OzSCfAr2N3x0bL9LsYiMzuXFhNWADczyrsmL/h8xnz/p+BAlJHEVhD7/oBFzxkvxah4o/BUg84Fj6uq+AVB/U7Gx4rWhhE9vs3+ObHN2MvJsYhOevkYBqN2G2MFUq+jEZHlQTxpLNYCzZVSjYEjwAhglGMHpVRH4L/AQK31CYd2P2AO8JnWerYH5ygI5c4VTc1VUX95sLtpe34s/78+/Lr1H16Zb99/eLhfc8b2dBYJzDd01sLGw65bhD9ttG7eKr7K6cf63OZM9X2X2p8OgV5PQq//Mw8VPrHDMBJ7fzNk3m+aCa2vq9ib1+WFUkYUVWgd5+z0nCxI3utsRA6tgi2zDcNbWY2F1jpbKfUAsBAjdHa61nqbUmoikKC1ngu8AYQAsy1FYQ5prYcANwM9gRpKqTGWW47RWm/01HwFobxwLIjk46VsrqX61c2LGeVHg4ggDp9JdWobGd/Qpe63WRGmvFxWO5S1iaeJbRiOt5fRP+/yfaduyLWZr7C9y2L4cxIkroAbP7bnHKScMHI11n8KfqFGhFOXf1/SmdDFxtvX2NOo1cpQP7aSdhbSTnv88R7Ns9Bazwfm52l73uG4v5txXwBfeHJuglARWfJ4b1tUk4938X5139q1EZ/9fdB2ntdQFMS7Izrw7uI97Dp+gZum/c0j/ZvzSH8jw7uaSfRWKgHo6z5ANe4J8x4z6n1f+x6c3GlkP2enQ+e7jZWH1JYofQKrl0lQQMWN0xKESxDHAkmBBUQsuaNZrVASJw2mQYSxMnFndLo1M17cy//P7r6YMrwDQzvUJy7ankdhVbO9bfoa/v35OtN7xb70G+dbDoOxS40N2W9ugT9eMooP3bfa0JkSQ1GpEbkPQaig+BRiXyE/vr/nCv7en0xYgHm+wqe3x5Odq502yq9qY4TYhjqM8bG4oP7cfdLlHk8ObMnrv+7kTGoWs9YcYmzPFnDXYlj9X2PTtrFrfQ2hciIrC0GoAHRoYHcjWOU9SkqtsACGdnCvpeTj7eUSUWV1WTlKpOfmo/Jw5Kx9f+RcWpZx4BsI3R8RQ1HFkJWFIFQAvrunK9aUiS/u6kJ2TtlGgn9yaxxfrj5o28h2XFks2n4cM1mgTo3CGdyuHl+sMuTU61Yr+oa8UHkQYyEIFQBHl5OvtxfF3K4oNv1b16Z/69q282A/5wnkFToEeOHa1rSrXw0/by8yc3JNDYpQdRA3lCAILkSEOGcHPzxrI0EOBmTRoz2JiaqOUopNLxjV51IyClfdT6iciLEQBMGFm+Ma8KglXNaKtYhSsJ83LWrbiw8F+Hrh7aVIycgq0zkKZYsYC0EQXPD19uLh/s1d2ttHVePXR5xlPZRSeCk45FAP/O1Fuxjx0d8kp2TYN76FSo0YC0EQ3PLydW2dzjs1inDKBbGSlaP52WFf470/9rJq/2k6vbyYzq+4r9chVB5kg1sQBLf0bB7pdO5VQFJ5elYOX6w66NSWme0iGC1UQsRYCILgloY1nFcRjpLoZlw1ZRkHk1Nd2rXWhdKjEiou4oYSBCFfoh0Mxq1dG5n2aRpplAk1MxQA//ljb+lPTChTxFgIgpAv3/y7K9fE1GXnSwNpEhli2ie+gKzzD5aWjrFIz8rhvi/XceDUxYI7C6WKGAtBEPKldlgA74+KzbcUa/uo/FVP07NKJ2lv/aEzzN/yD33eXMrnfye67bfn+AWy85YHFEqEGAtBEErM8M4NCuxzMbPkSXv+DuVgn/tpm8v1mSsP0PipeVz5zjJe/3VniZ8n2BFjIQhCiXG3eb1lwgDbcWpmdomfk1aAwZnw83asC5iPlx8o8fNKm8ocGSbGQhCEUiHA1/l1Uq9agJMgYUEv+sJwsRQMTnmx49h5Wjy7gIXb/invqRQLjxoLpdRApdQupdRepdR4k+vjlFLblVKblVK/K6UaOVy7TSm1x/K5zZPzFASh5Izt0cTpPCfPHsXFUtCOclydhAeZ1+koDy6kZxE9fh5zNiS57WOtB/Lvz9exbPdJNie51jivyHgsz0Ip5Q1MBa4EkoC1Sqm5WuvtDt02AHFa61Sl1L3AZGC4UioCeAGIwyj7u84y9oyn5isIQsl4qF9z0rNz6duyFr/vOM51HY1aGp/eEc9t09eQllXyVYGjwTmTmkVGdo5tH2Px9uMlvn9xWbb7FACPfrOJ6ztGmfaZtMC+h3Lr9DUAJE4a7PnJlRKeTMqLB/ZqrfcDKKVmAUMBm7HQWi9x6L8KGG05vgr4TWt92jL2N2Ag8LUH5ysIQgnw8fbi6atbAXB5E3sJVata7YX0ohuLzOxcLqRnUSPEn2Pn0nh9gfOmdUp6Nv4hxv3v+iyhuFMvMZk5+a+aTl7IKPEzzqdnua16WBZ40g1VHzjscJ5kaXPHncCCYo4VBKGCEmD55T87wb2Lxh2Pzd5Ep5cXo7Xm4VkbuZBhGByrZlVmAeGxP208UuRnFobcXE30+Hm8tWgXAOFBdkn3C+nOwok9Jy8pkT5WelYO0ePnETNhEd+sPVTs+5QUTxoLs/AI00BrpdRoDJfTG0UZq5Qaq5RKUEolnDzpWh9YEITyp2VdQ848xL/ojox5mw1xwpMpGU77Fdbgq53HLgDQ6rlfTcf/svlYkZ9ZGKxKutbMdGs5WoB2Exaxan+y7fzQafOsdihcdNTpi5m243cX73G5/tPGI2Wyae5JY5EEOAZfRwEu5baUUv2BZ4AhWuuMoozVWn+ktY7TWsdFRkbmvSwIQgXA19sLfx8vqhdjQ9oqRRX/yu9kZdt/L7apVw2A22eu5WxqJmlZ5m6g3zy0j3EmNdPpPO9Lf8RHqwC4/8v1+d7H0RCY8d8/93HFpD9s50fPpbv0+Xj5fr5Ze9ilvbTxpLFYCzRXSjVWSvkBI4C5jh2UUh2B/2IYihMOlxYCA5RS4UqpcGCApU0QhEpIgK836W5e6Pnh7SBzu+v4Bduxv8Mv+Q4Tf3MZd1f3xkV+VlE4k2p3NX295hAZJiuE9Kwc5m1xXtkse6KP0/naxNP5PqegXJE9xy+w9ch5U1dMaeMxY6G1zgYewHjJ7wC+1VpvU0pNVEoNsXR7AwgBZiulNiql5lrGngZewjA4a4GJ1s1uQRAqHwG+XqRnFT0hLceNym2NYD/TdoCEZ/vz7DWt3V5/Zd52osfPY91B51fK0bNpHEwunObUWYeVxVM/bOHfn69z6ZOS4bqh37BGELc5iDE++PWGfJ/jU4Am/JXvLAPg950n8u1XGnhUolxrPR+Yn6fteYfj/vmMnQ5M99zsBEEoKwJ8vUnPLr0a3bXCAkzbF4/rRc0QfwDu692Uj5btJydX21YoP244Yvu1/vQPW1n4qL3qn9XdU5hw1rOpBVf/c7eS2nDYnl/RvJa5MKOVIH/3elxljWRwC4LgcQ4mp/LTxqNMXbK30L/e3fHLg93dXmtSM9h2XD3Il+xcbdvPOH4+nUe+2Wi7Xj88sNhzcLdH0sVBfTdvOdmJQ9sAOAkytq1fLd/n7D9p/1sN62Tkb1gFElJJbMkAABE0SURBVFfuPVWEGZccMRaCIJQZbyzcRa83lhZ7/C8Pds/3Bevl4LYJtLyUrTIji/Jsdl80cRMBDH1/RYHzcLdq+PKuLky6oR0ASWfsNcm7NI5gdBfD/fT5nfF0ahROkJ83R8+mmd7HjIaWcrbWxMRbPlltu/bdPV0LfZ/iIsZCEASP8+EtsU7neX91F5bIUH+31x4f0MLp3FrVzxpxVDeP6ypvRJOVTUnnmL/lWL5yHGYb2r882B0fby+bG8y6j/Fg32bMGnu5zZD5+3jz/b1XkJqZw+oD+W/FdmtmT2603vdChvPf7vlrWhMXnX89kdJAjIUgCB7HGupqxSxfIC/Wze1H+je3tTm6cK6JqevUv3ueeuG7j6cAMO5bw/WU9wV/+qL9pZuVJ7nvvi/XM+T9leS62WA3MxbWFU/NPAatX6va+ZaUTU5xn929cq+RrzHj9s5UCzRCjy9m5JBRivs/hUWMhSAIHic82DnHYvrKA/m+JAGWWYT3HA2E9YUJ8M7wDkwbbV+x1K3mvHK4s3s0ANuOngfg/q+ccx46NbIXbPpgyT7TOaw6kGzanpHHDdXMYaO6SWSw22tm7D2Rku91gD6X1SLYstmdkpHNf363Vx48WcDfsbTwaDSUIAgC4CRVbmX38RS6hpi7lSb+vJ3pK42opUBfb366vxvJF51fir7eXgxsa19d1M7jZqruIMFx2CGLun2D6qRmZLNw23Eys3Px8/HincW7i/R9MrJzCQvwYWzPJnRoEE7HhnbDk1e/yV3m+tRRsdz/1Xq3Lrm8iX7W+6RkZDNng13GZHA75xWWpxBjIQhCuZCfCq3VUICRo9G+gfuyrZueH2Ba58IxF6PHZLtm6ae3d7Yl8rV4dgH//Vcnt/dOdSOrnp6VQ4CvNw/0bW56vTDERYcDcMKNyOA3Cc5Z2cEWY7Hn+AWOWDbGH7uyRYERVaWFuKEEQSgXUgtZDCm/2t8A1YJ8qVfdNQzW3T6BY2lWwCmhbnhcA4Z2qGc7/2OXebJbRnYu/r4le31ajdmzP24F4FByKl+vsQsFrrVsfls3tq0ri/lbjtmO7+3dtERzKApiLARBKBce+GoDT3632fRa/1a1bceO0h5FJTTA2XlyWe1QAny98PM2v+frw2IY1LaO7fyr1eYqr451NMyYOsrYS5n/UA+3fXwc5pCRncNdn63lqR+2sO6gUbZn7iZDDu+Nm2IAqG8xiN2bR3J5kwha1w1zuoenEWMhCEK5kdfVYmW3gw6UNg9IKhS/PtLT6fzpwa1QShHo5/qiHxnfEICODcOd2s1qh8/f8k++G9ODY+qSOGkwreuFFWqe51KzbNFbN374l9O1CMveizX09r3f97B4xwknpduyQIyFIAjlyvpDrgUwsx1CWfOWZy0K9fO4p8IsK43broh26WtN4qsdFsCB1662tZdGOVh3jLHM41SKe/VZd99+4+GyLcsqxkIQhDJh5fi+zLy9My9c6yzyZxYNdOx8us1VFFPf/eZ2YXj1+na245go416P9m/OZ3fEO/XbcNhutBz3O8xWFoBbV1ZRsK5wrn5vuVO7Y35HceqAeAIxFoIglAn1qwfS+7Ja3N7NWT7cUXZjz/ELRI+fh9aGdlPipME0rBFUoueO6tKQ2fd05c7ujW2CgkopWtYJdeq34ZD5L/Vebyzljplrndoigv24ubN5re2i4C7s9YOl9jyKgvI0ygoxFoIglDmO0tsJifZf9FbJbYB2pRgS2jk6gufyyJbn9fkH5hN19cfOE8zfcoycXM3WI+c4fTHTVi62JDSIMDeEby7abbnuXuywrI2IGAtBEMqcni3s0hwz/0okN1fzn9+dJUAey6P1VNpUC/Slc7R9M/t/Y+Kcrv9w3xVO5/d9uZ6mT8/nmv8YQoOzSqE6XbDJRrsjeV1dz1/Tmus71gfg9m7RJX5+UagYzjBBEC4p4htH8IdDwZ41iad56zd7FvWVrWvTqEaw2dBSQynF7HuuQGtNZk6uSyhsQfvqUSWQOLeSN/S1f6vaLN5hV8d99Epng3mHpQLg2ze3z1dvyhPIykIQhDLn9m7RfDP2ctv5B0vt2kz/urwRH98aZzbMIyilTHMm6lQzL7BkZdKNMaU6j+X/14e3h7d3ahvU1nxPo6wNBXjYWCilBiqldiml9iqlxptc76mUWq+UylZKDctzbbJSaptSaodS6j1VHn8dQRA8gr+PN12a2OW3raKB4JpIV17Urx7IeyM7ur3eIR8JkqJwTUxd+rasRYOIIBddKe8CyqqWJR4zFkopb2AqMAhoDYxUSuUtjHsIGAN8lWfsFUA3IAZoC3QGenlqroIglA9392js0mYmOlhe1HOzunjl+ral9oz3R8UyfUxn27l143rK8A6l9ozSwJMmPB7Yq7XeD6CUmgUMBbZbO2itEy3X8orDa/6/vbsPsqqu4zj+/swKLEaxC67KLKQgmIIa0iqWSroaKDr5nJQzUtqTTk5mo2mWZZOp0VhT2jhGqDSGT/lApKEjGDMOIKsCkqSuDxVKAj4hPqHrtz9+35Xj7r2c6+Dde3fv9zWzc8/53d85+/t9d/Z+z/mdc38H6oH+gIB+wAuEEPqUCZ9sBJ75QFmxJ9hVwrACc04BTBm3c8Hyj8K8sw5ic8d73c4yKq2cw1DNQPZ2gTVelsvMFgMLgbX+M9/MVnetJ+mbktokta1fv77r2yGEKjeowJDT1m4X7WnNDQP5/he635XVMLB8H+T1/eqqLlFAeZNFocG2kr63L2k0sCcwnJRgWiVN6lrPzK4xsxYza2lqaur6dgihynUdkz9+QjPHjC/pmLLHjGvuPr9TT07gVy3K2eM1wIjM+nDg+RK3PQ5YYmabzGwTcDdwQM42IYTeJnP4OHHkEK740vjcKcl72uZ3P3iMW0XXnHtUOa9ZLAPGSBoJPAdMA75S4rb/Ab4h6VLSGcrngd+UpZUhhIqZOGooZ7WOZsq4ndmtqTqmteiqdY8dObllBF8/eCQD+9dVXTLrKbJtmf83b+fSVNKHfB0wy8wukfQzoM3M5kraD7gdaATeAv5nZuP8TqrfA5NIxx5/N7Nztva7WlparK2trWx9CSGEvkjSQ2aW+8WWsiaLnhTJIoQQPrxSk0XtXaUJIYTwoUWyCCGEkCuSRQghhFyRLEIIIeSKZBFCCCFXJIsQQgi5IlmEEELI1We+ZyFpPfDvbdjFDsCGj6g5fVnEqTQRp9JEnEpTzjjtYma5k+v1mWSxrSS1lfLFlFoXcSpNxKk0EafSVEOcYhgqhBBCrkgWIYQQckWy2OKaSjegl4g4lSbiVJqIU2kqHqe4ZhFCCCFXnFmEEELIVfPJQtIRkh6X1C7p/Eq3p6dJmiVpnaRVmbIhku6V9KS/Nnq5JP3WY7VS0oTMNtO9/pOSpleiL+UkaYSkhZJWS/qnpO96ecQqQ1K9pAclrfA4XezlIyUt9T7fJKm/lw/w9XZ/f9fMvi7w8sclTalMj8pLUp2kRyTN8/XqjZOZ1ewP6aFMTwGjgP7ACmBspdvVwzGYBEwAVmXKfgmc78vnA5f78lTSI25FesztUi8fAjztr42+3Fjpvn3EcRoGTPDljwNPAGMjVt3iJGCQL/cDlnr/bwamefnVwBm+fCZwtS9PA27y5bH+/zgAGOn/p3WV7l8Z4nUO8Gdgnq9XbZxq/cxif6DdzJ42s83AjcAxFW5TjzKzRcBLXYqPAa735euBYzPlsy1ZAjRIGgZMAe41s5fM7GXgXuCI8re+55jZWjN72JdfA1YDzUSsPsD7u8lX+/mPAa3ArV7eNU6d8bsVOEySvPxGM3vbzJ4B2kn/r32GpOHAUcBMXxdVHKdaTxbNwH8z62u8rNbtZGZrIX1IAjt6ebF41VQcfQhgX9JRc8SqCx9aWQ6sIyXDp4BXzOxdr5Lt8/vx8PdfBYZSA3EiPXL6POA9Xx9KFcep1pOFCpTF7WHFFYtXzcRR0iDgL8DZZrZxa1ULlNVErMysw8zGA8NJR7l7FqrmrzUZJ0lHA+vM7KFscYGqVROnWk8Wa4ARmfXhwPMVaks1ecGHTPDXdV5eLF41EUdJ/UiJ4gYzu82LI1ZFmNkrwP2kaxYNkrbzt7J9fj8e/v5g0rBoX4/TgcAXJT1LGv5uJZ1pVG2caj1ZLAPG+B0I/UkXjuZWuE3VYC7QeZfOdODOTPmpfqfPAcCrPvQyH5gsqdHvBprsZX2Gjw//EVhtZldk3opYZUhqktTgywOBw0nXdxYCJ3q1rnHqjN+JwAJLV27nAtP8LqCRwBjgwZ7pRfmZ2QVmNtzMdiV97iwws1Oo5jhV+m6ASv+Q7lp5gjSuemGl21OB/s8B1gLvkI5STieNhd4HPOmvQ7yugKs8Vo8CLZn9nEa6uNYOfK3S/SpDnA4ind6vBJb7z9SIVbc47QM84nFaBVzk5aNIH2LtwC3AAC+v9/V2f39UZl8XevweB46sdN/KGLND2HI3VNXGKb7BHUIIIVetD0OFEEIoQSSLEEIIuSJZhBBCyBXJIoQQQq5IFiGEEHJFsgi9iqQOSct9VtOHJX0up36DpDNL2O/9kuJZ0BmSrpN0Yn7NUAsiWYTe5k0zG29mnwYuAC7Nqd9AmrGzKmW+rRtCVYtkEXqzTwAvQ5qzSdJ9frbxqKTO2YMvA3bzs5EZXvc8r7NC0mWZ/Z3kz2J4QtLBXrdO0gxJy/y5FN/y8mGSFvl+V3XWz5L0rKTLfZ8PShrt5ddJukLSQuBypWdi3OH7XyJpn0yfrvW2rpR0gpdPlrTY+3qLz1eFpMskPeZ1f+VlJ3n7VkhalNMnSbrS9/E3tkyKGAJxVBN6m4E+o2k96RkTrV7+FnCcmW2UtAOwRNJc0jMm9rI0sR2SjiRN+zzRzN6QNCSz7+3MbH9JU4GfkKaqOJ00Vcd+kgYAD0i6BzgemG9ml0iqA7Yv0t6Nvs9TSXP/HO3luwOHm1mHpN8Bj5jZsZJagdnAeODH/rv39rY3et9+5Nu+LukHwDmSrgSOA/YwM+uccgO4CJhiZs9lyor1aV/gU8DewE7AY8Cskv4qoc+LZBF6mzczH/yfBWZL2os0vcYvJE0iTfncTPrA6+pw4FozewPAzLLP8uicHPAhYFdfngzskxm7H0yaf2cZMEtpcsE7zGx5kfbOybz+OlN+i5l1+PJBwAnengWShkoa7G2d1rmBmb2sNFvpWNIHPKSHdi0GNpIS5kw/K5jnmz0AXCfp5kz/ivVpEjDH2/W8pAVF+hRqUCSL0GuZ2WI/0m4izdPUBHzGzN5Rms2zvsBmovgUzm/7awdb/jcEnGVm3Sb788R0FPAnSTPMbHahZhZZfr1LmwptV6itIj086csF2rM/cBgpwXwHaDWzb0ua6O1cLml8sT75GVXM/xMKimsWodeStAfp0bgvko6O13miOBTYxau9RnoMaqd7gNMkbe/7yA5DFTIfOMPPIJC0u6SPSdrFf98fSLPRTiiy/cmZ18VF6iwCTvH9HwJssPSsjHtIH/qd/W0ElgAHZq5/bO9tGgQMNrO7gLNJw1hI2s3MlprZRcAG0nTWBfvk7Zjm1zSGAYfmxCbUkDizCL1N5zULSEfI033c/wbgr5LaSDPC/gvAzF6U9ICkVcDdZnauH123SdoM3AX8cCu/byZpSOphpXGf9aRrHocA50p6B9gEnFpk+wGSlpIOzLqdDbifAtdKWgm8wZapqH8OXOVt7wAuNrPbJH0VmOPXGyBdw3gNuFNSvcfle/7eDEljvOw+0vOaVxbp0+2ka0CPkmZi/sdW4hJqTMw6G0KZ+FBYi5ltqHRbQthWMQwVQgghV5xZhBBCyBVnFiGEEHJFsgghhJArkkUIIYRckSxCCCHkimQRQgghVySLEEIIuf4PdUEKgTJEw50AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model is not doing that well - out of presented pairs it gets roughly 10% of examples wrong. I also did a cursory error analysis (not shown here for the sake of brevity) and the model is not doing that great at all.\n",
    "\n",
    "How can this be? Maybe the nearly absolute positional invariance through the use of global max pooling is not working that well. Maybe there is a bug somewhere? Maybe the model has not been trained for long enough or lacks capacity?\n",
    "\n",
    "If I do continue to work on this I will definitely take a closer look at each of the angles I list above. For the time being, let's try to predict on the validation set and finish off with making a submission.\n",
    "\n",
    "The predicting part is where the code gets really messy. That is good enough for now though."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'{name}-stage-2');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_whale_fns = set(df[df.Id == 'new_whale'].sample(frac=1).Image.iloc[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (\n",
    "    ImageItemList\n",
    "        .from_df(df, f'data/train-{SZ}', cols=['Image'])\n",
    "        .split_by_valid_func(lambda path: path2fn(path) in val_fns.union(new_whale_fns))\n",
    "        .label_from_func(lambda path: fn2label[path2fn(path)], classes=classes)\n",
    "        .add_test(ImageItemList.from_folder(f'data/test-{SZ}'))\n",
    "        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n",
    "        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n",
    "        .normalize(imagenet_stats)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3570"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.valid_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.93 s, sys: 1.13 s, total: 3.06 s\n",
      "Wall time: 3.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "targs = []\n",
    "feats = []\n",
    "learn.model.eval()\n",
    "for ims, ts in data.valid_dl:\n",
    "    feats.append(learn.model.process_features(learn.model.cnn(ims)).detach().cpu())\n",
    "    targs.append(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = torch.cat(feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3570, 512])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21.5 s, sys: 0 ns, total: 21.5 s\n",
      "Wall time: 21.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sims = []\n",
    "for feat in feats:\n",
    "    dists = learn.model.calculate_distance(feats, feat.unsqueeze(0).repeat(3570, 1))\n",
    "    predicted_similarity = learn.model.head(dists.cuda()).sigmoid_()\n",
    "    sims.append(predicted_similarity.squeeze().detach().cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3570"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_whale_idx = np.where(classes == 'new_whale')[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.2 s, sys: 0 ns, total: 1.2 s\n",
      "Wall time: 1.19 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "top_5s = []\n",
    "for i, sim in enumerate(sims):\n",
    "    idxs = sim.argsort(descending=True)\n",
    "    probs = sim[idxs]\n",
    "    top_5 = []\n",
    "    for j, p in zip(idxs, probs):\n",
    "        if len(top_5) == 5: break\n",
    "        if j == i: continue\n",
    "        predicted_class = data.valid_ds.y.items[j]\n",
    "        if j == predicted_class: continue\n",
    "        if predicted_class not in top_5: top_5.append(predicted_class)\n",
    "    top_5s.append(top_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25805322128851543"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# without predicting new_whale\n",
    "mapk(data.valid_ds.y.items.reshape(-1,1), np.stack(top_5s), 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.98 0.28244631185807656\n",
      "0.9822222222222222 0.2875023342670402\n",
      "0.9844444444444445 0.2908216619981326\n",
      "0.9866666666666667 0.29868347338935575\n",
      "0.9888888888888889 0.3054715219421102\n",
      "0.991111111111111 0.3146778711484594\n",
      "0.9933333333333333 0.32679271708683477\n",
      "0.9955555555555555 0.3353548085901027\n",
      "0.9977777777777778 0.34555555555555556\n",
      "1.0 0.3471428571428572\n",
      "CPU times: user 12.9 s, sys: 0 ns, total: 12.9 s\n",
      "Wall time: 12.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for thresh in np.linspace(0.98, 1, 10):\n",
    "    top_5s = []\n",
    "    for i, sim in enumerate(sims):\n",
    "        idxs = sim.argsort(descending=True)\n",
    "        probs = sim[idxs]\n",
    "        top_5 = []\n",
    "        for j, p in zip(idxs, probs):\n",
    "            if new_whale_idx not in top_5 and p < thresh and len(top_5) < 5: top_5.append(new_whale_idx)\n",
    "            if len(top_5) == 5: break\n",
    "            if j == new_whale_idx or j == i: continue\n",
    "            predicted_class = data.valid_ds.y.items[j]\n",
    "            if predicted_class not in top_5: top_5.append(predicted_class)\n",
    "        top_5s.append(top_5)\n",
    "    print(thresh, mapk(data.valid_ds.y.items.reshape(-1,1), np.stack(top_5s), 5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many reasons why the best threshold here might not carry over to what would make sense on the test set. It is some indication though of how our model is doing and a useful data point."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7960"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.test_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (\n",
    "    ImageItemList\n",
    "        .from_df(df, f'data/train-{SZ}', cols=['Image'])\n",
    "        .split_by_valid_func(lambda path: path2fn(path) in {'69823499d.jpg'}) # in newer version of the fastai library there is .no_split that could be used here\n",
    "        .label_from_func(lambda path: fn2label[path2fn(path)], classes=classes)\n",
    "        .add_test(ImageItemList.from_folder(f'data/test-{SZ}'))\n",
    "        .transform(None, size=SZ, resize_method=ResizeMethod.SQUISH)\n",
    "        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n",
    "        .normalize(imagenet_stats)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.9 s, sys: 1.79 s, total: 4.69 s\n",
      "Wall time: 5.03 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test_feats = []\n",
    "learn.model.eval()\n",
    "for ims, _ in data.test_dl:\n",
    "    test_feats.append(learn.model.process_features(learn.model.cnn(ims)).detach().cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.02 s, sys: 5.04 s, total: 14.1 s\n",
      "Wall time: 14.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_feats = []\n",
    "train_class_idxs = []\n",
    "learn.model.eval()\n",
    "for ims, t in data.train_dl:\n",
    "    train_feats.append(learn.model.process_features(learn.model.cnn(ims)).detach().cpu())\n",
    "    train_class_idxs.append(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_class_idxs = torch.cat(train_class_idxs)\n",
    "train_feats = torch.cat(train_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_feats = torch.cat(test_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 7s, sys: 2min 58s, total: 8min 6s\n",
      "Wall time: 8min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sims = []\n",
    "for feat in test_feats:\n",
    "    dists = learn.model.calculate_distance(train_feats, feat.unsqueeze(0).repeat(25344, 1))\n",
    "    predicted_similarity = learn.model.head(dists.cuda()).sigmoid_()\n",
    "    sims.append(predicted_similarity.squeeze().detach().cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.6 s, sys: 128 ms, total: 19.7 s\n",
      "Wall time: 19.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "thresh = 1\n",
    "\n",
    "top_5s = []\n",
    "for sim in sims:\n",
    "    idxs = sim.argsort(descending=True)\n",
    "    probs = sim[idxs]\n",
    "    top_5 = []\n",
    "    for i, p in zip(idxs, probs):\n",
    "        if new_whale_idx not in top_5 and p < thresh and len(top_5) < 5: top_5.append(new_whale_idx)\n",
    "        if len(top_5) == 5: break\n",
    "        if i == new_whale_idx: continue\n",
    "        predicted_class = train_class_idxs[i]\n",
    "        if predicted_class not in top_5: top_5.append(predicted_class)\n",
    "    top_5s.append(top_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_5_classes = []\n",
    "for top_5 in top_5s:\n",
    "    top_5_classes.append(' '.join([classes[t] for t in top_5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['new_whale w_9bedea6 w_448e190 w_ab629bb w_67e9aa8',\n",
       " 'new_whale w_edce644 w_dd79a10 w_99af1a9 w_ae393cd',\n",
       " 'new_whale w_4516ff1 w_d1207d9 w_02c7e9d w_8003858',\n",
       " 'new_whale w_0369a5c w_f66ec54 w_ae8982d w_d0475b2',\n",
       " 'new_whale w_8cd5c91 w_0cc0430 w_06460d7 w_e8b82f6']"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_5_classes[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.DataFrame({'Image': [path.name for path in data.test_ds.x.items]})\n",
    "sub['Id'] = top_5_classes\n",
    "sub.to_csv(f'subs/{name}.csv.gz', index=False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image</th>\n",
       "      <th>Id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47380533f.jpg</td>\n",
       "      <td>new_whale w_9bedea6 w_448e190 w_ab629bb w_67e9aa8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1d9de38ba.jpg</td>\n",
       "      <td>new_whale w_edce644 w_dd79a10 w_99af1a9 w_ae393cd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b3d4ee916.jpg</td>\n",
       "      <td>new_whale w_4516ff1 w_d1207d9 w_02c7e9d w_8003858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>460fd63ae.jpg</td>\n",
       "      <td>new_whale w_0369a5c w_f66ec54 w_ae8982d w_d0475b2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>79738ffc1.jpg</td>\n",
       "      <td>new_whale w_8cd5c91 w_0cc0430 w_06460d7 w_e8b82f6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Image                                                 Id\n",
       "0  47380533f.jpg  new_whale w_9bedea6 w_448e190 w_ab629bb w_67e9aa8\n",
       "1  1d9de38ba.jpg  new_whale w_edce644 w_dd79a10 w_99af1a9 w_ae393cd\n",
       "2  b3d4ee916.jpg  new_whale w_4516ff1 w_d1207d9 w_02c7e9d w_8003858\n",
       "3  460fd63ae.jpg  new_whale w_0369a5c w_f66ec54 w_ae8982d w_d0475b2\n",
       "4  79738ffc1.jpg  new_whale w_8cd5c91 w_0cc0430 w_06460d7 w_e8b82f6"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(f'subs/{name}.csv.gz').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(f'subs/{name}.csv.gz').Id.str.split().apply(lambda x: x[0] == 'new_whale').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████| 164k/164k [00:03<00:00, 46.1kB/s]\n",
      "Successfully submitted to Humpback Whale Identification"
     ]
    }
   ],
   "source": [
    "!kaggle competitions submit -c humpback-whale-identification -f subs/{name}.csv.gz -m \"{name}\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}