Deep_Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape([1,2,3]) = [3]\n",
      "shape([[1,2],[3,4],[5,6]]) = [3, 2]\n",
      "shape([[[1,2,3], [4,5,6]],[[7,8,9],[10,11,12]],[[15,16,17],[18,19,20]]]) = [3, 2, 3]\n"
     ]
    }
   ],
   "source": [
    "Tensor = list\n",
    "from typing import List\n",
    "\n",
    "def shape(tensor: Tensor) -> List[int]:\n",
    "    sizes: List[int] = []\n",
    "    while isinstance(tensor, list):\n",
    "        sizes.append(len(tensor))\n",
    "        tensor = tensor[0]\n",
    "    return sizes\n",
    "\n",
    "print(f\"shape([1,2,3]) = {shape([1,2,3])}\")\n",
    "print(f\"shape([[1,2],[3,4],[5,6]]) = {shape([[1,2],[3,4],[5,6]])}\")\n",
    "print(f\"shape([[[1,2,3], [4,5,6]],[[7,8,9],[10,11,12]],[[15,16,17],[18,19,20]]]) = {shape([[[1,2,3], [4,5,6]],[[7,8,9],[10,11,12]],[[15,16,17],[18,19,20]]])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is_1d([1,2,3]) = True\n",
      "is_1d([[1,2],[3,4]]) = False\n"
     ]
    }
   ],
   "source": [
    "# Check if tensor is 1D or of higher dimensional\n",
    "def is_1d(tensor: Tensor) -> bool:\n",
    "    \"\"\"If tensor[0] is a list, it's a higher-order tensor.\n",
    "    Otherwise, tensor is 1-dimensional (that is, a vector).\"\"\"\n",
    "    return not isinstance(tensor[0], list)\n",
    "\n",
    "print(f\"is_1d([1,2,3]) = {is_1d([1,2,3])}\")\n",
    "print(f\"is_1d([[1,2],[3,4]]) = {is_1d([[1,2],[3,4]])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "def tensor_sum(tensor: Tensor) -> float:\n",
    "    \"\"\"Sums up all the values in a tensor\"\"\"\n",
    "    if is_1d(tensor):\n",
    "        return sum(tensor)\n",
    "    else:\n",
    "        return sum(tensor_sum(tensor_i) for tensor_i in tensor)\n",
    "    \n",
    "print(tensor_sum([[1,2],[3,4]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor_apply(lambda x: x + 1, [1,2,3]) = [2, 3, 4]\n",
      "tensor_apply(lambda x: 2*x, [[1,2],[3,4]]) = [[2, 4], [6, 8]]\n"
     ]
    }
   ],
   "source": [
    "from typing import Callable\n",
    "def tensor_apply(f: Callable[[float], float], tensor: Tensor) -> Tensor:\n",
    "    \"\"\"Applies f element-wise\"\"\"\n",
    "    if is_1d(tensor):\n",
    "        return [f(x) for x in tensor]\n",
    "    else:\n",
    "        return [tensor_apply(f, tensor_i) for tensor_i in tensor]\n",
    "    \n",
    "print(f\"tensor_apply(lambda x: x + 1, [1,2,3]) = {tensor_apply(lambda x: x + 1, [1,2,3])}\")\n",
    "print(f\"tensor_apply(lambda x: 2*x, [[1,2],[3,4]]) = {tensor_apply(lambda x: 2*x, [[1,2],[3,4]])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zero_like([1,2,3]) = [0.0, 0.0, 0.0]\n",
      "zero_like([[1,2],[3,4]]) = [[0.0, 0.0], [0.0, 0.0]]\n"
     ]
    }
   ],
   "source": [
    "# Create 0 Tensor\n",
    "def zero_like(tensor: Tensor) -> Tensor:\n",
    "    return tensor_apply(lambda _: 0.0, tensor)\n",
    "\n",
    "print(f\"zero_like([1,2,3]) = {zero_like([1,2,3])}\")\n",
    "print(f\"zero_like([[1,2],[3,4]]) = {zero_like([[1,2],[3,4]])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor_combine(operator.add, [1,2,3], [4,5,6]) = [5, 7, 9]\n",
      "tensor_combine(operator.mul, [1,2,3], [4,5,6]) = [4, 10, 18]\n"
     ]
    }
   ],
   "source": [
    "def tensor_combine(f: Callable[[float, float], float],\n",
    "                   t1: Tensor,\n",
    "                   t2: Tensor) -> Tensor:\n",
    "    \"\"\"Applies f to corresponding elements of t1 and t2\"\"\"\n",
    "    if is_1d(t1):\n",
    "        return [f(x,y) for x,y in zip(t1,t2)]\n",
    "    else:\n",
    "        return [tensor_combine(f, tensor_i, tensor_j)\n",
    "               for tensor_i, tensor_j in zip(t1, t2)]\n",
    "    \n",
    "import operator\n",
    "\n",
    "print(\"tensor_combine(operator.add, [1,2,3], [4,5,6])\" +  \n",
    "      f\" = {tensor_combine(operator.add, [1,2,3], [4,5,6])}\")\n",
    "print(\"tensor_combine(operator.mul, [1,2,3], [4,5,6])\" +  \n",
    "      f\" = {tensor_combine(operator.mul, [1,2,3], [4,5,6])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Iterable, Tuple\n",
    "\n",
    "class Layer:\n",
    "    \"\"\"\n",
    "    Our neural networks will be composed of Layers, each of which\n",
    "    knows how to do some computation on its inputs in the \"forward\"\n",
    "    direction and propagate gradients in the \"backward\" direction.\n",
    "    \"\"\"\n",
    "    def forward(self, input):\n",
    "        \"\"\"\n",
    "        Note the lack of types. We're not going to be prescriptive\n",
    "        about what kinds of inputs layers can take and what kinds\n",
    "        of outputs they can return.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def backward(self, gradient):\n",
    "        \"\"\"\n",
    "        Similarly, we're not going to be prescriptive about what the\n",
    "        gradient looks like. It's up to you the user to make sure\n",
    "        that you're doing things sensibly.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def params(self) -> Iterable[Tensor]:\n",
    "        \"\"\"\n",
    "        Returns the parameters of this layer. The default implementation\n",
    "        returns nothing, so that if you have a layer with no parameters\n",
    "        you don't have to implement this.\n",
    "        \"\"\"\n",
    "        return ()\n",
    "\n",
    "    def grads(self) -> Iterable[Tensor]:\n",
    "        \"\"\"\n",
    "        Returns the gradients, in the same order as params()\n",
    "        \"\"\"\n",
    "        return ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Neural_networks import sigmoid\n",
    "\n",
    "class Sigmoid(Layer):\n",
    "    \n",
    "    def forward(self, input: Tensor) -> Tensor:\n",
    "        \"\"\"Applies sigmoid to each element of the input tensor,\n",
    "        and save the results to use in backpropagation.\"\"\"\n",
    "        self.sigmoids = tensor_apply(sigmoid,input)\n",
    "        return self.sigmoids\n",
    "    \n",
    "    def backward(self, gradient: Tensor) -> Tensor:\n",
    "        return tensor_combine(lambda sig, grad: sig * (1 - sig) * grad,\n",
    "                             self.sigmoids,\n",
    "                             gradient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "random_normal(2,3,4) = [[[-1.669912338256836, 0.8742237091064453, 2.031412124633789, -0.09915351867675781], [-1.1844158172607422, -1.3951969146728516, -1.2888240814208984, 0.7239246368408203], [-0.2172374725341797, 1.3999462127685547, -0.1493549346923828, -1.424551010131836]], [[-0.18418312072753906, 0.6897640228271484, 0.9515476226806641, -1.758260726928711], [-0.9138774871826172, -0.025033950805664062, -1.1354923248291016, 1.1315631866455078], [1.509866714477539, -0.46881675720214844, -0.1640605926513672, 0.14349937438964844]]]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from Probability import inverse_normal_cdf\n",
    "\n",
    "def random_uniform(*dims: int) -> Tensor:\n",
    "    if len(dims) == 1:\n",
    "        return [random.random() for _ in range(dims[0])]\n",
    "    else:\n",
    "        return [random_uniform(*dims[1:]) for _ in range(dims[0])]\n",
    "    \n",
    "def random_normal(*dims: int,\n",
    "                 mean: float = 0.0,\n",
    "                 variance: float = 1.0) -> Tensor:\n",
    "    if len(dims) == 1:\n",
    "        return [mean + variance*inverse_normal_cdf(random.random()) \n",
    "                for _ in range(dims[0])]\n",
    "    else:\n",
    "        return [random_normal(*dims[1:], mean = mean, variance = variance)\n",
    "                for _ in range(dims[0])]\n",
    "    \n",
    "print(f\"random_normal(2,3,4) = {random_normal(2,3,4)}\")\n",
    "\n",
    "def random_tensor(*dims: int, init: str = 'normal') -> Tensor:\n",
    "    if init == 'normal':\n",
    "        return random_normal(*dims)\n",
    "    elif init == 'uniform':\n",
    "        return random_uniform(*dims)\n",
    "    elif init == 'xavier':\n",
    "        variance = len(dims) / sum(dims)\n",
    "        return random_normal(*dims,variance = variance)\n",
    "    else:\n",
    "        raise ValueError(f\"unkown init: {init}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Vector_operations_on_data import dot\n",
    "\n",
    "class Linear(Layer):\n",
    "    def __init__(self, input_dim: int, output_dim: int, init: str = 'xavier') -> None:\n",
    "        \"\"\"\n",
    "        A layer of output_dim neurons, each with input_dim weights\n",
    "        (and a bias).\n",
    "        \"\"\"\n",
    "        self.input_dim = input_dim\n",
    "        self.output_dim = output_dim\n",
    "\n",
    "        # self.w[o] is the weights for the o-th neuron\n",
    "        self.w = random_tensor(output_dim, input_dim, init=init)\n",
    "\n",
    "        # self.b[o] is the bias term for the o-th neuron\n",
    "        self.b = random_tensor(output_dim, init=init)\n",
    "\n",
    "    def forward(self, input: Tensor) -> Tensor:\n",
    "        # Save the input to use in the backward pass.\n",
    "        self.input = input\n",
    "\n",
    "        # Return the vector of neuron outputs.\n",
    "        return [dot(input, self.w[o]) + self.b[o]\n",
    "                for o in range(self.output_dim)]\n",
    "\n",
    "    def backward(self, gradient: Tensor) -> Tensor:\n",
    "        # Each b[o] gets added to output[o], which means\n",
    "        # the gradient of b is the same as the output gradient.\n",
    "        self.b_grad = gradient\n",
    "\n",
    "        # Each w[o][i] multiplies input[i] and gets added to output[o].\n",
    "        # So its gradient is input[i] * gradient[o].\n",
    "        self.w_grad = [[self.input[i] * gradient[o]\n",
    "                        for i in range(self.input_dim)]\n",
    "                       for o in range(self.output_dim)]\n",
    "\n",
    "        # Each input[i] multiplies every w[o][i] and gets added to every\n",
    "        # output[o]. So its gradient is the sum of w[o][i] * gradient[o]\n",
    "        # across all the outputs.\n",
    "        return [sum(self.w[o][i] * gradient[o] for o in range(self.output_dim))\n",
    "                for i in range(self.input_dim)]\n",
    "\n",
    "    def params(self) -> Iterable[Tensor]:\n",
    "        return [self.w, self.b]\n",
    "\n",
    "    def grads(self) -> Iterable[Tensor]:\n",
    "        return [self.w_grad, self.b_grad]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "\n",
    "class Sequential(Layer):\n",
    "    \"\"\"\n",
    "    A layer consisting of a sequence of other layers.\n",
    "    It's up to you to make sure that the output of each layer\n",
    "    makes sense as the input to the next layer.\n",
    "    \"\"\"\n",
    "    def __init__(self, layers: List[Layer]) -> None:\n",
    "        self.layers = layers\n",
    "\n",
    "    def forward(self, input):\n",
    "        \"\"\"Just forward the input through the layers in order.\"\"\"\n",
    "        for layer in self.layers:\n",
    "            input = layer.forward(input)\n",
    "        return input\n",
    "\n",
    "    def backward(self, gradient):\n",
    "        \"\"\"Just backpropagate the gradient through the layers in reverse.\"\"\"\n",
    "        for layer in reversed(self.layers):\n",
    "            gradient = layer.backward(gradient)\n",
    "        return gradient\n",
    "\n",
    "    def params(self) -> Iterable[Tensor]:\n",
    "        \"\"\"Just return the params from each layer.\"\"\"\n",
    "        return (param for layer in self.layers for param in layer.params())\n",
    "\n",
    "    def grads(self) -> Iterable[Tensor]:\n",
    "        \"\"\"Just return the grads from each layer.\"\"\"\n",
    "        return (grad for layer in self.layers for grad in layer.grads())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "xor_net = Sequential([\n",
    "    Linear(input_dim = 2, output_dim = 2),\n",
    "    Sigmoid(),\n",
    "    Linear(input_dim = 2, output_dim = 1),\n",
    "    Sigmoid()\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1134\n",
      "[-18, -36, -54]\n"
     ]
    }
   ],
   "source": [
    "class Loss:\n",
    "    def loss(self, predicted: Tensor, actual: Tensor) -> float:\n",
    "        \"\"\"How good are our predictions? (Larger numbers are worse.)\"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def gradient(self, predicted: Tensor, actual: Tensor) -> Tensor:\n",
    "        \"\"\"How does the loss change as the predictions change?\"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "class SSE(Loss):\n",
    "    \"\"\"Loss function that computes the sum of the squared errors.\"\"\"\n",
    "    def loss(self, predicted: Tensor, actual: Tensor) -> float:\n",
    "        # Compute the tensor of squared differences\n",
    "        squared_errors = tensor_combine(\n",
    "            lambda predicted, actual: (predicted - actual) ** 2,\n",
    "            predicted,\n",
    "            actual)\n",
    "\n",
    "        # And just add them up\n",
    "        return tensor_sum(squared_errors)\n",
    "\n",
    "    def gradient(self, predicted: Tensor, actual: Tensor) -> Tensor:\n",
    "        return tensor_combine(\n",
    "            lambda predicted, actual: 2 * (predicted - actual),\n",
    "            predicted,\n",
    "            actual)\n",
    "\n",
    "\n",
    "sse_loss = SSE()\n",
    "print(sse_loss.loss([1, 2, 3], [10, 20, 30]))\n",
    "print(sse_loss.gradient([1, 2, 3], [10, 20, 30]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Optimizer:\n",
    "    \"\"\"\n",
    "    An optimizer updates the weights of a layer (in place) using information\n",
    "    known by either the layer or the optimizer (or by both).\n",
    "    \"\"\"\n",
    "    def step(self, layer: Layer) -> None:\n",
    "        raise NotImplementedError\n",
    "\n",
    "class GradientDescent(Optimizer):\n",
    "    def __init__(self, learning_rate: float = 0.1) -> None:\n",
    "        self.lr = learning_rate\n",
    "\n",
    "    def step(self, layer: Layer) -> None:\n",
    "        for param, grad in zip(layer.params(), layer.grads()):\n",
    "            # Update param using a gradient step\n",
    "            param[:] = tensor_combine(\n",
    "                lambda param, grad: param - grad * self.lr,\n",
    "                param,\n",
    "                grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Momentum(Optimizer):\n",
    "    def __init__(self,\n",
    "                 learning_rate: float,\n",
    "                 momentum: float = 0.9) -> None:\n",
    "        self.lr = learning_rate\n",
    "        self.mo = momentum\n",
    "        self.updates: List[Tensor] = []  # running average\n",
    "\n",
    "    def step(self, layer: Layer) -> None:\n",
    "        # If we have no previous updates, start with all zeros.\n",
    "        if not self.updates:\n",
    "            self.updates = [zero_like(grad) for grad in layer.grads()]\n",
    "\n",
    "        for update, param, grad in zip(self.updates,\n",
    "                                       layer.params(),\n",
    "                                       layer.grads()):\n",
    "            # Apply momentum\n",
    "            update[:] = tensor_combine(\n",
    "                lambda u, g: self.mo * u + (1 - self.mo) * g,\n",
    "                update,\n",
    "                grad)\n",
    "\n",
    "            # Then take a gradient step\n",
    "            param[:] = tensor_combine(\n",
    "                lambda p, u: p - self.lr * u,\n",
    "                param,\n",
    "                update)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "xor loss 0.000: 100%|█████████████████████████████████████████████████████████████| 3000/3000 [00:05<00:00, 572.63it/s]\n"
     ]
    }
   ],
   "source": [
    "xs = [[0., 0], [0., 1], [1., 0], [1., 1]]\n",
    "ys = [[0.], [1.], [1.], [0.]]\n",
    "    \n",
    "random.seed(0)\n",
    "    \n",
    "net = Sequential([\n",
    "        Linear(input_dim=2, output_dim=2),\n",
    "        Sigmoid(),\n",
    "        Linear(input_dim=2, output_dim=1)\n",
    "    ])\n",
    "    \n",
    "import tqdm\n",
    "    \n",
    "optimizer = GradientDescent(learning_rate=0.1)\n",
    "loss = SSE()\n",
    "    \n",
    "with tqdm.trange(3000) as t:\n",
    "    for epoch in t:\n",
    "        epoch_loss = 0.0\n",
    "    \n",
    "        for x, y in zip(xs, ys):\n",
    "            predicted = net.forward(x)\n",
    "            epoch_loss += loss.loss(predicted, y)\n",
    "            gradient = loss.gradient(predicted, y)\n",
    "            net.backward(gradient)\n",
    "    \n",
    "            optimizer.step(net)\n",
    "    \n",
    "        t.set_description(f\"xor loss {epoch_loss:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.6425160695224112, -1.4948117798303162], [-4.567646572029666, -3.3649176350731915]]\n",
      "[1.7673716823255192, 0.3872701437947276]\n",
      "[[3.1986204791704025, -3.5018030621426206]]\n",
      "[-0.64627659633622]\n"
     ]
    }
   ],
   "source": [
    "for param in net.params():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def tanh(x: float) -> float:\n",
    "    # If x is very large or very small, tanh is (essentially) 1 or -1\n",
    "    # We check for this because, e.g., math.exp(1000) raises an error.\n",
    "    \n",
    "    if  x < -100: return -1\n",
    "    elif x > 100: return 1\n",
    "    \n",
    "    em2x = math.exp(-2*x)\n",
    "    return (1-em2x)/(1 + em2x)\n",
    "\n",
    "class Tanh(Layer):\n",
    "    def forward(self, input: Tensor) -> Tensor:\n",
    "        # Save tanh output to use it in backpropagation\n",
    "        self.tanh = tensor_apply(tanh, input)\n",
    "        return self.tanh\n",
    "    \n",
    "    def backward(self, gradient: Tensor) -> Tensor:\n",
    "        return tensor_combine(\n",
    "        lambda tanh, grad: (1- tanh**2)*grad,\n",
    "        self.tanh, gradient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Relu(Layer):\n",
    "    def forward(self, input: Tensor) -> Tensor:\n",
    "        \n",
    "        self.input = input\n",
    "        return tensor_apply(lambda x: max(x,0), input)\n",
    "    \n",
    "    def backward(self, gradient: Tensor) -> Tensor:\n",
    "        return tensor_combine(lambda x, grad: grad if x > 0 else 0,\n",
    "                             self.input,\n",
    "                             gradient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Revisiting FizzBuzz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Neural_networks import binary_encode, fizz_buzz_encode, argmax\n",
    "\n",
    "xs  = [binary_encode(n) for n in range(101, 1024)]\n",
    "ys = [fizz_buzz_encode(n) for n in range(101, 1024)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Network\n",
    "\n",
    "NUM_HIDDEN = 25\n",
    "random.seed(0)\n",
    "\n",
    "net = Sequential([\n",
    "    Linear(input_dim = 10, output_dim = NUM_HIDDEN, init = 'uniform'),\n",
    "    Tanh(),\n",
    "    Linear(input_dim = NUM_HIDDEN, output_dim = 4, init = 'uniform'),\n",
    "    Sigmoid()\n",
    "\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "fb loss: 64.5411689216839 acc: 0.9501625135427952: 100%|███████████████████████████| 1000/1000 [08:18<00:00,  2.13it/s]\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "\n",
    "def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float:\n",
    "    num_correct = 0\n",
    "    \n",
    "    for n in range(low, hi):\n",
    "        x = binary_encode(n)\n",
    "        predicted = argmax(net.forward(x))\n",
    "        actual = argmax(fizz_buzz_encode(n))\n",
    "        if predicted == actual:\n",
    "            num_correct += 1\n",
    "        \n",
    "    return num_correct/(hi - low)\n",
    "\n",
    "optimizer = Momentum(learning_rate = 0.1, momentum = 0.9)\n",
    "loss = SSE()\n",
    "\n",
    "with tqdm.trange(1000) as t:\n",
    "    for epoch in t:\n",
    "        epoch_loss = 0.0\n",
    "        \n",
    "        for x,y in zip(xs, ys):\n",
    "            predicted = net.forward(x)\n",
    "            epoch_loss += loss.loss(predicted, y)\n",
    "            gradient = loss.gradient(predicted, y)\n",
    "            net.backward(gradient)\n",
    "            \n",
    "            optimizer.step(net)\n",
    "        \n",
    "        accuracy = fizzbuzz_accuracy(101, 1024, net)\n",
    "        t.set_description(f\"fb loss: {epoch_loss} acc: {accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_results 0.9\n"
     ]
    }
   ],
   "source": [
    "# Check results on the test-set\n",
    "print(\"test_results\", fizzbuzz_accuracy(1, 101, net))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert hot bit encoded output to probability value using Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def softmax(tensor: Tensor) -> Tensor:\n",
    "    \"\"\"Softmax along the last dimension\"\"\"\n",
    "    if is_1d(tensor):\n",
    "        # Subtract the largest value for numerical stability\n",
    "        largest = max(tensor)\n",
    "        exps = [math.exp(x - largest) for x in tensor]\n",
    "        \n",
    "        sum_of_exps = sum(exps) # this the total \"weight\"\n",
    "        return [exp_i/sum_of_exps for exp_i in exps] # Probability is of total weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftmaxCrossEntropy(Loss):\n",
    "    \"\"\"\n",
    "    This is the negative-log-likelihood of the obeserved values, \n",
    "    given the network model. So if we choose weights to minimize it,\n",
    "    our model will be maximizing the likelihood of the observed data.\"\"\"\n",
    "    \n",
    "    def loss(self, predicted: Tensor, actual: Tensor) -> float:\n",
    "        # Apply softmax to get the probabilities\n",
    "        probabilties = softmax(predicted)\n",
    "        \n",
    "        # This will be log p_i for the actual class i and 0 for the other\n",
    "        # classes. We add a tiny amount to p to avoid taking log(0)\n",
    "        likelihoods = tensor_combine(lambda p, act: math.log(p + 1e-30)*act,\n",
    "                                    probabilties, actual)\n",
    "        # And then we just sum up the negatives\n",
    "        return -tensor_sum(likelihoods)\n",
    "   \n",
    "    def gradient(self, predicted: Tensor, actual: Tensor) -> Tensor:\n",
    "        probabilities = softmax(predicted)\n",
    "    \n",
    "        return tensor_combine(lambda p, actual: p-actual,\n",
    "                         probabilities, actual)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  No final sigmoid layer now\n",
    "# Training much faster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "fb loss: 129.330 acc: 0.94: 100%|████████████████████████████████████████████████████| 100/100 [00:47<00:00,  2.15it/s]\n"
     ]
    }
   ],
   "source": [
    "    \n",
    "net = Sequential([\n",
    "        Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'),\n",
    "        Tanh(),\n",
    "        Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform')\n",
    "        # No final sigmoid layer now\n",
    "    ])\n",
    "    \n",
    "optimizer = Momentum(learning_rate=0.1, momentum=0.9)\n",
    "loss = SoftmaxCrossEntropy()\n",
    "\n",
    "with tqdm.trange(100) as t:\n",
    "        for epoch in t:\n",
    "            epoch_loss = 0.0\n",
    "    \n",
    "            for x, y in zip(xs, ys):\n",
    "                predicted = net.forward(x)\n",
    "                epoch_loss += loss.loss(predicted, y)\n",
    "                gradient = loss.gradient(predicted, y)\n",
    "                net.backward(gradient)\n",
    "    \n",
    "                optimizer.step(net)\n",
    "    \n",
    "            accuracy = fizzbuzz_accuracy(101, 1024, net)\n",
    "            t.set_description(f\"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Results 0.93\n"
     ]
    }
   ],
   "source": [
    "print(\"Test Results\", fizzbuzz_accuracy(1,101,net))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add Dropout to avoid overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dropout(Layer):\n",
    "    def __init__(self, p: float) -> None:\n",
    "        self.p = p\n",
    "        self.train = True\n",
    "\n",
    "    def forward(self, input: Tensor) -> Tensor:\n",
    "        if self.train:\n",
    "            # Create a mask of 0s and 1s shaped like the input\n",
    "            # using the specified probability.\n",
    "            self.mask = tensor_apply(\n",
    "                lambda _: 0 if random.random() < self.p else 1,\n",
    "                input)\n",
    "            # Multiply by the mask to dropout inputs.\n",
    "            return tensor_combine(operator.mul, input, self.mask)\n",
    "        else:\n",
    "            # During evaluation just scale down the outputs uniformly.\n",
    "            return tensor_apply(lambda x: x * (1 - self.p), input)\n",
    "\n",
    "    def backward(self, gradient: Tensor) -> Tensor:\n",
    "        if self.train:\n",
    "            # Only propagate the gradients where mask == 1\n",
    "            return tensor_combine(operator.mul, gradient, self.mask)\n",
    "        else:\n",
    "            raise RuntimeError(\"don't call backward when not in train mode\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m pip install mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This will download the data, change this to where you want it.\n",
    "    # (Yes, it's a 0-argument function, that's what the library expects.)\n",
    "    # (Yes, I'm assigning a lambda to a variable, like I said never to do.)\n",
    "# mnist.temporary_dir = lambda: '/tmp'\n",
    "mnist.temporary_dir = lambda: 'C:/Users/Ruchika/DSS/mnist_train/raw'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAADuCAYAAAB4fc+hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsnXdUFNf7xp/ZXUBAAbGBqFiiEhULoqhgTewV0UQsiCXGgr2B2KIYEDsCGnsDMWqsiSaKCgkGxK4oiIiKIChdWNru3t8ffHd+LKAiu3MHzXzO2XPY2XHex9mZu3fee+/7MIQQCAgICAhwh4hvAQICAgJfOkJDKyAgIMAxQkMrICAgwDFCQysgICDAMUJDKyAgIMAxQkMrICAgwDFCQysgICDAMUJDKyAgIMAxQkMrICAgwDGST9m5du3apHHjxmoHff78OVJTU5nK/ntBh6BD0PHf1QEAt27dSiWE1PlcdHxSQ9u4cWPcvHmz8qr+h7W1tVr/XtAh6BB0/Hd1AADDMC8+Jx0aTR0oFArk5eUhLy8PO3fuxJQpU/Du3TswDAM9PT1NhvooiYmJqF27NiQSCSQSCerUqdSPHyeYmZnh7du3vMXfvXs3xGIxnjx5wpuGqkBBQQFycnKwd+9eyGQyqrEzMzNRp04dSCQSPHjwgGrskqSmpiI5ORkMw0AsFqu85HI5VS1yuRyDBg2iGvNTWLduXaXbMbUb2qysLKSnp8PNzQ2zZs1C9erVUb16dcyaNQvXrl3DwoULYWhoiG+++UbdUJ+EnZ0dMjIyAAB169ZFRkYGpxfOjRs3Krwv7XNRmgULFkAkEoFhKv0U+NmzYsUK9OnTB0ZGRvjxxx/x888/U42vp6eHYcOGUY1ZmuTkZHTp0gXdunVjr4eSL1dXVxQUFFDTU1BQgIcPHyInJ4dazE9h3bp10NbWrtS/Vbuh7datG+rUqQNvb2/s2rUL/fr1g6OjI3R0dBAXF4ddu3YhMzMT586dUzdUhZg3bx4kEgkSEhLYbYcPH4a9vT3mzp3LWdzAwMAK7ffu3TvUqlWLtx72o0ePkJ+fDysrKzRp0oRa3LCwMJiZmUEsFmP06NHo0qULJBIJxGIxNQ35+fmoVasWJBIJPD09kZubi7S0NADAmjVrqOkAip/+MjMzqcYsTVhYGJ4/fw6ZTIZNmzZh8+bN2Lx5M/v5li1b8OrVK6qaEhISkJWVRTVmaWJjY5GXl6ey7e+//8ZXX32FR48eVeqYn5SjLY8DBw7gxIkT2LhxIwDgzJkz0NbWRnJysrqHrhRHjhwBIQT29vZwcHDA+PHj8fXXX2Pp0qXYv38/L5pKMn36dPTs2ZOX2E+fPmUfzby8vCCRqP31VwiFQoHBgwcjJycHUVFRaN68OeRyOfV0UlhYGHsTt2rVCqGhoTAwMKCqQUlRURF704aHh6NRo0YwNDSkqsHe3h7p6ekQiUSoXr06u33r1q14+fIlJk+eDHNzc6qaGIah2osuj9GjR+PEiRP46quv2G0LFizAnj17UL9+/UodU+0ebadOnbB+/Xp4enoCALS0tAAAJiYm6h76k0hNTcWcOXOgra2NLl264MSJE3B0dIRcLkf9+vVhbW2NX375BXPmzNF47AMHDlRov9zcXISEhGDMmDEa1/AxpFIpWrZsicTERDx48AC9e/emEtfHxwdaWlpYsGABcnNzYWFhAbFYDBsbGwBAREQEFR0A0LlzZzx58gRZWVm4f/8+b40sANSoUQOPHz8GAMyYMQPBwcHUNYhEIhgYGKg0sv/88w9evnwJANi+fTu1H2OlHgA4evQotZil2b59O6KiolQaVBcXF9y5cwcdO3as9HE1dhbnzZuHGzduICoqCm3atNHUYSvMokWLcOTIEdy5c0fll6g08fHxGo/98OFDtG/f/qP7ubu7IykpqdJ5HnUYOHAg+7eFhQW1uPPnz4eOjg7c3NygpaUFmUyGe/fuITY2Ftu2bVN7JPtTqFGjBmrUqEEtXkWoKoX3Q0NDsXDhQkRFRbHblA0fLapVq0Y1Xnl4eXlBS0tL5Wlr586daNSokVppLo2dSW1tbZw4cQI9evRgRy1pJrUPHz6MESNGwNLSErq6uuXuQwjh7MJu2bIl+3dSUhICAgLg4eGBmjVrwsDAAKampjh48CBEIhH1CyoyMhLXr1/H0KFDqeYFc3NzwTAMXr9+jcDAQAwaNAg6OjqwsbFBQUEBZs2aRU2LkhcvXmDixIkQi8XsSHuDBg3w5s0b6loAsANPtHn9+jVatmzJzsrp3bs37ty5g8LCQkyYMAEymYx9OqVNamoq9ZjBwcEQi8VITk6GnZ0dZDIZcnJy0LdvX8yfPx/Pnj1T6/ga/8mKiorC6NGjAQDe3t549+6dpkOU4c6dO2AYho37PhiG4aQHpRxBtrOzg52dHRo2bIhJkyZh8+bNWLRoEc6fP4+YmBjo6upS7yVkZmaiS5cuUCgU+Oqrr6Cvr08ttrIHYGxsjClTpuCvv/5Co0aNAAD16tWjpgMonjr04sULNGvWDKdOncK0adNgaGgIhmEgk8nw+++/U9VTFVB2PAghUCgU7N+HDh3C3bt3edN18OBB6jH79+8PhmHQunVrXLlyBYMGDYKlpSWuXr2KDRs2qH18jd/1pqamOHDgAIKDg7Fu3TpMnTpV0yHKkJ+fj/r162Pw4MHlfi6TybB582aMGjUKy5Yt03j8NWvWYMiQIWjevDmaN28OJycnxMTEID09He7u7ujRowcMDAyQnJz8wbQGF2zatAkikQgikQhLly6lGrtatWqoXbs2AMDT0xOJiYlsimXGjBlUtdy9exfNmjWDv78/7ty5g82bN6N169YAiqc5TZo0ifpcWuD/G7tLly5RjWtqaorIyEjEx8cjMTERr1+/RmJiItasWfOfm/YXFhYGbW1tmJiY4MKFC7C3t8eVK1fYXHWTJk2Qnp6uVgyNZ7qTkpLw22+/4d9//wVQvHCABj/88INKUl9JYmIizM3NMWnSJAQFBXEWv6IzGs6fP8+ZhvLw8vICUPwYz0duuPTsk7Nnz4JhGMyfP59KfIVCgS1btsDDwwMnT57E8OHD8eLFC9SoUQNt2rTBw4cP0aRJE8TExKBOnTp49+4dmjZtSm0xh7JR2717N3bs2EElphJDQ8MyMx169+6NVatWUdVRmoKCAqSmprI/0lyTlJQEqVTKvv/5559x6tQp/P3332jWrBmMjIygo6OjVgyN9mjfvn2Lrl27Yu7cuQgKCoJYLIam1hR/CEJIuSP/R48eRbt27TB79mzs3r2bcx1VGT4a2arA+fPnsXTpUpw7dw79+vXDjBkz0KZNG/j6+iI0NBQWFhbQ0dFB27ZtcfbsWYwdO5btJHypyOVyFBUVldl+6dIl3ldmtW/fHnl5efDx8aEWs2TKMT8/HwEBAfj666/RrVs31KtXT+1GFtBAjzY3NxcJCQnsYxgA9OvXDz/99BM6d+6s7uErBMMwePHiBSQSCSwtLdkljQMGDMDq1avh4uJCRUdFiI+PR9OmTTmPk5iYCFtbW2hpafHeQ+ETb29vAECvXr0AFOf/1q1bB2Nj4zL7du/eHd27d6cpDxMnTmRzkunp6eXq0iS2traIiIjAihUrVKa3LVq0iO1dh4WFVWgWDReMHDkSz58/x08//cRL/A4dOuDp06fl/hCpg1oNbV5eHjp37ozo6Gh2W3h4ONq3b091xLJkI69sZFetWoUVK1ZQ01BRrly5QmUJbk5ODhITE9G8eXPqudkPoVAoqA4IBgcH499//8WzZ8/w/fffUx0MrAhz5szhZfBn7dq1ZbbVr18fEyZMYOc480VmZiZu3LjBi47Y2FhOVpBWuqFVjihbWVlhw4YNmDlzJm/z4AwMDHgZyKjKmJmZYfDgwVWucMzly5fRv39/TJw4EcePH+c8no6ODnr16sX2aKsa7dq1o3rthoWFQSaToUGDBkhNTcXp06fRq1evcsc3+OD169ewsbGhOte7JFx9F5VuaGlX9vkSUCgU1GJVr14dp0+fphavotjZ2eG7777Dr7/+isLCwv9s7phPJBIJb0vkP4afnx/fEjhBcFgQoIqOjg4CAwMhk8mERlbgPwPzKSulGIZ5C6DSBXdLYF7Z6uiCDkGHoOM/r0MtLXzo+KSGVkBAQEDg0xFSBwICAgIcIzS0AgICAhwjuOD+x3TIZDI8fvwYlpaWvOr4EIIOQcfH+NxccFUq+Hzs1bFjR6IJ/necT4ot6FBfx+rVq4mJiQmZNGkSrzo+hqBD0JGenk60tLTImzdvyv0cwE0aOj5GRXVoLHVgYGBQxtztv0JhYSFCQ0Ph6ekJfX19jB8/Hq6urnB1dcX27dvZKkB8kZqaColEgjVr1sDb2xv79u2jroEQglmzZiE7O5t67KrItWvXMHfuXGhra0NbWxvDhw/nW5IKhYWFMDc3p+5rdvfuXejp6cHf3x+FhYVVyr1aHTRWvat69eplas8+fPiQF7cF2mzbtk1lmWtpo8b58+fD2toaFy9ehJGREW15Ko0bjSI/5aGs+Tp06FAMGDCAFw1VBalUim+++QbGxsYYM2YMTp8+Tc289EO8e/cO7969g76+Pm7dugVjY+P3FtHnCjs7O4wYMYKTcqZ8orEebVJSErp166ZSSMbMzExTh1eLR48eYeXKlRCJRKz7Q4cOHTR2/ClTpiA1NRWpqalQKBQqr7y8PDRu3Bg3btzQSAHhT2X79u1o0aIFa7dOu2iKEi0tLRw7dgx//vknL/FL8uTJE8yYMQNaWlqsw4BEIoGVlRXnsaOiolCjRg3I5XK8ffsWhw4dQnZ2Nnx8fNCiRQvO45fmxIkT7D1hYmKCjRs3wtjYGH379sXWrVs1Urmqonh7eyMpKQmBgYG8PhE/e/YMYrFY5dpYtGiRWsfU6KyDTZs24caNG+x7TVfAqQxLly5FmzZt4OHhAUNDQ7i7u6OgoAB37tzRWAxjY2P2VZqIiAjWBmPatGkai1lRXF1d0aJFiyrhlfX1119j586dVC2OlOTk5MDe3h5NmzaFjY0Ndu3ahR49eqjsQ8PmuqioSKUIkpKRI0dCKpVSd4B1dnYGAKxfvx5v377F5s2bWfcLW1tbajpkMhl8fHx4NcwEip82HB0dYWJiAldXV9y6dQsA1DbP1GhDW1hYqPJLdPXqVU0e/pPw8PCAWCxG9erV2d5lRkYG1qxZQ8XZMzo6GosXL0avXr2gpaWFs2fPUrVuzs/Px/r161FQUABXV1eVz9avX4/169dT01KSgoICXnq1RkZGOHfuHN68eYO9e/dCLpcjODhYpYiIsvIbl7Rv3x73798vs71evXpISkrCli1bONdQkq+//hpAcRUxuVwOZ2dnyOVyBAQEUHXAtbe3L/PDxwfOzs64desWEhMT4eHhgXbt2qF27dpVx2GhQ4cOePjwIYoH4orp16+fpg5fYYqKirBr1y4MHDgQAwYM4K2upqWlJVt4RyQSoUGDBlTjMwyDGzdugBDCpguOHj0KhmHY/NfMmTOrRE+Xax4+fMj+fe/ePTRr1qzc/Uo6n9JGHYdVdfjmm29w+/Zt3Lt3D+PGjUNcXBx+/fVXODg4UNVx8eJFFTeWmJgYFcNTWihnCchkMqSmpuLQoUMwNzdXu0CTxnq07u7uZWrQ8lEHVVtbG7Nnz0bHjh1hbW1N9Ve5JGPGjGEfwQoKCmBlZQWRSASJRILc3FzO458/fx5nzpzBtm3b0KRJE8yZMwfjx4/HggUL8OLFCzx+/Bg9e/ak8rjMN7m5uRgxYgQyMzNVGtl9+/ZBIpGgU6dO1EfXS/P27VsAwPfffw+g+BF29+7d7HauUJqVdunSBU+fPsX48eOpN7IA8Pfff6Nu3bpo0aIFwsPD0bhxY+jr65fb++eS+Ph46OrqQkdHBw0aNMDPP/+MiIgImJqaqnVcjbVCo0aNQvPmzZGdnQ25XA57e3tNHfqT+Pbbb3HlyhVERUWVmwujxeHDh5GVlYXnz5/j6NGj2LBhAwgpdht1cnLCiRMnOE34K29YFxcX5Obmwt/fH4aGhpg2bRokEgm+//57DB8+vIxn1JeIjY0NTpw4obLt9u3brPOGq6srb/VYZTIZsrOzWdfZ8ePHsxbt6enpKkX1ueDSpUvsU+iECRN4KUKenJzM9ui///57dOnSBQCwZMkSrFy5kmq5z5s3byI+Pp41UZ0yZYpmDszFRF+FQkF8fX3J119/TTIzM8t8zsXE5/j4eCKTyYhUKiV+fn7EyMiIpKSkfFAnzQnYISEhhGEY9nXw4EFOdYjFYiIWiwkhhAwcOJCIxWISExNT5jOa5yMrK4swDMPqeB80vheRSPTe88CljsLCQpKZmUk2bNhAevfuTaytrVWuC4lEQpo3b078/f1JRkYG5+dDJBKxr4kTJ37wnHGl49GjR+y9KpVK2e3JyclEJBKV+2/A8YIF5fXx9u3bD+5XUR2c1DrYsGEDXFxc4O7uTqXH1KBBA9y+fRtisRi6urqQyWQYMGBAleqt9ejRAz///DP7nkZObtKkSUhMTMTjx48xf/58DBgwAKampsjIyFBx/aSNiYkJb7EB1aL1pXu6XOLo6AhDQ0PMmTMHgwcPxq5duxAZGYm2bdsCKB7ALCoqYqefcTnn+tq1axCLxdi9ezd7LfC1OIAQwt6rJeftJiYm8noP29vbayz1qLZnWGRkpMpoYU5ODjw9PQGAmqPmmjVrMGrUKABgLaRfvHhBdQ6gkpycHBw5cgTTp08v89nt27cBFFe4p2FcuX//fgwbNgxZWVnYsmULCCGoV68e7wNgfNoOFRUV4dKlSwCK0yo0F08EBQUhISFBZWB048aNiIuLg6mpKdVC6Mo8rJOTEzvd7n2DhFzzvhTali1bOPHv+hhv3ryBiYkJFi5cqLkfO3W63HFxcQQAAUDGjx9PDA0NCQDCMAzp0KHDe7vbX/LabeUjYGFhISGk+FHR39+fGBkZEYZhCIAyj85c6Lhz5w4ZNWoU+3gcFBTE+yO7TCYjDMOQEydO8KbjY+kCLnW0a9eOLFq0iDg7O5PLly+Tjh07kv79+5OEhATq50MkEhEHBwdy/fp1IhKJiKurK/XzURI9PT1y5swZkpCQQIKDg4mlpSXJzs5+7/7gKHWwbNmyCl8fn6JDrR5t06ZNcebMGQwfPhxHjhxht9eqVYvtvf1X+fbbb1GnTh28ffsWf//9N7vd0NCQygqg9u3bUzE//BSU6ZJz587xMrKtHMFXPqrTRrmOXy6XIzAwEM7Ozvjll1940QIAdevWhZ2dHQCwT6F8ERAQAEdHR+Tn58PQ0BBr1qzh5clr27ZtnEzzUzsBMWzYMBQ37AJA8aKN8PBwvHjxAr/++iu6deuGbt26YfDgwexF/V+mX79+vDSyycnJWLp0KZYtW4bZs2dTj6+Ez9x4Sfbv3w8XFxcEBARgzJgxfMvBiBEjqEx7fB9FRUVYt24dnJyc4O/vr/Hj8zPJ9AtGIpHAzs4OdnZ2GDduHN9yqhwXL17kJe6RI0cQEBCA169ffzEVodTByckJTk5OfMuoMvzxxx9Yu3YtZ0ugBYcFgf8EixYtgkwmExpZgXIZPnw45HI5ZwucBBdcQYegQ9DxuelQS4vggisgICDwBSKkDgQEBAQ4RmhoBQQEBDhGcMEVdAg6BB2flQ7g83PB/aSGtnHjxrh582blVf0PZWm2yiLoEHQIOv67OgCAYZhKD2bxoUNIHQhQx8XFhRd/LAEBvqhUQ1tQUFAhPzCJRIJdu3ZVJoTGCA4OhpaWFsRiMY4fPw6FQsGrnqrAvn37IBLx8xubnZ2NXbt2ITY2lrqnXFFREZycnKCvr88aEorFYrRq1Qpubm7/yWsjMjISdnZ2EIlE7CsyMhKRkZGcxx09ejQYhmHjTp06FVlZWUhISEBhYSGn8WlTqbvtl19+wcmTJyu078yZMysTQiMEBwdj586d7PuxY8eWsUSnQVZWFjw8PCASiXhd/qlk9erVYBiGF++u3NxctoGl6X5x+vRp/PDDDwgICEB+fr7KZzExMfD29sauXbvw+vVrappKY25ujuTkZM7jvH37FosXL8bixYvRpUsX/Pvvv2AYhn116dIFXbp0QWpqKmca+vbti99++w0ikYiNu3//fvTs2RM9e/bEt99+i379+rHGpp87le7WjB8/HrVr137v53z9IkVGRmLChAkQi8UYMGAATp8+jb/++gt//fUXdS1yuRx6enrYv38/Fi1ahBcvXnCyjvpTiI2NZb2Z+vfvTz1+/fr12b9pWkr//vvvOHz4MHx8fJCXlwe5XM6+lMyaNUvFxZkrbty4gaCgIJVtz549w5IlS6jU6t2/fz82b96MzZs3IyAg4L1FdurVq8e6HWia93V4Hjx4gBcvXiAsLAzBwcFo3rw5JkyYUObHkRYKhQJpaWlIS0tT6ziV6lIoTdMyMzORn5+PatWqldnnzZs3agmrLMoLo+RjoHJRBu1HwwsXLuDw4cNsEZXg4GC1vYfUITMzk7XmGD16NPX4Fy5coB5Tia+vL37++ecPLsGtUaMG6taty7mWPXv2lEmpnTp1Co6OjpzHBooLwv/www8AgJo1a2LgwIFQKBQYOXIkHj9+rOJTFh8fz4mGdu3aAQCOHTsG4P+tl+7du1dm38DAQHh7e1O7dwoKCjBlyhQUFhaisLAQZ8+eBaBe+1GpHm3//v3Rpk0bAMXlAPPy8iotQFNERkZCLBZDJBLBwcEBly9fZnM/ffr0AQCqeckOHTrgp59+UqlUNXnyZMyZM4eahtIYGxsjLCwMmzZtKtOj4oLCwkJcunQJy5cvB8MwGDRokNo9g8qio6Oj0shGR0fD2dlZxeni1atX6Nq1K6c63r17V27abdWqVdQKf9epUwc1a9ZEzZo1UVhYCEIIRo0ahYyMDJVG9tq1a0hJSeFEw+3bt3H79m00b94czZs3Z98rnzKkUimmT58OAwMDAMU/DlxjZWUFhmGgq6sLXV1d7N+/n/UrU3cFbaWTZP369cPDhw8RHh6O9PR0mJmZAQB7ER06dEgtYZ+K8jHY3t4e27dvx8OHD2FnZ8ca8Cktt2lx//59PH78mH0fGhoKAwMDzJo1i6oOJZ6eniCEgGEY9pxwTWZmJms57+rqikWLFvE2CFeatm3bqqQNfH19qRg01qhRA7Vr14ZMJlPJUUulUhgbG5fZzjULFizAjh07ymyvWbMm9XumJDo6Ohg8eDAOHz4MAFixYgXnMUUiEQYMGABPT0+0a9cODMOw8S0sLNQ6dqW/UW9vbxQUFMDX1xeNGjVSS4S6JCYmIisrS+XGqVevHr755hv2vba2NtXUwc6dO9GiRQsUFBTA3t4ef/75J6RSKS/2Oi9evGB7ladOnaJ2I9etW7dMTyAvLw9mZmZsnpgmcrkcHTp0QFRUFOsvB4C1cqHFvXv3oK2tDUII5s+fj9jYWBBC8Ouvv1KvDRsTE1Pudi6KX38Kubm5GDp0KADg/PnzsLW15Txmybm1hw8fxqxZs5CTk4O3b9+iVq1aah1bre7F4sWLVUYr3/fikuTkZHz77bcf7SkVFhZS6009fvwYQ4YMAQC0bt0aFy9eRKtWrXhpZAFg9+7dIITAysoKvXv35kWDEl1dXVhaWvISe9++fYiKigJQbMqYk5NDvZEFgGrVquHatWuYOnUqbt68iZiYGDAMg++++466lmPHjiE0NBShoaEqqQs+fgiB4jzon3/+yXoA6ujoYODAgdR1TJ06FTk5ObCwsFC7kQU4KPzdpk0bjBkzBkZGRlQeUV1cXPD06dOP7lfSToZrjIyMMHjwYGRkZCAzMxOEEKxcuZJa/JIcO3YM69evB8MwuHr1Ku/GjHyRlJSELVu2ACg2Jhw2bBivenr06MGaml67dg19+vThJa1ibGzM9hbfvXuH1atXs7Y2cXFx1A0bL126pGLqykcje/fuXRQVFaFp06YaazfUamjr1KmDoUOHok2bNli7dq3KZ69evWIb2vfNTNAkH5q+1bZtW0RFRSEhIYGKfbGpqSlu374NQgiqVauG58+fw9zcnPO4pUlPT8fYsWMBqFps88GLF8UrFY2NjbFq1So8e/YMUqkURUVFePr0KTp27MhZ7Ddv3qBhw4YAihdM6Ovrq3yekpICd3d3xMbGAiie/tS0aVPO9JREJpPhp59+ouKK/DEkEgkWLVrENrSaqgdQEWbOnIkbN27gzp07AIrHXP744w9q8ZUYGRkhOzsbKSkpGi0Sr1ZDq6Ojg1OnTn10Py5vckIIFAoFvv322zJxCgsLMXPmTERFRcHCwkJlDicNnjx5gk2bNvHSyALAy5cveYlbkjdv3mDv3r1YtmwZAKB27drsRPhmzZqhoKAArq6unDW0iYmJKpbiycnJaNasGfz9/dnrZd68eSr/hlYjCxR3QkJDQ7Fx40ZqMd9HWlqayrS/kjMyuEZpUlmjRg0sX76c2oBtSbKzs5GdnQ1TU9MPrhGoDJyNijRo0IAd9DA0NIRUKuVk+srJkyfx5MkTtG7dmp3eBRTnekQiEezt7Xnpzb18+RKWlpa8LiVU5rnCwsJ4iR8UFMTODY2OjsZXX31F9eYFADMzM9y5c4fNj3+oxsLOnTupzKMtyYYNGwAA8+fPpxKv9PkfNGgQXr16hfv376tsX7hwIRU9gGpH7N27d1i6dCmePHnCPnkozxGXg7hPnjyBhYUFnj59yskPLadJodDQUNSrVw9AcYk1rmjRogW2bt2qsq1BgwaYNm0a9WlmSsaPH0/10as0ycnJyMnJgZ+fH2xsbHjRMGDAADg6OuLSpUto2bIl9UZWiUQiea/zbpMmTZCbm4vc3FxMnToVw4cPp6pt7dq1KjlJ2vzxxx9lGtk6depg8eLF1DTcvn27zLa9e/fCx8cHPj4+0NHRYQeXuaJ69erQ19fn7GmG03k+5ubm1EYvZ82axdsc1fJQKBSIjo7mLb6JiQn09fUxZMgQqktdS2JkZITAwEBeYpfm119/5VtCudSrVw8nTpygFk8ulyMgIEDFAdfPzw+dOnVChw4deBmQ69SpE+RyOV6+fInHjx/j+PHj2L9/PwBg165d6N2al+YSAAAgAElEQVS7N9th44r69etzWgdFsBvniMWLF/M+OT8uLo7X+AIfp2PHjtSn/Y0bNw7jxo2jGrMiNGrUCI0aNUL//v2xZ88evuVolKqxTOcLhPYjqMDnyfnz5/mWIEABwW5c0CHoEHR8bjrU0iLYjQsICAh8gQipAwEBAQGOEVxwBR2CDkHHZ6UD+PxccEEIqfCrY8eORBP87zifFPtz1rFt2zYCgBQUFPCq42MIOgQdn4MOQggBcPNz0iGkDjjG398fbm5uGDt2LKceTAKfJw8ePICDgwP27t3LtxQBDvliG1o3Nzc4OTmBYRgcOnSIlxViL168wPHjx5GVlYWAgADqtRaqIi9evADDMKwD7e7du/mWxCvdu3dnjSP5MA4tyb1796ClpQUtLS1MnTqVVy18kpmZyV6fEydO1Mgxv9gFC97e3gCKq6YrbTCMjY05X8pXEl9fX0ycOJFqxfyqzooVK1iLIaC4alOvXr3QvHlznpUBXl5eCAkJAUDH3yw9PV2lcS1dVYwms2fPxrlz59j3XFv6VGUWLVoEoNgs4H1Ltz+VSvVoQ0NDYWtry7b65b3e56xJE0tLS2RnZ0MulyM7OxvOzs4IDg6mEvv06dPYtGkTnJ2dqcT7EP/++y+qV69e5ju6fPkyVR1paWk4evRome3KYtw0yc7OhpeXF5o2bcoWqA8JCcGKFSuoNLLe3t6s7x7w/0WQ+OLUqVNISEhg30+fPr3cGgSaYtasWewP7vtefHgRtmrVil3+e//+fY3VLa5UV6tnz56QSCT46quvMHjwYNSrVw/t27cHUGwHsXLlSl5untIYGhpCX18f+fn5CAwMREZGBvr160elmteJEyfYGqh8EhoaijFjxuDHH3/EiBEjUFRUhL59+wIotrimyerVq1Xeu7u7Y926dVQ1KFm6dCl27twJ4P+rm3Xr1o1a/Fu3biElJQUikQhr1qyhFrc0a9euZb+X0lZPz58/h5WVFSdx8/Pz0bNnT9ZOqCQymQyXL19GTEwM267QQmntM2jQII2m+irV0BYPtpUlMzOTrUREw7XyY/zzzz8qFaNatGjBllzjmsDAQNZBk088PDyQkpKCjRs34u7duyoFZqZNm0ZNh5aWFvu38oeuPDdYGly/fh3Pnz9HVlYW67JKG2UhmePHj2PEiBG8aHj37h18fX0hEonQpk0b9OjRA+vXr8fevXsxb948fP/990hJSYGxsbHGY39o8C8/Px96enqoWbOmxuNWBAsLC5U0iru7O8zMzDBz5sxKH1NjycNbt26pdLM9PDw0dWiN0KVLF4SGhlIt1ce3PxcA1p24SZMmMDIywoMHDwAUW5/zQUkjz+TkZF40rF27Fj179sTDhw9hYmJCtdB3VSEpKQn29vZIT08HANbZAChbCP2/Qm5uLgCwReqB4sFbLy8vAMX3TGWdYjSWFOrcubPKjWNmZsbmAmm6z0ZFReGrr74CUJw6+PHHHyGXyxEWFkatkX327BmOHDmi4s+Vk5MDV1dXGBkZwd/fn4oOoNiWpbCwENHR0fj5558BFBeZVj4200bZ0AP0b+hnz56BYRg4OTnB1dUVoaGh1D2xAODGjRvs3w4ODmzBegcHB+Tn51PRsHPnTty8eRNWVlbIyMhQ+czNzQ0KhQJWVlacFOv/GGfOnAEAtjRiREQE++KStWvXolOnTmxls/nz56Np06YYOHAgli9fDplMVulja6yhjY6OZl/Xr1/HxIkTcezYMejo6GD48OFUGtvMzEy0bdsW8fHxAIADBw5QbdRK0qpVK/bviIgINtfk4uKCgwcPUtUiFouRnZ3N/lKvXbuWas9+/PjxUCgUaN26tYqNtUKhgEKhoPboHBERgcDAQNb1oapx+vRpHDt2jEosT09PiEQiuLm5oXr16uV+durUqTKfccmbN2/w+PFjLFmyBECx917NmjXRtWtX9O/fn1NrJqlUisOHD6s4NCvNCgICAtCrVy+1jq+x1IFyek5cXBy8vLxw4MABGBoawsbGBo0bN8bbt285Ld6bkJBQ5mQoe7Z8kJCQgPbt2yMiIgI9evRAy5Yt4eXlhbS0NF50bdu2DQ8ePIC2tna5AxBc8eTJE9y6dQsikUglPxweHk59lL10A7tr1y6q8ZWUtMuxtbVFy5Yt2Zzl5MmTMXr0aJUfJC5JSkqCXC4v94eXxrzvu3fvIiYmBtu3b8ft27dVevQMw6Bp06Y4ePAgTE1NOckVK1EoFEhOTmbHlhISEnD16lU0aNAAhoaG2LZtG44cOVLp42v8Sp86dSqOHj3Kus02bNgQ2tranH9pjRs3xvPnz/H48WMEBQVxGutjGBkZsS64p06dQmRkJO7fv48RI0agbt26mDBhAlU9SUlJWL9+PVasWEF9ysyDBw9YO3jlI9nKlSvRvXt3AOXbmGiS69evl7tdmZfNysriNH55lHwEDQsLQ3h4OPsUBgAjR46kpmXcuHFsI6tcsABw/70AQEFBAaysrJCYmIg///wTaWlpyMjIYGfrpKen4+bNm2jdujWnjSxQbHfUunVrfPvtt4iKikLjxo1Z/zI9PT3Exsaq1bvXeEMbGhqKyZMns9bN//77L+cGhSEhIejTpw+kUilatGiB0NBQTuN9DGNjY/z0008IDg6Gl5cXWrZsiXv37uHs2bPYvXs31cf2nJwcLFmyBD169CgzvYo2kydPhq+vLzuly9nZmfMcqa2tLQYOHMi+9/LyAsMw6N+/P+7evcvLrIPST3aPHj1S8ZejYYyoTNts3rwZ9+7dg1gsRvv27aFQKLB69WqVR2iu0NHRwbVr1zBnzhzo6+tDV1cXhoaGvEyLrFatGgYNGoSCggJ2+iNQvJCjoKAAK1euVC+AposxuLu7E5FIpPJyd3cncrmc3UeTxSmeP39OatSoQXbt2kUIIWTLli1ELBaTFStWfFQrl0UyAgMDibOzM2EYhtSrV4+4u7tT1yGXy4lYLCbVq1f/6LngSseJEyeIRCJReYlEIiKRSKjoCAsLIwMGDCAAyPTp00lcXFyFzgVX50NJTk4O8fPzY+8RAEQkEpH+/ftT0VH6O5FIJMTQ0JC0b9+el/NRkuHDh5Pipun9gKOiMp6enqRNmzZEJBKRmTNnktOnT6u0XZXVofETJZPJiFQqJXv27CGenp7E09OTKBQKlX00+YUdPHiQiEQiUrt2beLk5MReuPn5+R/VWlWqEXGl4+XLl8TIyIicOHGCNx1Xr14lRkZGKjd0u3btSFBQEPXz8anQ0LF//37i4OBAxo4dS/bv30+kUikVHaNGjVL5TqZOnUrOnTvH+/kghJCgoCDCMMwH9+GqoSWEkIyMDCISiciTJ08+qrWiOjS+CF8sFkNXVxdTpkzR9KHLxcnJCX379kWDBg1w5MgRODo6Yvny5dQN76oatra2CA8PR0ZGBm+T8gGgV69eyMjIgJaWFho1aoTTp09TeSz9XHB2duZlmfbx48epx6wow4YNY1OPfGBkZKTx1aNfRLUTU1NTKstqPxeysrLw+PFjrFq1itdGtiRFRUV8SxD4TNDV1eVlfjOXfBENrYAqhoaG7IofAQEB/hFccAUdgg5Bx+emQy0tgguugICAwBfIF+uwICAgIFBVEFxwBR2CDkHHZ6UD+PxccD+poW3cuDFu3rxZeVX/w9raWq1/L+gQdAg6/rs6AIBhmErnWPnQIaQOBAQEBDhG4w2ts7MzGIbB2bNnNX3oTyI5ORlJSUlISkoCUFzqrEePHrxqEhAoCSEEvXv3xvjx4+Hp6UmtFm1VpGvXrmAYpkqcgydPnqB27dpsPW2GYdR2KNZ4QyuVSiESidQvwqAGeXl5aNasGRo2bIhGjRohMzMTQLGVzX+VESNGQCQSQSwWw9raGlKplPOY8+bNQ0JCQpkXjdifAwzD4KuvvsLJkydx+/ZtNG7cuErMf/b394e/vz9rWskwDOulxQWJiYmIiIiArq4uoqOjOYtTEZo1a4YuXbogMzOT/b+LRCJYWlrC19cXqamplTquxhva7OxsAHT9qEpSo0YN1KhRA0uWLMGKFStgbW2NX375BZMmTcKePXuo6cjOzsbp06dVXD35aGBCQkJQv359ZGdn4/z583j37h3u3LmDbdu2cR5769atOHLkCCZOnKjysrOzo1rBrCQPHjyAkZERJBIJGjdujAYNGkAikVCzhJfL5di6dSt7E+/Zswf5+fm4ePEizM3NYWpqSkVHecTExIBhGMyaNQuzZs1it/v5+XHqHh0eHg5vb29IpVJYWVlh3759nMUqSWm34+zsbLx69QoMw6BWrVro3r07AgICAACvXr3Crl27VGoqfwoav7ouXboEACr1NWkilUphZ2eHVatWISYmBrq6uti2bRvVJbpJSUlo1qwZCgoKsHr1ashkMnh4eFAr5lwSR0dHmJmZ4cqVK5BKpbh27RqmTp2KuXPnUonv5uYGNzc3lW0JCQlo0qQJlfglyc7ORocOHQAAwcHB6NmzJwoKCqCvr09Nw4IFC+Dn54dTp06hb9++OH78OObPn4+CggLk5ORgxYoV1LSUxsLCAkBxwwqgjBkhrQaQFiXLZwKAgYEBzp49iyZNmrDthrKGMgBcvXoVtWrVqlSsL24JLsMwCAsLQ2JiIvT19bFjxw7o6OhwaoNRkilTpqB69eqQSqXsr19iYiJvZpUpKSlsgR89PT0EBARgxIgRvDT6ALBnzx78+OOPVHrUJZk7dy78/Pzw+PFj1g1ELpezT160Hln9/f1haWnJ2vc4Oztj3Lhx8PDwQLdu3dC/f38qOpQor1FllSk+0NfXR0REBFuk3cPDA2PHjq20EaI6KBQKdOnShX0yB4DffvsNQ4cOVc8RRJPlxQghbJnC1atXv3cfLsutOTs7E7FYTOrXr09MTU2JWCwmL1++pK6jJOPGjftg2TcudTAMQzp27Eju379Ppk6dSkQiEcnNzaWuQ4lIJCK9e/d+rwaudIjFYiIWi9n3r1+/Jv3792e3x8bGUtHh7e1NAJBz584RmUxGCCFk6dKlpEmTJlTPByGEREdHEwDEz8/vvbFp6JDJZIRhGMIwDJkxYwZhGIZcvHjxg1rAUZlE5fUgFovJpEmTSHx8vEZ0aDRHqxwxrFOnDm+Wxfv378f169eRnJyM169fo0uXLrxUbC/J8+fPebMeVygU8PPzw9y5c3Hr1i0A4KU3GxQUBLFYjOjoaFy5coUXDV26dMHly5fRo0cPtGjRAkFBQViwYAFWrVpFzcdt8eLFIIRALBajdu3aYBgGzs7OePbsGZX4JVGmCoKDgzkd7PoYSqdshUKBbdu2YezYsWUe62kREBDA5mUPHTqEjh07auS4Gm1olV5IgwYNYj3DaJOQkIDvvvsOpqamqF+/Pq91LYHivOCtW7d4nfFgY2ODK1eu4N69e5VO5qvLw4cPwTAMYmNj8ccff/DyvYSHh2PAgAG4fv06Xr9+DT09PTx48AA//vgjdS0DBw7EqFGjABRb2fCBMhf722+/wcLCgjfH6JJoaWlh9uzZvNWT/v777/H999/D19cXZmZmyMzMxNGjR9W24/riFix0794dr169QkREBObNm4e0tDRe9Vy5cgUFBQVYsGABrzoiIiJACFFxYKXJxIkTsXbtWowZMwZDhgyBhYUFxGIxIiIiqMTfsmUL68u1du1a6Ovr486dO7h06RIv58TV1RV79+7FyJEj4eDggHPnzlHXMHPmTPj5+bEN7qxZs3jt2SqxsbFB586dea0xPX36dISGhsLJyQnjx49HQUGBWsf7ohraxYsX49WrVwgPD4eZmRm7PScnhzdNhw8fhp2dHTsAwxfdunVDr1692AUctGnevDnc3NyQnZ0NhULB3kSlZyRwxezZs/H06VPIZDK4ubkhIiICtra2vDxpSKVSrF+/HpmZmTh58iTi4+Ph6OjIy/S/mTNnsg0uACxbtoy6hvIYOnQobz19Jebm5mwv/+7du2odi5OGlo+0wbVr17B582asXr2aXZOdk5MDQohaNsFfAm/evAHDMPDy8uJbigp8alI+YWzfvp167Js3b+Lrr79mp5U1btwYJ06cwO+//05dixIu58lWFr7TfgDw559/AgCOHTum1nE0Or1LedG6uLho8rAVYvny5WAYBiNGjMDDhw9hY2OD/Px8TJo0ibqWkpw6dUpjBSwqg3Je8dGjR9G5c2fedJREKpViyJAh2Lp1K2+awsPD0bdvX3zzzTfUYz98+BCnT59WWbTRpk0bnD9/nkp8f39/lQUJSkaOHImTJ09S0VCaVq1awdzcvMwiAq5JSEhA69atkZubizNnzuDEiRM4e/asyvSuy5cvqz2Y/cWkDqysrAAA7du3R/v27VFQUAB3d3f4+vryrAyoV68eb7Hnzp2LuLg4jB49mjcNpZk7dy5CQkJ4+UEGgMzMTOjp6fFmUKilpQUTExP2fWpqKjw9PWFubk4lfnmNrJ+fH2+NLFD8dJOSkgI/Pz+kp6cjMTGRSty6deuyCzNGjBiBI0eOsI2soaEhxo4dq5EZQxrt0fbo0QOenp68GKv5+PjAx8eHetyP0bx5c5V8MU0KCwuxZ8+eKnFegoKCcOXKFezduxceHh5qF+moLP369UNISAj++usv1KhRgxcNP/zwA+rUqcOum58yZQq8vLxQu3ZtKvGLp39WLaKiogAUj2k0bdoU2dnZ2LJlC+dxdXR04OXlhdTUVBw4cAC///47J4tGNNrQ2trawtbWVpOH/KwJDAzktSeZkpKCESNGwMnJiTcNSmxtbbFw4ULcu3cPbdq04U3HlStXAAA9e/bkTQMAvH37ltf4VZUJEyZgwoQJ1OPu2bOH01ooX9wS3KrE2LFjeY3fsGFDnDp1ilcNSho2bEjtcfBDKOd6CwjQRHDBFXQIOgQdn5sOtbQILrgCAgICXyBfzKwDAQEBgaqK4IIr6BB0CDo+Kx2A4IJbIaqKm6agQ9Ah6Pj8dACCC66AgICAQCnUbmjT09Mxe/ZsiEQi6OvrQywWQ19fX71q5Brg7NmzKn5dDMPgzZs3vGoSEFASHx/PXqNPnjzhW06VY9euXbCxscHff//NtxTWDVcd1G4NXVxckJGRAT8/P0RERODVq1dISEhAu3bt1D10pUhLS8OhQ4cwYsQIFRdPkUiEoUOH4sULTc3qEPic8fX1ha+vL5o1awaGYXDy5EkEBATAwMCA8/J8eXl5sLW1hb29PRiGQffu3dG1a1d07doVt2/f5jR2VWPr1q1lKsrJZDIsXLgQkZGR2Lx5M0/KilEoFBo5jtoNbWBgII4cOYIZM2agTZs2MDU1RXZ2Nh4+fEh1qV9+fj4WLlyIunXrlikk06BBAwDFVZP++ecfTuJLpVJcvXqVdVSVSCQQi8Uqfzdp0oTKssLyyMjIAFDsEqxuJaJPYd68edDW1mZ7BRKJRKUHl5+fr5EeQ0WxtraGWCzGypUrkZWVBS8vL4hEInz33XdwcnJC7969OdXy7NkzNGzYECkpKey2t2/f4saNG7hx4wY6deqEgwcPchb/fRgYGKhcrwYGBpzHjI+Px4IFC9j7syS5ubmcx68IypWuGzduVOs4Gn++LywshL29PerWrUu1mn9ISAi2bt1a7me7du3iPL6Liwv69u37wX0SEhKwePFieHt7q12x/VNQKBQqpSOVxoBc8/LlS/j6+kIulyMtLQ15eXk4duyYim3MypUrAQBjxozhXI/Sar1t27aIj4+Hu7s77O3t2c/19PSwZs0aTjV07tyZ/dFTMmHCBBXrloULF1K7Pnx8fCAWi9GiRQt4eHjg8ePHqFu3Lq81nKsKd+/eRUxMDIyMjDB16lS1jqWxhjYnJweHDh1CtWrVEB0dDRcXF+Tl5Wnq8B9l0KBBAIorZfn6+kKhUCAnJwcpKSno2bOn2ifqY3h6erI+adbW1pDJZJDL5ZDJZOzfyl7MsmXLqFWy9/HxgUQiwf3793Hs2DEwDEPNJqRly5Z49+4d5HI5jIyMoK2tDQcHB4hEIkilUlSvXh2bNm1CZGQk69PEJRs3bkSLFi0QHh4OQ0NDZGdnw93dnf08LS2N85RX6UZWLpfj4MGDOH/+PGJjY7Fv3z5kZGQgLCyMUx0A4O7ujmfPnkEul+PmzZtwc3NDbGwsUlJSqJVsrKp4enqiQ4cOyMrKQlpamtoFiDTW0FpZWcHZ2RkA2BKFffr00dThP8jz58/Zv5OSkjBjxgwAgK6uLmrXro2CggJkZWVxqqFevXpYv349rK2tcfPmzQ/GMzExgZGREad6gOKbeN68eWjbti0IIfjxxx/RpEkTzuMqsbOzw+XLl1FUVMRue/DgAfbu3YtWrVqxP8TKEpdcM27cODx58oQtizdt2jT2kXDgwIHQ1tbmNH5JA0Y/P78yjgpNmzbFxIkTAfx/x4FLoqKiyjwFKn3levXqxXl8pb04ALYHf/z4cSpPNx9j586dYBhGYxXeNFZUpryR04kTJ6JWrVqc+3YpH/felzoIDw+nUntULBYjPDz8vZ/b2dkBAJ4+fUrFs16Zurl//z6bczMyMsKKFSvw+++/4+7du7h9+zbat2/PSfxLly5hzpw52LhxI5sbd3Z2RlpaGhISEtCiRQvcuXOHk9jl0aJFC1y5cgVjxoxRycPeunWLs3NQknHjxgEoThVMnz79g/vSSB2cO3cO48aNw7hx4xAQEICgoCAwDIOjR49y7lIcGxvLdogAvPd+eN89zSXLly9HQkICrKyscPXqVY0ck9M5WDQKOx88eBAHDhyAiYkJunfvXu4+/fv3x4IFC0AI0dgoYmV48uQJCCFUGlkAEIlEuHHjBrZt26ayLTAwEFlZWVi5ciXntXJ9fHxw4cIFPH/+HM+fP8fevXuxadMmAMUmibTOhZKePXuqWMbUrl2bSiMLFOfKCSEfHSTesGEDleu0Z8+eOHr0KIYOHYqYmBi4ubmBEILBgwdzHtvMzAxDhw794D7GxsbU6xZnZmbCx8cHDMNgwYIFGuvRcj7ZletfZuX0LScnpw/eMMq5tHzN77169SoYhkGnTp2oxrW2tsbs2bMBFA88PXv2DHFxcYiLi8Pq1atRp06lizFVGD09PTRs2BANGzYEUDwoCIDKDV2awsJCHDlyhH0/bdo0arGV1+CUKVM+uB+t6/TKlStwd3fHsmXL8O+//7Kxue7NAsXXxO7du2Fubq7yOnPmDPv0ZWhoqNJJoMGOHTuQm5uLxYsXa7TMKaf1aIcNG8Z53mvdunUf3ScgIAC+vr4YN24cvvvuO071lIeHhwdWrVqFLVu2YM6cOdTj//vvvzhw4ECVKAA+ZMgQXLhwAW3btoWuri71+MqYKSkpcHFxwdOnT6lrmDdv3gfnyy5atIialrVr1wIA+vTpg5CQENy7d49abD09PcTHx5fZHhwcjE6dOiE+Pp51oaDF8uXLAUDjpqFq/2yuX7++3AneCoUCKSkpnE+XKTlq/D6cnJyQn58PbW1taGlpcaqnNH/++SdWrVoFhULBS5K/sLAQmzdvhoODA/XY5ZGdnQ19fX14e3tTj62c5nfkyBFqtjElUT7N3Lt3D+np6dTjf4iQkBB4eHjw6n6hROmCQZv79+8DKDap1DRqN7S//PILJBIJ/vnnHzx69AhLliyBkZERJBIJ3r17V64RHBfs3r0bHTp0gFQqhVQqRXR0NDp06ICaNWtCV1cX9erV49Sq4n0MHjwYDMNg8uTJqFu3LvX4FhYWMDAwYK2t+SQhIQFhYWHYu3fvR+ccaxpjY2PMmDEDeXl5cHR0xJw5c3D8+HG1C6V8Cj4+PnB0dAQA1KlTB2KxWKVHFxMTA2NjYwAoMyOBK+7fvw+xWIyrV6/Czc2NSswPkZubSz1dAADJycno168fgOJFNJpG7dRBWFgY7O3t0aNHDwDFxm8Mw2D06NFUbu6WLVuiQYMGePXqFTIyMtjktVIHAJw8eZLaJP334erqykvcxo0bY9myZbzELk1KSgqqVauGjh07Uo+dlZUFfX19NpW1d+9eNGrUiLod/YEDB6Crq4t9+/YBACwtLaGlpQWRSIT8/Hzk5+dDKpVynnJTsnLlSjAMg65du1KJ9zHkcnmZJbk00NPTg0RS3BxyYdqpdo/W1NQU4eHhUCgU7KiqQqGgtszTxsbmvSXP5s6di9jYWN4a2ZJTiCwsLKjH3759O86dO8eLK3F5hIaGIjIyEpqqBfqp5Obmsst9ly9fjri4OLYHSQuJRILdu3dDLpfjzZs36Nu3LzIzM5GZmQl7e3vcvn0bOjo6VFZVRkRE4MaNG4iPj6eeUnsfBgYG7CP8X3/9Rc3zzsDAAK9evYJcLkdgYKDGj/9FmDPWqVOH80Ig6rJkyRKq8fLy8uDp6cnOOKgK9OjRA5aWlti6dStvuurUqYMVK1Zg5syZVJeIl0etWrV4Nc88e/Ysevfuzc4GqSq0adOG12mYXPBFNLRVFT4bf11dXV4ewT6EtbU1b+ekqv8Q80FFZuwIaAbBBVfQIegQdHxuOtTSIrjgCggICHyBCFY2AgICAhwjNLQCAgICHCPYjQs6BB1VREdcXBwMDAzK1J/4r56PD/G52Y2z1YQq8urYsSPRBP87zifFFnQIOr50HQDIyJEjedfxPqqKDkIIAXDzc9KhkdTBhwwPP1Sf9b9AVlYW2rdvz06Ut7S05EWHVCrF5MmTIRaLsXv3bl40XLx4kV0RVZKCggIkJibyoKhq4ODgAIZh4Ofnh5MnT1KN/fr1a2zfvh3bt2/HhQsX8OjRIyQnJ1PVABQXXldW4mvatCmvc5z79Omj4v2XkJCg9pJotRvaBw8eoHPnzuXOU/znn3/Qs2dPdUNohMDAQLayPk0aNGiABw8esO/5ms/p4uKCw4cPQyQSYebMmTh79izV+D4+PnB2di7XsmbatGlVbtI8LRwcHPDbb78BAC/Xp6mpKWbPnvEowokAACAASURBVI3Zs2dj4MCBaNWqFQwNDVWKcnONl5cXLl68iOnTp4MQwpax1HQFrYoQGxuL169fQyQSsa9mzZphxYoVah1XrYa2sLAQ7dq1Q1paWrn1M+3s7NC0aVNYWlqioKBAnVBqIRKJMH78ePj7+1ONO2TIEEilUpiZmSE2Nha5ubmIjIykqqFBgwbQ0tIq46xqb2+PoKAgajp8fHzw5s0bBAcHl/ns0qVL1Fxw34dMJsOJEyewcOFCaq68MTExbCMbHR3NebwPceDAAYhEImhpaSEnJ4dKBa3s7GwwDANzc3MQQrBjxw4AQLdu3eDp6Qlzc3MwDEOtwX3z5g1at26NuLi4Mp+pu4xdrYZWWaxk/vz5H+zqR0VFcWbzXR5//PEH29VXFh5fsGABtfhAcdGS4OBgmJmZ4dq1a2jatCmqVatGvYpWSVvr0qxfv56ajvT0dDRv3rzcz5KTkzFs2DBqWkqzadMmdO3aFd999x1rnUKjrKTy/omOjkbLli05j/chfv31VwB0n7jat2+P6dOnsxXNSjJz5kw4OjqCEAI3Nzc0bdqUcz0mJiZszZbSL7XrSFc2iVyzZk3CMAy5fv36B5PF8+fPJwzDkIYNG7LbuEyqL1iwgDAMQwghJDU1ldSqVYvk5eWVuy+XOkQiERGJRKSgoOC9+3CtQyQSEYlEQiQSicrfJd///PPPnOqQyWRk5syZxMjIiCQmJpb5/Pr164RhGJKWlsb5+SCEkLy8PNK8eXP2+2EYhkilUlJYWFhmXy51ACDFt9/HoTEIxTAMYRiGhISEEJlMRvLz8znV0aRJkwr//wkhZPr06SQuLo59Dw4GwzIzM4mWlhbR0tIio0aNIhcuXCC5ubnsths3bpT5NxXVUelaB5mZmQDw0Xqe69evx9atWyGTySob6pPYsmULW/fV2toa6enp1H2psrOz2b9plbsrTUmvpVq1auHt27esnoSEBNy6dQuDBg3C1q1bOa1DGhkZiR07diAqKgr169dX+Uwmk2HVqlUYM2YMlZ6+QqGAhYUFEhISYGlpiaFDh2LMmDG8OD0oKS+dxUeuVknNmjWppE7i4+M/yZF5x44dmDFjBpte4IJvvvmG/Xvv3r2spY4SdWyo1Coq4+Tk9MHyaq9evWL/5nokMyIiAg4ODnByckKvXr1Qo0YN5Obm4uXLl0hMTERcXBxbM5drlG6nQ4cOLXPBNmzYUMUenQvevXvHlv8LCQlBt27dcP/+fTAMAwsLC2hpabE/Rlw/snfr1g3Gxsb4+uuvy3y2bNkyXL58mVqlpvT0dLx8+ZId4be1taUStzTvS7MFBwfzMjAWFBQEQgiMjY3LNC5ViZ07d3LW0EZERODu3bsQiUSYOnVquedh3759mDx5cuUCVLbLzTAMMTc3J1lZWezr1q1bxMXFhX0pH0eULyVcPArNmDGjTDwApHr16qR+/frk6NGjZf4NV49k5ubm7KOpSCQijo6O5K+//mLfc60jNDSUiEQiAkDlkbwkd+/eLaNH0zqOHDlCGIYhZ86cKVeDWCxWuS640lGScePGEW1tbdK7d+/37sOlDj8/PzZt4OfnV+bz8rZznTqwsbEhDMOQIUOGUDsfAwYM+KTUASFEZX9oOHVgZmZGxGIxWbhwIcnNzVX5TJk6eI8mblMHAPDy5UsYGRlVaF+ufym9vLwwatQoxMTEYP369VAoFLCxsQHDMGjYsCF69+7Nafz3MWnSJPj4+FBxFgWK5y2PGjWKnQWyd+9eLF68uNx9uXZaPXLkCKZOnapiK3369Gncv38f69atg0KhQOfOnTnVUJ4md3d3jTqcfgpKa6fie1QV2rNigOLpTNHR0TA0NMTSpUvZ7dnZ2Zzes05OTrh48SJnx/9UzMzMkJycjN9//x1Lly4t936dN28eO1j6qVT6TisqKkJeXh7c3d3h7u6Oa9euISYmBnl5eexLOWIHgPOJ2AYGBujTpw8AIDExES9fvsTx48fx66+/YtOmTahXrx6n8UvTr18/9OvXD3v27IGenh4yMjIAgLXL4Iru3bsjPT0dzs7OSEpKgkwmg6enp8o+WlpasLKygqenJ4qKijjVc/ToURgbG7NzEpcsWQIdHR3I5XJMnz4dYWFhnMYHgM2bN6u8X7t2LdV5oh8jJiYG/v7+mDVrFkaOHEk1bfDo0SNkZ2dj7NixsLOzAwC8ffsWd+/e5TSuo6Mjpk+f/knfw4ABAzjTc+fOHQDA48ePyyyBVlLZRhYA90twk5KSCMMw5PLly+w2Lh+FGIYhAwcOrJA2LlMHFhYWJCsrixBCSFFREfHy8iIikYjY2tpyqkM5q6D044+S7OxsdtZB6ZF2TZ+PDRs2sGmcNm3akEmTJrGfaWlpkf3795erUdM6GIZhZzxkZ2cTQ0ND8vz583Jjc6mDEEJGjhzJpgdKphHwnuW3XOlQ0q5dO8IwDHn58iW7LS0trdwRdk3ryMrKIgDI9OnT36uv5L6BgYHse2g4dWBhYUHEYjGJjY0t85kmUgca+8I+RJ8+fdhGhxDuLpxffvmFzJs3r8K6uNLx+vVrlRyt8lXexatpHcqGdu3atSoxJk2axH526tQpquejNBcuXCAMwxCZTEZFR2ZmJunevTsRiUTEy8uLnDx5skI6uTof0dHRKnna8nK1NHQQQoixsXG5eXKaOjw9PcttcLOyskhYWBgBQJo0aaLymaYb2ho1ahAARCwWl/u6cOFCudorqoOKlc2pU6eojGYePHiQ6mqn92FiYoJGjRohISEBxd9F8UizOtNDKoqxsTHS09Px008/4dixY+wId1RUFMzMzGBjY8O7I/DevXsBgNpqMENDQ1y9ehVZWVnUzRjLo2XLlux1wTcZGRnU8+SlcXV1RUhICHbu3ImdO3eW+dzT05PzdMrWrVsRERGB/fv3c3J8Kg0tjUa2qKgIMTExVWbNfHx8PC9xU1JSkJaWBhMTEzx69Igd8Lpz5w7Mzc1haGjIi66SuLq6stOYaCEWi6tEI1vVmD59Oi+DcKW5cOECr/EnT56MyZMnQyQSsR2BrVu3YtiwYTAwMFC7DftizBmLiorQunVrvmVUCWrVqsX5IJc6dOzYUTBLrCJUhUa2KrFjxw5O5up+MQ4Lenp6CAkJ4VuGgICAQBkEF1xBh6BD0PG56VBLi+CCKyAgIPAF8sWkDgQEBASqKkJDKyAgIMAxggvuf0hHSkoKXr16hY4dO/Kq42MIOgQdH+M/74I7cuRI8uDBgw/uU1XcNP9LOsLCwkiHDh3KrR5GU0dFEHQIOj4G/osuuECx22uLFi1w+vRpdO7cGW/evNHUoT9bcnJyeKt5WpoOHTpQL6yjJCwsDGKxGIsWLYKDgwNiY2N50VGS2NhYhIaGwtraGo0bN0ZoaCi12N7e3li4cCH69evHOr+am5tTi/9/7Z17XE35/v9fa++KTDWFKEYxJhkqckszQnUMGdei0Tkuh4Mk5C6X8z3kLoVc08itmTFRMhIyGAcnuSSFdlJI0U4X3dvty/r9sR9r/brKaK/PaljPx2M/7NZq1vs1u70/+7Pen/fn/XoXUqmUmGdaQ8TExEAsFtewpPqro7ENC127dmU7VOno6LCNpT9l9PT08Pr1a8hkMrRo0YJXLdOnT0dsbCwGDhxING5iYiIGDx4MiqKwc+dO0DSNGzdu4Nq1a+jevTtRLdVxdnauYXHu6OgImUzGaXe1N2/eYPPmzdi1axfb1c7ExASAuuVoSkpKvQ3SSRISEsJ5+8z3gaZpjBo1Ci9evCDWYpRLNPKKTp48GYWFheweeh8fH01ctklUVFRgx44d2LFjBztriI+PJ67j+fPn7JY+vnj79i1OnjyJefPmEWlLWJ3jx49jyJAh8PHxYd8X+fn5sLKyIqqjOuXl5TUGWYbKykrOYpqamqJdu3a4efMm0tLS2FvK169fQyKRQFtbG/PmzeMsfmPcvXsXYrEY69atg5+fX7PeWUiCq1evwsbGBtbW1tDS0oKWlhaWLl36wdfTyNe3lZUV/Pz84OjoiKioKJiZmWnish+EXC7H7t27sWTJErahCvNvREQE7OzsiGs6efIkrz5QoaGhmDhxYp2+tCQICAhAVVUV653WpJ6eGmLdunX1HtfT0+MsJuPLVp+b6sqVK3kf2IKCgniNX50HDx6801WbS/Lz8+Hj48M2p6JpmtXCWFR9CBoZaH19fTVxmSYREBAAf39/Njesq6uLmzdvwtbWFpcuXcLw4cNhamrKs0ryuLu7IyIiAqWlpaAoCk+ePMG0adNw+/ZtZGZmomPHjpxr0NHRwatXr3D8+HH22G+//cZ53PowMzNjZ7Pfffcdjh07xt6+c0lDdtXLli1j99afO3eOcx0N8csvv7DPuTTrbIygoCD8+9//BqB+zUg1iYqIiMCCBQsglUpB0zSmT5+OLl26YPr06ezE0dbW9oOv/1E0lZHL5axdS5cuXWBhYYGAgAC2yczw4cMBgNdbM76IiIiAjY0NKIqCp6cnwsLC2HMhISFYu3YtER2173LGjBlDzBmZ4dmzZ3j9+jUAdYf/H3/8ES1btsTnn3+OoqIizu1baqNSqbB9+3YAwNChQ4m7NTNUb/DD98w2OTmZTasMGzaMSMwHDx7ghx9+AKB2zu7WrRv2798PbW1t1rm5MbfvxuAk603qBWLQ1tZmbXPS09Nx4cIF9OzZE3K5HBMmTMD8+fNRWVn5TsdeLmnbti0vcZOTkzFlyhTExcWhQ4cOuHz5MrZt2walUom7d+8S03H//n1YWFggKysL169fZ4+T7BwVGhoKCwsLbN26FXK5HGFhYezA9vDhQwDqvBwpKioq4OjoCBcXF9y4cQOXL18mFrs2CxcuBAA4OTnx5qXGUH09Y+TIkURi9urVCwqFAgqFArdu3cKxY8cgl8sxePBgSKVSXL58Gbdu3WpSDI3OaJmyHT5ztAxVVVWwtrZGWloaZDIZb4MsALi4uBCPKZfL4efnh0WLFqGoqAhFRUUYPnw4lixZAkC9QDZ//nwiWmxtbZGSkgJAvcoeHR2N77//nmgJ4P79+0HTNDw9PeusqvNRyuTg4IB79+7h0KFDvJYASqVSnD17FiqVCpcuXeJNBwDk5uayC9d8M3v2bNy8eRMDBw7EkCFDmnw9jc5onz9/XudYQUGBJkO8NxkZGUhLS4ORkRGvgyzATz5SIpEgMjIS+vr6WLhwIZYvX46DBw8CUA+yy5YtQ5s2bYjrAsCmMjZs2MC5CSBDQkIC2rVrh88++4xIvMa4d+8exo4dixkzZvCqIyQkBNnZ2fi///s/XnUAwIkTJ9i0gSYGt6YQHR0NiqI09uWj0YE2ODgYgPqWUKFQQCwWN+goySWpqamwsbHB8+fPkZ+fTzw+Q1lZGQDwUt516tQpAOo61vDwcAwcOBBXrlzB5s2b0aZNG7aOkw86dOiAK1euACBThRAVFQUASEpKqvc8U1s8duxYzrUAaisbMzMzVhefMBUYfLsC9+vXj01hrFq1in1/8IGbmxv09PSQmZmpsRpejQ20xcXFuH37NgD1aiFjXUwauVwOR0dHeHt7857C2L9/P8zNzXmZRTFv2uTkZPTq1Quurq5wdXXFmjVrcPjw4SbnnJoKk2YivYGiNq9evUJmZibRipSuXbvi1atXCAkJIRazMfjeYJSYmAiKojB27FjMnj2bNx1ubm6IjIxEQkICuxCmCTQ20BoZGbFlMyqVCnfu3EFpaSlxyxJmB9aOHTuIxq2Ps2fPYuXKlbysJhsZGSEvLw8BAQF48OABunbtitjYWMjlckydOpWtayWBv78/7t27B7lcjoyMDPTs2ROenp6QSCSYM2cO5/GZHLmdnR0ePHiAjIwMeHl5QSwWo1OnTkhLS0NWVhbnOhiioqKgUCgwe/ZsmJiYYPTo0RCJRBg9ejRGjx4NiqJgaGjI+SIqk9KSSqWcxmmM6vXdp0+f5s33b8WKFThz5gxmzJih8S8ejZd3jR8/HoMHD8aIESOgq6ur6cu/k/j4eHTo0AH/+9//iMZtrhgZGTUbby47OzuMGDEC58+fZxc7vvrqKyKxmS/fFy9eoE+fPjXOGRkZ4csvvySig0FHRwdLly5FcnIyLl68iOjoaABg/126dClWrFjB+UDLpJf45MSJE1izZg0A8JqXffLkCUJDQwFwsy9AYwMt3x/oAwcOYO7cubzmHmsjeJipWbBgAVauXIkLFy5g69atGD16NLFBlkGpVCIxMRF9+/bFqlWr4OLigm+++Yaohur4+/vzFrs5MWnSJEyaNIlvGejRowdMTEw4q+3+KDYsAMDcuXMb3FopwC8tWrQgvjmhPnr37s37hKA5cezYMRw7doxvGbxTUlICQJ3q44qPYqBdvXo1Xr16RWQrpYCAwMeFvr4+5xMBwQVX0CHoEHT81XQ0SYvggisgICDwEcJ/h18BAQGBjxxhoBUQEBDgGMEF9xPTce/ePcEFV9Dxl9YBCC6470VzcdPkUsfdu3fp5cuX0/Pnz6dfv35Ny2QyXnRURyKR0Oo/Ob86GkPQIehoDHyqLrg9e/aElpYWBg4ciEGDBqFPnz44evSopi7/p2G2NVZ/kHJftbOzg52dHQICArBv3z507NgRrVq14rWWdN++fbyaITYnEhIS2PcEYxqpUChQUFDAe6vATxGZTMY671Z/fExobKBNTU1lzedu3LiBuLg4tnsVaZRKJWJiYtjeltUffMA0txk3bhyKi4t50cA0lpZIJLzEby7QNI0TJ06Aoijo6elh5syZWL58OY4cOYLdu3dz0mlNoVAgISEBGzdurPFe9PHx+eQ3UFRWVrJNZPr06YMePXqw5yIiIviSVS8dO3aEWCz+oM5iGhlof/rpJwDqJhXMN1GLFi3g6upa44UjwQ8//FCnYYpSqWyyFcWfQaVSoWvXroiIiIBCocCzZ8+Qm5uL8+fPw8jIiJgOBjc3NwDqQcbS0pLzeGlpadDS0mJnJoyLqFgsRr9+/eqcI8nkyZNx9OhRKJVKFBcXIyQkhLX5efToEWvKp0lMTU2Rnp6Ovn37gqZplJeXIzw8HCYmJtDW1sasWbM0HvN9OX/+PHR0dOqdUS5evJjz+A4ODggLC8O1a9fg5eWF/Px87N69G1evXoW7uzvxfinR0dHQ0tKCTCarcbysrAxSqRQURcHJyelPX1dj7/L27dujdoJ59OjRSE1N1VSIRikrK6vRKMPAwADdu3dHWVkZysvLiek4deoU2rdvX6NrV+vWrXmZUe/btw+RkZFEZ7IWFhZ49epVDbuc+mxJMjMz0aVLF2K6APU2y+pfxIWFheyskvGY0zRv3ryp8bOuri5cXV2RmZmJ1atXQ19fn5O4jXH16lW4ublBqVTCxcUFjx49woYNGzB16lQAasPGwMBATjUkJCQAAL799ltcv34dJ06cwODBgwGoLbEuXbqEJ0+eoFu3bpzEl8vlNYwBfv31VwDqmTbTjAgAXr58CSMjI/Tv3//DAmkqiSwWi2mxWEx7eHjQu3btol1cXNhjteEqqf7mzRtaJBLRIpGIDg8Pp/Py8miapumgoCBaJBLRaWlpRHQ0xIULF2ixWEynpqYS0wGAdnV1ZX/eu3cvDYB98Pl6AKBFIlGd41zqkMlktJaWFr148WK6d+/etFgspj/77DP6yZMnRHVQFEVTFEV37tyZXrhwIZ2dnd3g73KhQyqV0m3atKFFIhEdHBzMHi8rK6M7dOhAi0Qi+tixY5zqyM/Pp0UiEe3o6Njg//utW7dokUhEz5kzp8ZxaHgxTC6X07NmzaJbtmxJ+/n51Th36tQpWiwW0/7+/rREIvkgHRrL0R46dAg+Pj4IDw/HokWLEBsbCx8fH6JT/9OnT7PPJ06cyFq1xMfHE9PwLhwdHfH1118Tj7tp0yYA6tmtt7c3AGDv3r3sMb5gFqNIoqOjg8DAQOzcuRNJSUmgaRpJSUmwsLAgqsPU1BT9+/fHixcvsGvXLjg4OBDN1zo6OqKwsBDe3t6YOHEiysvLkZmZiR49eiAnJwcGBgZwd3cnosXa2rrBc4zFd1FREacaunfvjtDQUGzatIm1O2fw9PQEAHzzzTcfnHpr8kArl8uxb98+TJs2DQEBAVAoFFAqlVAoFAgICMDnn3/e1BDvRWVlJRITE2FtbV2niTPjWU+yNV9cXFyNR0ZGBnR0dNCqVSs8fvyY8/ipqamgKAoSiQSWlpZwc3ODt7c3XF1dQdM05s6dy7mGd8EYMz579oxo3KKiIkgkErbX64EDB4j3owWA7OxsxMfHQ6VSobi4GGvXroW2tjYCAgKIxGdeg6CgIBgZGbHvy5cvX6Jbt254/fp1jVtnLmDy4StWrGjwd5g0D3NLr2mqqqrg6emJ169fIyUlBYsWLapxPj4+Hm/fvkXfvn2b1FazyQPtzZs3sWDBggbPk7IvXrNmDQ4cOIDx48cTtSWpzbVr17B//344ODiwj0GDBsHa2hodO3ZEYmIikQVCpsqA+QaOjIwEUHcl19nZmXMt9XHlyhV06dKFuEHk0KFDkZ6ezr4ev//+O9H49aGnp4cpU6agZcuWWLZsGQoLCzmPaWhoiLy8PDg7OyMwMBCzZs1iP6vr168n4gry8uVLzmM0RkhICA4dOgSFQoGxY8fC1tYWJ0+eRI8ePfD1119j2LBhAIARI0Y0KQ7nS769evVCaWkp9PT0OI3DmPwx3dprQ8q3q6GBSyaTsZYhMTExnCX3GZgUQXVcXV3rHCNRhVCb3NxcLF68GFFRURozv3sfCgsLYWBggO3bt8PS0hKDBw9uFi4DDCkpKRgwYADWrFmDPXv2cJpWSUhIwKxZs3D58mX88ccfNc7Z2NhwFrc6Dg4O2LZtG5KTkzXqz/VnKCwsBE3TUCgU7MI904hcpVJBJBLB19e3yb2umzyjLS0tfef5jRs3cl5PK5PJQNM0NmzYUKfQ+e3btwAAc3NzTjUA6nQBk/yOiIhAeno6FAoFJBIJfvzxR/bckiVLiOW/qhMZGYl9+/axdZzqXD5ZysvLMWjQIEilUgwYMIBY3JycHPTu3RuxsbGwsrKCtrY2tm/fTiz++2Bubg6pVIr9+/dzblFvbm6O2NhYSCQSJCYmsrfMbm5unE8CGJhb8fDw8AZ/JycnB4Da3ocL1qxZA6VSWefRq1cvGBkZISwsDBs3bmxynCbPaPX09EDTNNLS0uosKHTo0AFSqRTt27dvaph3kpmZCQB1ymTy8/MxceJEACCWk2RmIdbW1iguLkZycjJOnz6Nc+fOsedMTEyI+ZrVXuxiZrp8bVzYtGkT0tPTiZd1zZ8/H1lZWTXyjmVlZbyVVjUGqdtqCwsLVFZW4vjx4+jTpw8OHz5MJC6gTl8AwJEjR+rdKFJUVITvvvsObm5uWLt2LTFdpaWlePr0KRITEzWXv29qWQRN03RcXBxbynX79m160qRJtLm5Od25c2c6KCiozu9rukykoKCALeuq79EQmtYhkUjY16H6QyQS0WKxmG7Xrh2dkpLCuQ5GC6qVcVUv8SL1etQHRVG0SCSipVIpUR0ikYgODAxkf/7Pf/5Dt23blt67dy9RHe8DRVE1Soy41PHgwQO6bdu29JkzZxrVxdXfRSQS0VlZWeyx4uJieu/evbRIJKL19PToioqKOv8dOOp1sH37dlosFtNyubzR1+PP6NBIjnbgwIGwtLREamoq7O3tQdM0KIrCoUOH2BkllzS020pHR4dY1QPQeL7zyZMnxPRYWlpC/T5oXlAUhZkzZ2rczvldVFVVAVD3oMjNzUVGRgbWr18PX19fXndlvauTWn0bPLjAw8MDBQUFvC2Krlq1Cps2bUK/fv0QGhqKpUuXorS0FFlZWXBycsKmTZuILMwB6pysr68vzMzMNL5jUWNXe/TokaYu9UGUl5fj4cOHbN5v69atcHNzI36L2hxMCJsrL1++RPv27bFt2zbk5uYSG2x1dHTQtWtXODg4YOTIkdiyZUuz6DFgY2MDkajuMsmdO3fq2KJzwW+//QYzMzOEhYURWyyuzfr167F+/Xr2ZxcXF150AMDChQsRFBQELy8vjV/7ozBnBNS9Ffr27dssPkACDZOTkwNnZ2dYWlqyPTJI8OTJE2Kx3hdtbW3I5XL25/3792PmzJlEZnBPnz7F1KlT8fTpU7amWOD/b07QNB/NQCvQ/OnUqRNUKhXfMpoV1atk5s2bRyzuV199xVbkCKgJCgri7NqCC66gQ9Ah6Pir6WiSFsEFV0BAQOAjRDBnFBAQEOAYYaAVEBAQ4BhOXXAfP34MpVJZpw1ac3HTFHQIOgQdfz0dgOCCS9O0uonuunXr2ObGtWkubpqfmo7MzEy6Xbt29MGDB3nV0RiCDkFHY+BTdcFlWLBgAbS1tbF27VqcOnVKKOepxqhRo3Dx4kVeYo8bNw6dO3dGXl4e/vWvf/GiAVDv0tq2bRv69++PpUuXEuv5UFuDmZkZ6431wfYkHwkymQzFxcWQSCSseahcLoehoSFevXpFRENkZCQ6dOgAiqIwdepUnD9/nkjcxoiLi0N0dDTUY+qHo9GBtrS0FHv27IFIJEJwcHC9bfk+BRraNHHt2rV6dwJxjUKhwNmzZwGouxXxoQFQt6QbMWIEVq5cifv372Pnzp2sPxQpiouLMW7cOGRnZ7PHcnNzERUVRVRHc6GiogL+/v5o3bo1rKysMGTIEMTHx6OiogIlJSVswyYuKSgogLu7O3JzcyESifDzzz/z3piewdfXF2PHjm2ye7XGPnEzZsyAjY0Nrl69CqVSyesecgAIDQ1FaGgoWrduDWtra4SGhrIt17imoTcJSYNIBpVKxe52uX//fpP7an4IV65cgZaWFoyNjXHo0CHWgaO0tBTGxsZITEwkpsXc3LzGXYWBgQGysrJYp2BSZGRkIDQ0lJ1Vi0Qi9jlJ2rZti7Vr16KgoAAFBQW4+u8ljQAACJdJREFUdu0aoqOjif1NfvrpJxgbG+PRo0esO8uCBQuaRVPwnJwcpKWloVWrVk2enGhkZ1hpaSkiIyNRUVFBfIZSm6ioKISHh+PXX39l2xIWFxdj9uzZsLS0JNKTISQkBMHBwZzHeR9CQ0Nx5MgRbNmyBVZWVrxomDdvHiZPnox169bV6Av89u1byGQyYi4LcrmcnZm0bNkSz58/R0lJCVG/sJMnTyI0NBSxsbGgaXXzJRsbG9jY2CAsLKzehu1cUVlZicrKSkyZMgUGBgbs8WHDhmHKlClwcnLCwIEDOdUwceJEFBUVsQ2ZysrKEBISwmnM9+Xnn3+GVCrFuXPnmtxOs8kz2qioKBgYGODLL7/E27dviZvtJSQkYO/evdDX14eWlhYmTJjQYCPhyspKotqaA56entDS0sKyZcuQkpKC0aNHo0WLFvDz8yOmYfXq1QgLC0PXrl1rHP/iiy/QqlUrdOrUiYgOe3t79nlRURGMjY2hqdXn98XGxgZTp05FeXk5VCoVlEol7t+/j+7du8Pe3h7+/v7EtLRs2RIURdVxWHByckJ2djYuXbrEuQYdHR32DjAvL48d8PnO0VZVVbG2T021sQE0MNCeP38eNE3Dzs6ujuOtTCbDhQsXcP369aaGaZC//e1v8PHxQUVFBXusd+/eyMjIQHp6OtLT09njtd0tP3aYVMWgQYPw8OFDODk5ISYmBgqFgmgK4fvvv2ef5+XlAQCOHj0KQO05R4KEhAQ8f/6c/Zlpg0c6X21paQkPD48aDchLSkrwyy+/wMXFhXNDxNp88cUXyMrKQl5eHvLy8tCvXz8AwO7du4nqWL16NevsoK2tzXsXvLS0NNy6dUtj12ty6uDq1augKAqBgYEAgD/++AO+vr7IzMyErq4unj17BgMDA7x48YKTXqyXL19G27ZtG50VtWrVCv/85z81Hr82tWcBFy9eZD/My5YtY83euCYpKQm2trYICgqCt7c3m/sLCgpCp06dMH78eDx8+JBIOsHQ0BCFhYUYMGAATExM2Ftmkh8mprLAw8MDYWFhNc45OzuzZpZ80KtXLzx//hxJSUnEYyclJWHOnDkwMTEBAAwZMoSXQS4yMhLdunXD5cuXcevWLcybNw/Pnj3jbVFs/PjxMDY2btCD8M/S5IH26dOnANSzV11dXQwbNgza2trIyspCeXk5zMzM8Pe//52zhteM73tDbNiwAYDaVpoEu3btAk3TmDRpEsLDw9lBhWHLli1EdDDY29uznm2TJk2Cl5cXKIrC6dOnierQ19dHXFwc2rRpA5VKVaMHKUnqex8cP36cN3PA7OxsFBQU8FYJUhvS7wuGlJQU9rmzszNOnjyJXr16sS01SZOeng5jY2N4eHho5Hoa++tOnjwZBw8ehKenJ+7cuYPWrVtDV1eXuJ00g0KhwMOHDxEQEIDt27dj1KhRROLOnj0bFEXh0qVLmDx5MnJzcyGVSiGVSjnrdVkfTN7NysqKrQEcOXIk+4G+e/cuMS0MOTk5oCgKIpEIDg4OxOM3xObNm3mLHRgYiJKSEjg6OvIS38/Pr8aaxoIFC3jRUZvOnTuDoihe3qfMWo6WlpbG1pyaPKN1d3dHeHg4YmJiEBMTA5VKBblcDjs7O9y5cweAepZHGm1tbYhEIgwYMAALFy4kFnfMmDH1btKQyWREbXWYvLiOjg5r5ZKamoqioiKMHDkSVVVVRBfE2rdvj/z8fGLxGqK27X15eTl2794NU1NTXvTs3LkTgLo8kjReXl44c+YMLl68iMePH2PRokUICwvDkSNHiGupTe2/EykqKiowYsQITJs2Dbt27dKYeWeTZ7S13Sl79uwJKysr3LlzB8bGxli6dCm0tbWbGuZPIZPJIBKJYG9vz2vurTrJycm8xFWpVGzH/o0bN6J3794wNzfH77//TkzDixcvkJ+fD09PT84dkd8XpVKJ3bt3Y8eOHTA1NcW1a9eIa1CpVKBpGvr6+hq7RX1frl+/joMHD+LMmTPsrTqgztE2B1JTU3mJe/z4cdy4cQNbt27VqENyk2e03bt3h0qlwqlTp3D27FkcP34cWlpakEgkxPzhq+Pt7Y3g4GBcuXKl2bxpABCdzQLqCovIyMg6X3IuLi51LMi55Pbt23B2dkZeXh4MDQ0RHByMAQMG8Pa3qW9DQGZmJjp27Ehcy7Zt20BRFJHdV7UZOnQoRCIRu0h48+ZNfPvtt4iJiSGuRalUori4GEZGRlAqldi5cydWrFiBxYsX4x//+AdRLV5eXujcuTOMjT+4v3m9aCxHO2HCBBw9ehQqlQpVVVW8DLI5OTkIDg6Gvr5+sxpkARAtigfUdxbVV9f9/f2RnZ2NPXv2ENWxePFiVFRUICsrC6tXrwbAf41kdTw8PHhLGwDA1KlTeblNpigKLVu2xJUrVzB37lxQFMVLeRmgvl3v0aMH/vvf/8Ld3R0rVqwAAKKpLUBdgWFvb4/Hjx9r/NrNY6lTA8yfPx/dunVDQUEBCgsL+ZbDO2KxGB4eHlAqlVAqlVi8eDFMTEyIr24fPnwYenp66N27N96+fYvy8nIYGhoS1QCAfR1qP8LCwnhZ8S8pKcGqVavg5eXFS/ygoCCUl5dj48aNGDVqFBQKBVauXElcB6BeT3nz5g2cnJxgb28PiUQChUJBzGYcUO9StLW1xY0bNzj5svkozBnlcjlOnz4NNze3GlsJmxukS7uaAxYWFoIJYD0wq9ljxoxBeno6cbvvuXPnNpvGLS1atOB9g4KhoSGnDtp/+YF2+vTpyMzMRFZWFt9SBATeGz09PaGF6CeE4IIr6BB0CDr+ajqapEVwwRUQEBD4CPloFsMEBAQEmivCQCsgICDAMcJAKyAgIMAxwkArICAgwDHCQCsgICDAMcJAKyAgIMAxwkArICAgwDHCQCsgICDAMcJAKyAgIMAx/w/tNfkGxT0bggAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 100 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Each of these functions first downloads the data and returns a numpy array.\n",
    "# We call .tolist() because our \"tensors\" are just lists.\n",
    "train_images = mnist.train_images().tolist()\n",
    "train_labels = mnist.train_labels().tolist()\n",
    "    \n",
    "assert shape(train_images) == [60000, 28, 28]\n",
    "assert shape(train_labels) == [60000]\n",
    "    \n",
    "import matplotlib.pyplot as plt\n",
    "    \n",
    "fig, ax = plt.subplots(10, 10)\n",
    "    \n",
    "for i in range(10):\n",
    "    for j in range(10):\n",
    "        # Plot each image in black and white and hide the axes.\n",
    "        ax[i][j].imshow(train_images[10 * i + j], cmap='Greys')\n",
    "        ax[i][j].xaxis.set_visible(False)\n",
    "        ax[i][j].yaxis.set_visible(False)\n",
    "    \n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the MNIST test data\n",
    "    \n",
    "test_images = mnist.test_images().tolist()\n",
    "test_labels = mnist.test_labels().tolist()\n",
    "    \n",
    "assert shape(test_images) == [10000, 28, 28]\n",
    "assert shape(test_labels) == [10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[60000, 28, 28]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shape(train_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recenter the images\n",
    "    \n",
    "# Compute the average pixel value\n",
    "avg = tensor_sum(train_images) / 60000 / 28 / 28\n",
    "    \n",
    "# Recenter, rescale, and flatten\n",
    "train_images = [[(pixel - avg) / 256 for row in image for pixel in row]\n",
    "                    for image in train_images]\n",
    "test_images = [[(pixel - avg) / 256 for row in image for pixel in row]\n",
    "                   for image in test_images]\n",
    "    \n",
    "assert shape(train_images) == [60000, 784], \"images should be flattened\"\n",
    "assert shape(test_images) == [10000, 784], \"images should be flattened\"\n",
    "    \n",
    "# After centering, average pixel should be very close to 0\n",
    "assert -0.0001 < tensor_sum(train_images) < 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hot encode the test data\n",
    "def one_hot_encode(i: int, num_labels: int = 10) -> List[float]:\n",
    "    return [1.0 if j == i else 0.0 for j in range(num_labels)]\n",
    "\n",
    "assert one_hot_encode(3) == [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]\n",
    "assert one_hot_encode(2, num_labels=5) == [0, 0, 1, 0, 0]\n",
    "    \n",
    "train_labels = [one_hot_encode(label) for label in train_labels]\n",
    "test_labels = [one_hot_encode(label) for label in test_labels]\n",
    "    \n",
    "assert shape(train_labels) == [60000, 10]\n",
    "assert shape(test_labels) == [10000, 10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic regression model for MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "mnist loss: 0.357 acc: 0.898: 100%|█████████████████████████████████████████████| 60000/60000 [09:39<00:00, 103.51it/s]\n",
      "mnist loss: 0.361 acc: 0.891: 100%|█████████████████████████████████████████████| 10000/10000 [00:20<00:00, 498.65it/s]\n"
     ]
    }
   ],
   "source": [
    "# Training loop    \n",
    "import tqdm\n",
    "    \n",
    "def loop(model: Layer,\n",
    "            images: List[Tensor],\n",
    "             labels: List[Tensor],\n",
    "             loss: Loss,\n",
    "             optimizer: Optimizer = None) -> None:\n",
    "    correct = 0         # Track number of correct predictions.\n",
    "    total_loss = 0.0    # Track total loss.\n",
    "    \n",
    "    with tqdm.trange(len(images)) as t:\n",
    "        for i in t:\n",
    "            predicted = model.forward(images[i])             # Predict.\n",
    "            if argmax(predicted) == argmax(labels[i]):       # Check for\n",
    "                correct += 1                                 # correctness.\n",
    "            total_loss += loss.loss(predicted, labels[i])    # Compute loss.\n",
    "    \n",
    "            # If we're training, backpropagate gradient and update weights.\n",
    "            if optimizer is not None:\n",
    "                gradient = loss.gradient(predicted, labels[i])\n",
    "                model.backward(gradient)\n",
    "                optimizer.step(model)\n",
    "    \n",
    "            # And update our metrics in the progress bar.\n",
    "            avg_loss = total_loss / (i + 1)\n",
    "            acc = correct / (i + 1)\n",
    "            t.set_description(f\"mnist loss: {avg_loss:.3f} acc: {acc:.3f}\")\n",
    "    \n",
    "    \n",
    "# The logistic regression model for MNIST\n",
    "    \n",
    "random.seed(0)\n",
    "    \n",
    "# Logistic regression is just a linear layer followed by softmax\n",
    "model = Linear(784, 10)\n",
    "loss = SoftmaxCrossEntropy()\n",
    "    \n",
    "# This optimizer seems to work\n",
    "optimizer = Momentum(learning_rate=0.01, momentum=0.99)\n",
    "    \n",
    "# Train on the training data\n",
    "loop(model, train_images, train_labels, loss, optimizer)\n",
    "    \n",
    "# Test on the test data (no optimizer means just evaluate)\n",
    "loop(model, test_images, test_labels, loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep neural network for MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "mnist loss: 0.486 acc: 0.857: 100%|██████████████████████████████████████████████| 60000/60000 [24:44<00:00, 39.21it/s]\n",
      "mnist loss: 0.313 acc: 0.908: 100%|█████████████████████████████████████████████| 10000/10000 [00:47<00:00, 211.87it/s]\n"
     ]
    }
   ],
   "source": [
    "# A deep neural network for MNIST\n",
    "    \n",
    "random.seed(0)\n",
    "    \n",
    "# Name them so we can turn train on and off\n",
    "dropout1 = Dropout(0.1)\n",
    "dropout2 = Dropout(0.1)\n",
    "    \n",
    "model = Sequential([\n",
    "        Linear(784, 30),  # Hidden layer 1: size 30\n",
    "        dropout1,\n",
    "        Tanh(),\n",
    "        Linear(30, 10),   # Hidden layer 2: size 10\n",
    "        dropout2,\n",
    "        Tanh(),\n",
    "        Linear(10, 10)    # Output layer: size 10\n",
    "    ])\n",
    "    \n",
    "    \n",
    "# Training the deep model for MNIST\n",
    "    \n",
    "optimizer = Momentum(learning_rate=0.01, momentum=0.99)\n",
    "loss = SoftmaxCrossEntropy()\n",
    "    \n",
    "# Enable dropout and train (takes > 20 minutes on my laptop!)\n",
    "dropout1.train = dropout2.train = True\n",
    "loop(model, train_images, train_labels, loss, optimizer)\n",
    "    \n",
    "# Disable dropout and evaluate\n",
    "dropout1.train = dropout2.train = False\n",
    "loop(model, test_images, test_labels, loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save Model\n",
    "import json\n",
    "\n",
    "def save_weights(model: Layer, filename: str) -> None:\n",
    "    weights = list(model.params())\n",
    "    with open(filename, 'w') as f:\n",
    "        json.dump(weights, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Model\n",
    "def load_weights(model: Layer, filename: str) -> None:\n",
    "    with open(filename) as f:\n",
    "        weights = json.load(f)\n",
    "\n",
    "    # Check for consistency\n",
    "    assert all(shape(param) == shape(weight)\n",
    "               for param, weight in zip(model.params(), weights))\n",
    "\n",
    "    # Then load using slice assignment:\n",
    "    for param, weight in zip(model.params(), weights):\n",
    "        param[:] = weight"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}