8_Optimizers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aa7de84f",
   "metadata": {},
   "source": [
    "# Aim : To Compare different available `Optimizers` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "42e12314",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sonu.ramkumar.jha\\desktop\\experiments\\env\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n",
      "c:\\users\\sonu.ramkumar.jha\\desktop\\experiments\\env\\lib\\site-packages\\numpy\\.libs\\libopenblas.GK7GX5KEQ4F6UYO3P26ULGBQYHGQO7J4.gfortran-win_amd64.dll\n",
      "c:\\users\\sonu.ramkumar.jha\\desktop\\experiments\\env\\lib\\site-packages\\numpy\\.libs\\libopenblas.WCDJNK7YVMPZQ2ME2ZZHJJRJ3JIKNDB7.gfortran-win_amd64.dll\n",
      "  warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "# TensorFlow and tf.keras\n",
    "import tensorflow as tf\n",
    "\n",
    "# Helper libraries\n",
    "import numpy as np\n",
    "from tensorflow.keras import initializers\n",
    "from tensorflow.python.keras import activations\n",
    "\n",
    "print(tf.__version__)\n",
    "\n",
    "# downloading fashion_mnist data\n",
    "fashion_mnist = tf.keras.datasets.fashion_mnist\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n",
    "\n",
    "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n",
    "               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\n",
    "\n",
    "train_images = train_images / 255.0\n",
    "\n",
    "test_images = test_images / 255.0       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5bf87d91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "optim = tf.keras.optimizers.SGD()\n",
    "model.compile(optimizer=optim,loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e64a53b0",
   "metadata": {},
   "source": [
    "<img src=\"images\\model.jpeg\" height=50% width=50% alt-text=\"Case 1 Gradient Descent\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d72e889",
   "metadata": {},
   "source": [
    "**Forward Pass:**\n",
    "\n",
    "$$\\overbrace{\\begin{bmatrix}\n",
    "    i_{1}  \\\\\n",
    "    i_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    i_{784}\n",
    "\\end{bmatrix}\\begin{bmatrix}\n",
    "    w^{1}_{11} & w^{1}_{12} & w^{1}_{13} & \\dots  & w^{1}_{1128} \\\\\n",
    "    w^{1}_{21} & w^{1}_{22} & w^{1}_{23} & \\dots  & w^{1}_{2128} \\\\\n",
    "    \\vdots & \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "    w^{1}_{7841} & w^{1}_{7842} & w^{1}_{7843} & \\dots  & w^{1}_{784128}\n",
    "\\end{bmatrix}+\\begin{bmatrix}\n",
    "    b_{1}  \\\\\n",
    "    b_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    b_{128}\n",
    "\\end{bmatrix}}^{Input of First Activation Function} =\\begin{bmatrix}\n",
    "    s_{1}  \\\\\n",
    "    s_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    s_{128}\n",
    "\\end{bmatrix}$$\n",
    "\n",
    "$$Activation1 \\ \\left (\\begin{bmatrix}\n",
    "    s_{1}  \\\\\n",
    "    s_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    s_{128}\n",
    "\\end{bmatrix} \\right)=\\begin{bmatrix}\n",
    "    i^{`}_{1}  \\\\\n",
    "    i^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    i^{`}_{128}\n",
    "\\end{bmatrix}$$\n",
    "\n",
    "$$\\overbrace{\\begin{bmatrix}\n",
    "    i^{`}_{1}  \\\\\n",
    "    i^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    i^{`}_{128}\n",
    "\\end{bmatrix}\\begin{bmatrix}\n",
    "    w^{2}_{11} & w^{2}_{12} & w^{2}_{13} & \\dots  & w^{2}_{110} \\\\\n",
    "    w^{2}_{21} & w^{2}_{22} & w^{2}_{23} & \\dots  & w^{2}_{210} \\\\\n",
    "    \\vdots & \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "    w^{2}_{1281} & w^{2}_{1282} & w^{`}_{1283} & \\dots  & w^{2}_{12810}\n",
    "\\end{bmatrix}+\\begin{bmatrix}\n",
    "    b^{`}_{1}  \\\\\n",
    "    b^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    b^{`}_{128}\n",
    "    \\end{bmatrix}}^{Input of Second Activation Function}=\\begin{bmatrix}\n",
    "    s^{`}_{1}  \\\\\n",
    "    s^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    s^{`}_{10}\n",
    "\\end{bmatrix}$$\n",
    "\n",
    "\n",
    "$$Activation2 \\ \\left (\\begin{bmatrix}\n",
    "    s^{`}_{1}  \\\\\n",
    "    s^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    s^{`}_{128}\n",
    "\\end{bmatrix}\\  \\right )=\\begin{bmatrix}\n",
    "    y^{`}_{1}  \\\\\n",
    "    y^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    y^{`}_{10}\n",
    "\\end{bmatrix}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f0e7d1f",
   "metadata": {},
   "source": [
    "**Whole Forward Pass:**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f904989e",
   "metadata": {},
   "source": [
    "$$Activation2\\left (Activation1\\left (\\overbrace{\\begin{bmatrix}\n",
    "    i_{1}  \\\\\n",
    "    i_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    i_{784}\n",
    "\\end{bmatrix}*\\begin{bmatrix}\n",
    "    w^{1}_{11} & w^{1}_{12} & w^{1}_{13} & \\dots  & w^{1}_{1128} \\\\\n",
    "    w^{1}_{21} & w^{1}_{22} & w^{1}_{23} & \\dots  & w^{1}_{2128} \\\\\n",
    "    \\vdots & \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "    w^{1}_{7841} & w^{1}_{7842} & w^{1}_{7843} & \\dots  & w^{1}_{784128}\n",
    "\\end{bmatrix}+\\begin{bmatrix}\n",
    "    b_{1}  \\\\\n",
    "    b_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    b_{128}\n",
    "\\end{bmatrix}}^{Input of Hidden Layer}  \\right )+\\begin{bmatrix}\n",
    "    w^{2}_{11} & w^{2}_{12} & w^{2}_{13} & \\dots  & w^{2}_{110} \\\\\n",
    "    w^{2}_{21} & w^{2}_{22} & w^{2}_{23} & \\dots  & w^{2}_{210} \\\\\n",
    "    \\vdots & \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "    w^{2}_{1281} & w^{2}_{1282} & w^{2}_{1283} & \\dots  & w^{2}_{12810}\n",
    "\\end{bmatrix}+\\begin{bmatrix}\n",
    "    b^{`}_{1}  \\\\\n",
    "    b^{`}_{2}  \\\\\n",
    "    \\vdots  \\\\\n",
    "    b^{`}_{128}\n",
    "    \\end{bmatrix}\\  \\right )$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a98c0e2e",
   "metadata": {},
   "source": [
    "**In Simple Form**\n",
    "\n",
    "$$Activation2 \\ (Activation1 \\ (I_{1784} \\ W^{1}_{784128}+B^{1}_{128})+W^{2}_{12810}+B^{2}_{10}) = y{`}$$\n",
    "\n",
    "**Cast Function(Error):**\n",
    "$$C = (y-y{`})^2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e57367f",
   "metadata": {},
   "source": [
    "**Backword Pas:**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e1b59cc",
   "metadata": {},
   "source": [
    "**We have 4 values to update before the 2nd forward pass -** \n",
    "\n",
    "$$\\vec W^{1}, \\vec B^{1}, \\vec W^{2} and \\ \\vec B^{2}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52fb5f0d",
   "metadata": {},
   "source": [
    "## Gradient Discent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a654ad3",
   "metadata": {},
   "source": [
    "**Gradien Discent Formula**\n",
    "\n",
    "$$\\boxed{\\vec{W_{new}} = \\vec{W_{old}} - \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{old}}}}\\$$\n",
    "\n",
    "**Where**\n",
    "\n",
    "\\begin{equation}\n",
    " \\left.\\begin{aligned}\n",
    "        \\vec{W_{new}} = New \\ Weight\\\\\n",
    "        \\vec{W_{old}} = Old \\ Weight\\\\\n",
    "        \\eta = learning \\ rate\n",
    "       \\end{aligned}\n",
    " \\right\\}\n",
    "\\end{equation}\n",
    "\n",
    "\n",
    "**According to Chain Rule**\n",
    "\n",
    "\\begin{equation}\n",
    "\\frac{\\partial \\vec C}{\\partial \\vec W^{1}} = \\frac{\\partial C}{\\partial y^{`}}\\times\\frac{\\partial y^{`}}{\\partial Activation2}\\times\\frac{\\partial Activation2}{\\partial Activation1}\\times \\frac{\\partial Activation1}{\\partial W^{1}}\\\\\n",
    "\\frac{\\partial \\vec C}{\\partial \\vec B} = \\frac{\\partial C}{\\partial y{`}}\\times\\frac{\\partial y{`}}{\\partial Activation2}\\times\\frac{\\partial Activation2}{\\partial Activation1}\\times\\frac{\\partial Activation1}{\\partial B}\\\\\n",
    "\\frac{\\partial \\vec C}{\\partial \\vec W^{2}} = \\frac{\\partial C}{\\partial y{`}}\\times\\frac{\\partial y{`}}{\\partial Activation2}\\times\\frac{\\partial Activation2}{\\partial W_{`}}\\\\\n",
    "\\frac{\\partial \\vec C}{\\partial \\vec B{`}} = \\frac{\\partial C}{\\partial y{`}}\\times\\frac{\\partial y{`}}{\\partial Activation2}\\times\\frac{\\partial Activation2}{\\partial B_{`}}\\\\\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55e298b6",
   "metadata": {},
   "source": [
    "**Step:1**\n",
    "$$When \\ \\vec W = W_{0}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "772f751c",
   "metadata": {},
   "source": [
    "$$\\therefore W_{1} = \\vec W_{0} - \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}$$\n",
    "<img src=\"images\\gd1.png\" height=50% width=50% alt-text=\"Case 1 Gradient Descent\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccfd074d",
   "metadata": {},
   "source": [
    "**Step:2**\n",
    "    $$When \\ \\vec W = W_{1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6224e061",
   "metadata": {},
   "source": [
    "$$\\therefore W_{2} = \\vec W_{1} - \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}$$\n",
    "<img src=\"images\\gd2.png\" height=50% width=50% alt-text=\"Case 2 Gradient Descent\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9dfc4f95",
   "metadata": {},
   "source": [
    "`Similarly`\n",
    "$$When \\ \\vec W = W_{n-1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "552f1c4a",
   "metadata": {},
   "source": [
    "$$\\therefore W_{n} = \\vec W_{n-1} - \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{n-1}}}$$\n",
    "<img src=\"images\\gdn.png\" height=50% width=50% alt-text=\"Case n Gradient Descent\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce07bf6e",
   "metadata": {},
   "source": [
    "**Observations**:\n",
    "    - As you can see Wn is directly propotional to Dc/Dw. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "df8586fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.7167 - accuracy: 0.7689\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.5043 - accuracy: 0.8265\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.4610 - accuracy: 0.8391\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.4365 - accuracy: 0.8456\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.4192 - accuracy: 0.8522\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.4062 - accuracy: 0.8564\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3961 - accuracy: 0.8604\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3866 - accuracy: 0.8634\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3788 - accuracy: 0.8662\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3716 - accuracy: 0.8688\n",
      "313/313 - 0s - loss: 0.4070 - accuracy: 0.8539\n",
      "test_loss 0.4069558382034302\n",
      "test_accuracy 0.8539000153541565\n"
     ]
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=10, batch_size=32)\n",
    "\n",
    "test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "print('test_loss', test_loss)\n",
    "print('test_accuracy', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40d16a85",
   "metadata": {},
   "source": [
    "# Momentum Optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d8fbe50",
   "metadata": {},
   "source": [
    "**With momentum the weight update formula becomes**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a64ebac9",
   "metadata": {},
   "source": [
    "$$\\vec{W_{new}} = \\vec{W_{old}} - m_{new}$$\n",
    "$$m_{new} = \\beta \\ m_{old}+\\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{old}}}$$\n",
    "\n",
    "`Where m is called momentum and` \n",
    "\n",
    "$$\\beta : coefficient \\ of \\ momentum$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d71ad92",
   "metadata": {},
   "source": [
    "**Step:1**\n",
    "$$When: \\ \\vec W_{old} = W_{0} \\ and \\ m = m_{0} = 0$$\n",
    "\n",
    "$$\\therefore m_{1} = \\beta \\ m_{0}+\\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}\\$$\n",
    "\n",
    "$$\\therefore m_{1} = \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}} \\ \\left (\\because m_{0}=0\\right )\\tag{i}$$\n",
    "\n",
    "$$\\therefore \\boxed{\\vec W_{1} = W_{0}-m_{1} = \\vec W_{0} - \\eta \\frac{\\partial \\vec\n",
    "{C}}{\\partial \\vec{W_{0}}}}\\tag{Same as Gradient Discent Formula}$$\n",
    "<img src=\"images\\gd1.png\" height=50% width=50% alt-text=\"Case 1 Momentum\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee424e86",
   "metadata": {},
   "source": [
    "**Observations**\n",
    "- As you can see for the first time weight update happens like the `Gradient Discent` when `m=0`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "533ebd4f",
   "metadata": {},
   "source": [
    "**Step:2**\n",
    "$$When: \\ \\vec W_{old} = W_{1} \\ and \\ m =  m_{1} = \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}\\tag{from eq(i)}\\$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "177ff260",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec m_{2} = \\beta \\ m_{1} + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}\\tag{ii}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "171467c7",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{2} = \\vec W_{1} - m2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab546ba1",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{2} = \\vec W_{1} - \\beta \\ m_{1} + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c248c04c",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{2} = \\vec W_{1} - \\beta \\left( \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}} \\right)+ \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d2e50c5",
   "metadata": {},
   "source": [
    "$$\\therefore \\boxed{\\vec W_{2} = \\vec W_{1} - \\eta \\left( \\beta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}+\\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}} \\right)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2761de89",
   "metadata": {},
   "source": [
    "**Case:1**\n",
    "$$For \\ \\beta = 0$$\n",
    "$$\\therefore \\boxed{\\vec W_{2} =\\vec W_{1}-\\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}}\\tag{same as Gradient Discent}$$\n",
    "<img src=\"images\\mc2.png\" height=50% width=50% alt-text=\"Case 2 Momentum\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0ce4136",
   "metadata": {},
   "source": [
    "**Case:2**\n",
    "$$For \\ \\beta = 0.9$$\n",
    "$$\\therefore \\boxed{\\vec W_{2} = \\vec W_{1} - \\eta \\left( 0.9 \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}+\\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}} \\right)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68bd8734",
   "metadata": {},
   "source": [
    "**Observations**\n",
    "- For the second time weight depends on 90% of the initial weight plus 100% last weight."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25bac8a2",
   "metadata": {},
   "source": [
    "**Step:3**\n",
    "$$When: \\ \\vec W_{old} = W_{2} \\ and \\ m =  \\vec m_{2} = \\beta \\ m_{1} + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}\\tag{from eq(ii)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dafcff6",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec m_{3} = \\beta \\ m_{2} + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{2}}}\\tag{iii}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c1fae82",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec m_{3} = \\beta \\left( \\beta \\ m_{1} + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}\\right) + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{2}}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b16c51d",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec m_{3} = \\beta \\left( \\beta \\ \\eta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}} + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}\\right) + \\eta \\ \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{2}}}\\tag{from eq(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a810bf4d",
   "metadata": {},
   "source": [
    "$$\\therefore \\boxed{ \\vec m_{3} = \\eta \\left (\\beta^{2} \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}+ \\beta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}+\\frac{\\partial \\vec{C}}{\\partial \\vec{W_{2}}} \\right)}\\tag{iv}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07fd680b",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{3} = \\vec W_{2}-m_{3}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f830bf39",
   "metadata": {},
   "source": [
    "$$\\therefore \\boxed{\\vec W_{3} = \\vec W_{2}-\\eta \\left (\\beta^{2} \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{0}}}+ \\beta \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{1}}}+\\frac{\\partial \\vec{C}}{\\partial \\vec{W_{2}}} \\right)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57de1df7",
   "metadata": {},
   "source": [
    "`Similarly`\n",
    "$$When: \\ \\vec W_{old} = W_{n-1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bad3a273",
   "metadata": {},
   "source": [
    "$$\\vec W_{n} = W_{n-1}-\\eta \\left (\\sum{\\substack{\n",
    "            3\\le i \\le 0\\\\\n",
    "            0\\le j \\le 3}}\n",
    "\\beta^{i-j} \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{j}}}\\right)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98b821b8",
   "metadata": {},
   "source": [
    "$$\\vec W_{n} = W_{n-1}-\\eta \\left (\\sum{\\substack{\n",
    "            3\\le i \\le 0\\\\\n",
    "            0\\le j \\le 3}}\n",
    "\\beta^{i-j} \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{j}}}\\right)$$\n",
    "<img src=\"images\\mn.png\" height=50% width=50% alt-text=\"Case n Momentum\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55bc12d4",
   "metadata": {},
   "source": [
    "**Case:1**\n",
    "$$For \\ \\beta = 0$$\n",
    "$$\\boxed{\\vec W_{n} = \\vec W_{n-1} - \\frac{\\partial \\vec{C}}{\\partial \\vec{W_{n-1}}}}\\tag{Same as Gradiend Discent}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82fdcfe0",
   "metadata": {},
   "source": [
    "**Observatinos**\n",
    "- As you can see in Case:1 for beta = 0, weight update happens like Gradient Discent.\n",
    "- For beta=0.9 (practically good), Weight depends on 81% of past weight, 90% recent past weight and 100% the last weight.\n",
    "- Therefore `momemtum optimizer` has and advantage over `Gradient Discent` that with the `impace of previous weight` it can jump through `saddle point`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b47903e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.5101 - accuracy: 0.8170\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3991 - accuracy: 0.8548\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3682 - accuracy: 0.8665\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3441 - accuracy: 0.8745\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3270 - accuracy: 0.8792\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3120 - accuracy: 0.8849\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3035 - accuracy: 0.8867\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2893 - accuracy: 0.8939\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2817 - accuracy: 0.8959\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2741 - accuracy: 0.8988\n",
      "313/313 - 0s - loss: 0.3553 - accuracy: 0.8704\n",
      "test_loss 0.3552625775337219\n",
      "test_accuracy 0.8704000115394592\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "optim = tf.keras.optimizers.SGD(momentum=0.9)\n",
    "model.compile(optimizer=optim,loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "model.fit(train_images, train_labels, epochs=10, batch_size=32)\n",
    "\n",
    "test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "print('test_loss', test_loss)\n",
    "print('test_accuracy', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5534f1ae",
   "metadata": {},
   "source": [
    "## 3. Nesterov Accelaraged Gradient (NAG)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "626b179a",
   "metadata": {},
   "source": [
    "**Weight Update Formula**\n",
    "\n",
    "$$W_{new} = W_{old} - m_{new}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f97aca3b",
   "metadata": {},
   "source": [
    "$$where \\ m_{new} = \\beta \\ m_{ols} + \\eta \\ \\frac{\\partial C}{\\partial \\left ( W_{old}-\\beta \\ m_{old}\\right )}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f70dbc1a",
   "metadata": {},
   "source": [
    "**Step:1**\n",
    "\n",
    "$$When \\ m = m_{0} = 0 \\ and \\ \\vec W = W_{0}\\tag{i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75ba9fec",
   "metadata": {},
   "source": [
    "$$m_{1} = \\beta \\ m_{0} + \\eta \\ \\frac{\\partial C}{\\partial \\left ( W_{0}-\\beta \\ m_{0}\\right )}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f29b40b4",
   "metadata": {},
   "source": [
    "*since m_0 = 0*\n",
    "$$\\therefore m_{1} = \\eta \\ \\frac{\\partial C}{\\partial  W_{0}}\\tag{ii}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f9f8839",
   "metadata": {},
   "source": [
    "$$\\vec W_{1} = \\vec W_{0}-m_{1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee5ad5a4",
   "metadata": {},
   "source": [
    "$$\\boxed{\\therefore \\vec W_{1} = \\vec W_{0}- \\eta \\ \\frac{\\partial C}{\\partial  W_{0}}}\\tag{from eq(ii)}$$\n",
    "<img src=\"images\\gd1.png\" height=50% width=50% alt-text=\"Case 1 NAG\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e9b6212",
   "metadata": {},
   "source": [
    "**Step:2**\n",
    "    $$Whem \\ m = m_{1}, \\  W = W_{1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2332fc70",
   "metadata": {},
   "source": [
    "$$\\therefore m_{2} = \\beta \\ m_{1} + \\eta \\ \\frac{\\partial C}{\\partial \\left ( W_{1}-\\beta \\ m_{1}\\right )}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4789ec76",
   "metadata": {},
   "source": [
    "$$\\therefore m_{2} = \\eta \\left (\\beta \\frac{\\partial C}{\\partial  W_{0}}+ \\frac{\\partial C}{\\partial \\left(W_{1}-\\beta \\ m_{1}\\right) }\\right)\\tag{from eq(ii)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cbd8315",
   "metadata": {},
   "source": [
    "`Now : `\n",
    "$$\\vec W_{2} = \\vec W_{1}-m_{2}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27ccc5b3",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{2} = \\vec W_{1} - \\eta \\left (\\beta \\frac{\\partial C}{\\partial  W_{0}}+ \\frac{\\partial C}{\\partial \\left(W_{1}-\\beta \\ m_{1}\\right) }\\right)\\tag{from eq(ii)}$$\n",
    "<img src=\"images\\nag2.png\" height=50% width=50% alt-text=\"Case 2 NAG\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4ad25253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_2 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4975 - accuracy: 0.8235\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3906 - accuracy: 0.8594\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3573 - accuracy: 0.8695\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3377 - accuracy: 0.8777\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3201 - accuracy: 0.8827\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.3049 - accuracy: 0.8895\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2948 - accuracy: 0.8913\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2837 - accuracy: 0.8946\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2751 - accuracy: 0.8985\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2655 - accuracy: 0.9020\n",
      "313/313 - 1s - loss: 0.3505 - accuracy: 0.8717\n",
      "test_loss 0.35048532485961914\n",
      "test_accuracy 0.8716999888420105\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "optim = tf.keras.optimizers.SGD(momentum=0.9, nesterov=True)\n",
    "model.compile(optimizer=optim,loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "model.fit(train_images, train_labels, epochs=10, batch_size=32)\n",
    "\n",
    "test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "print('test_loss', test_loss)\n",
    "print('test_accuracy', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7b2f792",
   "metadata": {},
   "source": [
    "**Observations**\n",
    "- AS you can see."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddb5939a",
   "metadata": {},
   "source": [
    "## Elongated Ball Problem"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21059d91",
   "metadata": {},
   "source": [
    "## Adagrad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fe70851",
   "metadata": {},
   "source": [
    "**Weight Update Formula**\n",
    "\n",
    "$$\\vec W_{new} = \\vec W_{old} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{old}}}{\\sqrt{S_{new}+ \\epsilon}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8870b9a",
   "metadata": {},
   "source": [
    "$$where \\ S_{new} = \\ S_{old} + \\frac{\\partial \\vec C}{\\partial \\vec W_{old}} \\cdot \\frac{\\partial \\vec C}{\\partial \\vec W_{old}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "199f5dae",
   "metadata": {},
   "source": [
    "$$\\therefore \\ S_{new} = \\ S_{old} +   \\left (\\frac{\\partial C}{\\partial W_{old}}\\right)^{2}\\tag{i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7fb45f8",
   "metadata": {},
   "source": [
    "**Case:1**\n",
    "    $$When \\ S=S_{0} = 0,\\ and \\ \\vec W = \\vec W_{0}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22680dd9",
   "metadata": {},
   "source": [
    "$$\\therefore S_{1} = S_{0} + \\left (\\frac{\\partial C}{\\partial W_{0}}\\right)^{2}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a51623de",
   "metadata": {},
   "source": [
    "$$\\therefore S_{1} = \\left (\\frac{\\partial C}{\\partial W_{0}}\\right)^{2}\\tag{ii}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b19912b0",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{1} = \\vec W_{0} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{0}}}{\\sqrt{S_{1}+ \\epsilon}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fdafadd",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{1} = \\vec W_{0} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{0}}}{\\sqrt{\\left (\\frac{\\partial C}{\\partial W_{0}}\\right)^{2}+ \\epsilon}}\\tag{from eq(ii)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89c93b3a",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{1} = \\vec W_{0} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{0}}}{|\\frac{\\partial C}{\\partial W_{0}}|}\\tag{E is neayly 0}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dda31fa6",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{1} = \\vec W_{0} - \\eta$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5cbd271",
   "metadata": {},
   "source": [
    "**Case:2**\n",
    "    $$When \\ S=S_{1},\\ and \\ \\vec W = \\vec W_{1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06b04c7e",
   "metadata": {},
   "source": [
    "$$\\therefore S_{2} = S_{1} + \\left (\\frac{\\partial C}{\\partial W_{1}}\\right)^{2}\\tag{from eq(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ee1451",
   "metadata": {},
   "source": [
    "$$\\therefore S_{2} = \\left (\\frac{\\partial C}{\\partial W_{0}}\\right)^{2} + \\left (\\frac{\\partial C}{\\partial W_{1}}\\right)^{2}\\tag{from (ii)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "054c0bb5",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{2} = \\vec W_{1} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{1}}}{\\sqrt{S_{2}+ \\epsilon}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd9d0b5c",
   "metadata": {},
   "source": [
    "$$\\therefore \\vec W_{2} = \\vec W_{1} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{1}}}{\\sqrt{\\left (\\frac{\\partial C}{\\partial W_{0}}\\right)^{2} + \\left (\\frac{\\partial C}{\\partial W_{1}}\\right)^{2}+ \\epsilon}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b2d1cd1",
   "metadata": {},
   "source": [
    "`Similarly when`\n",
    "$$When \\ S=S_{n-1},\\ and \\ \\vec W = \\vec W_{n-1}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f31b13a2",
   "metadata": {},
   "source": [
    "$$\\therefore S_{n} = \\left (\\frac{\\partial C}{\\partial W_{0}}\\right)^{2} + \\left (\\frac{\\partial C}{\\partial W_{1}}\\right)^{2}+\\hdots+\\left (\\frac{\\partial C}{\\partial W_{n-1}}\\right)^{2}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be3e49e0",
   "metadata": {},
   "source": [
    "$$\\therefore S_{n} = \\sum_{n=0}^{n-1}\\left (\\frac{\\partial C}{\\partial W_{n}}\\right)^{2}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8b819d9",
   "metadata": {},
   "source": [
    "$$\\vec W_{n} = \\vec W_{n-1} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{n-1}}}{\\sqrt{S_{n}+ \\epsilon}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce79098d",
   "metadata": {},
   "source": [
    "$$\\vec W_{n} = \\vec W_{n-1} -\\eta \\frac{ \\frac{\\partial \\vec C}{\\partial \\vec W_{n-1}}}{\\sqrt{\\sum_{n=0}^{n-1}\\left (\\frac{\\partial C}{\\partial W_{n}}\\right)^{2}+ \\epsilon}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cc775657",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_3 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 1.0356 - accuracy: 0.6855\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.6952 - accuracy: 0.7764\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.6255 - accuracy: 0.7971\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5876 - accuracy: 0.8075\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5626 - accuracy: 0.8144\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5441 - accuracy: 0.8197\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5299 - accuracy: 0.8237\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5187 - accuracy: 0.8269\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5093 - accuracy: 0.8287\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5013 - accuracy: 0.8306\n",
      "313/313 - 0s - loss: 0.5237 - accuracy: 0.8190\n",
      "test_loss 0.5236613154411316\n",
      "test_accuracy 0.8190000057220459\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "optim = tf.keras.optimizers.Adagrad()\n",
    "model.compile(optimizer=optim,loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "model.fit(train_images, train_labels, epochs=10, batch_size=32)\n",
    "\n",
    "test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "print('test_loss', test_loss)\n",
    "print('test_accuracy', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adaeccec",
   "metadata": {},
   "source": [
    "**Observations**\n",
    "- As"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}