numpy_tensor_recap.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "numpy_tensor_recap.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOVTbo9XrOuSljXKJZu9H7L",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jpcompartir/dl_notebooks/blob/main/numpy_tensor_recap.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "uMCimU190KHM"
      },
      "outputs": [],
      "source": [
        "import numpy as np"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Recapping Numpy - scalars, vectors, matrices and tensors."
      ],
      "metadata": {
        "id": "Kg72YF_X0b32"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.array(12)\n",
        "x"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RsX7LBd00M2Q",
        "outputId": "6619d9d6-4da0-4f59-c61c-68d1d0a71b7c"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(12)"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x.ndim #Recall scalars are rank-0 tensors or single digits"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oNukyprb0SjE",
        "outputId": "9e406ed2-d33d-4995-a1e1-e245145027f2"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.array([12, 3, 6, 14, 7]) #A vector or rank-1 tensor - a list, or array, of scalars\n",
        "x, x.ndim, x.dtype, x.shape #shape is (5,)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A-fzwN2S0XoO",
        "outputId": "49852f22-9b31-4fa4-91cc-3f884cb0b25e"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([12,  3,  6, 14,  7]), 1, dtype('int64'), (5,))"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.array([\n",
        "              [5, 78, 2, 34, 0],\n",
        "             [6, 79, 3, 35, 1],\n",
        "             [7, 80, 4, 36, 2]\n",
        "])#Now we create a matrix, a rank-2 tensor or an array of vectors, "
      ],
      "metadata": {
        "id": "Y_GSSgnb0W5b"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x, x.ndim, x.dtype, x.shape "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GHRP9Chq1QMH",
        "outputId": "fe59c8b3-3a74-4af7-9a3c-07b16fc88f44"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[ 5, 78,  2, 34,  0],\n",
              "        [ 6, 79,  3, 35,  1],\n",
              "        [ 7, 80,  4, 36,  2]]), 2, dtype('int64'), (3, 5))"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.array([[\n",
        "               x,\n",
        "               x,\n",
        "               x\n",
        "]]) #Now we create a rank-3 tensor, or 3-d tensor. We could pack rank-3 tensors into an array and create a rank-4 tensor, and so on and so forth"
      ],
      "metadata": {
        "id": "mQUI6MjV1ua5"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x, x.ndim, x.dtype, x.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qLNSXqv1Ygal",
        "outputId": "1f88d9cf-7fcc-4100-cebf-22d4e1e75245"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[[[ 5, 78,  2, 34,  0],\n",
              "          [ 6, 79,  3, 35,  1],\n",
              "          [ 7, 80,  4, 36,  2]],\n",
              " \n",
              "         [[ 5, 78,  2, 34,  0],\n",
              "          [ 6, 79,  3, 35,  1],\n",
              "          [ 7, 80,  4, 36,  2]],\n",
              " \n",
              "         [[ 5, 78,  2, 34,  0],\n",
              "          [ 6, 79,  3, 35,  1],\n",
              "          [ 7, 80,  4, 36,  2]]]]), 4, dtype('int64'), (1, 3, 3, 5))"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.array([x, x, x]).shape #rank-4 tensor, 3 samples of the previous tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "11q6gdLVY0-8",
        "outputId": "ca2aab87-0268-409a-bcbd-bd22600d5ff0"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3, 1, 3, 3, 5)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The key descriptions of a tensor are its: \n",
        "number of axes, or rank, a rank-3 tensor has 3 axes, and a matrix has two axes. \n",
        "shape - a tuple of integers that describes how many dimensions the tensor has along each axis. So vectors have no shape - they have no axes, a vector has a shape with a single element, which itself has a dimension (the number of scalars in the vector)\n",
        "\n",
        "data type - float16, float32, float64, unit8 and so on. Can also be strings."
      ],
      "metadata": {
        "id": "z9eKtM7y2Te3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.datasets import mnist\n",
        "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
      ],
      "metadata": {
        "id": "TXMnPHIx1_UM"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_images.ndim, train_images. shape #Rank-3 tensor, an array of 60,000 matrices of 28 x 28 integers. Each matrix is a grayscale image of coefficients between 0 and 255"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SliJE4h528DB",
        "outputId": "174381e4-3e19-419f-8a4e-599471ac144e"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3, (60000, 28, 28))"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_images.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4zGctfS03Hv4",
        "outputId": "bad066b2-f578-4845-c5f4-61ce2739d669"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dtype('uint8')"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "digit = train_images[4]\n",
        "plt.imshow(digit, cmap=plt.cm.binary)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "dQQcLjHY3Dt1",
        "outputId": "77f65124-b49f-41d2-ee30-df10f10b94bb"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANpElEQVR4nO3db6xU9Z3H8c9HtxpDS4TlSpCSvbXyhKwpbSaySbGyaRbUaLAmEokSTIj0ASY2qXENakqMGt0sbWpcmtBVSrUrmrQKD0yRJY3YJ4TRsAqarmggFdF70ZhSo7LY7z64h+aKd35zmf/l+34lNzNzvnPmfDP64cyc35nzc0QIwJnvrH43AKA3CDuQBGEHkiDsQBKEHUji73q5sRkzZsTw8HAvNwmkcvDgQR09etQT1doKu+0rJP1U0tmS/jMiHiw9f3h4WPV6vZ1NAiio1WoNay1/jLd9tqT/kHSlpHmSltue1+rrAeiudr6zXyrpQES8FRHHJW2RtLQzbQHotHbCPlvSH8c9frta9jm2V9uu266Pjo62sTkA7ej60fiI2BgRtYioDQ0NdXtzABpoJ+yHJc0Z9/ir1TIAA6idsO+RNNf212yfI+kGSds60xaATmt56C0iTti+VdJ2jQ29PRYR+zvWGYCOamucPSKek/Rch3oB0EWcLgskQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1Ioq0pm20flHRM0meSTkRErRNNAei8tsJe+eeIONqB1wHQRXyMB5JoN+wh6XnbL9lePdETbK+2XbddHx0dbXNzAFrVbtgXRsS3JF0paY3t75z6hIjYGBG1iKgNDQ21uTkArWor7BFxuLodkfSMpEs70RSAzms57Lan2P7KyfuSFkva16nGAHRWO0fjZ0p6xvbJ1/mviPhtR7oC0HEthz0i3pL0jQ72AqCLGHoDkiDsQBKEHUiCsANJEHYgiU78EAYDbPfu3cX6448/Xqzv2rWrWN+3r/VTK9avX1+sX3jhhcX6iy++WKyvWLGiYW3BggXFdc9E7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2c8ATz31VMPabbfdVly32aXCIqJYX7RoUbF+9Gjja5HefvvtxXWbadZbadtbtmxpa9t/i9izA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLMPgBMnThTre/bsKdZvueWWhrWPPvqouO7ll19erN9zzz3F+sKFC4v1Tz/9tGFt2bJlxXW3b99erDdTqzGp8Hjs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZB8ATTzxRrK9atarl1168eHGxXvotvCRNnTq15W03e/12x9HnzJlTrK9cubKt1z/TNN2z237M9ojtfeOWTbe9w/Yb1e207rYJoF2T+Rj/C0lXnLLsTkk7I2KupJ3VYwADrGnYI2KXpA9OWbxU0ubq/mZJ13a4LwAd1uoBupkRcaS6/66kmY2eaHu17brterPrnQHonraPxsfYVf8aXvkvIjZGRC0iakNDQ+1uDkCLWg37e7ZnSVJ1O9K5lgB0Q6th3ybp5LjGSklbO9MOgG5pOs5u+0lJiyTNsP22pB9JelDS07ZXSTokqfzD5OTuvvvuYv2BBx4o1m0X62vWrGlYu++++4rrtjuO3sz999/ftdd++OGHi3W+Nn5e07BHxPIGpe92uBcAXcTpskAShB1IgrADSRB2IAnCDiTBT1w74N577y3Wmw2tnXvuucX6kiVLivWHHnqoYe28884rrtvMJ598Uqw///zzxfqhQ4ca1ppNudzsMtZLly4t1vF57NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2Sfpww8/bFjbsGFDcd1mP1FtNo7+7LPPFuvtOHDgQLF+4403Fuv1er3lbV9//fXF+h133NHya+OL2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs0/S8ePHG9bandaq2SWRR0bKc3Bs2rSpYW3r1vIl/ffv31+sHzt2rFhvdg7BWWc13p/cdNNNxXWnTJlSrOP0sGcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ5+kc845p2HtggsuKK7bbJx8eHi4WG82lt2O2bNnF+vNpnR+5513ivUZM2Y0rF1zzTXFddFZTffsth+zPWJ737hl62wftr23+ruqu20CaNdkPsb/QtIVEyz/SUTMr/6e62xbADqtadgjYpekD3rQC4AuaucA3a22X6k+5k9r9CTbq23XbdfbPYccQOtaDfvPJH1d0nxJRyStb/TEiNgYEbWIqA0NDbW4OQDtainsEfFeRHwWEX+R9HNJl3a2LQCd1lLYbc8a9/B7kvY1ei6AwdB0nN32k5IWSZph+21JP5K0yPZ8SSHpoKTvd7HHgXD++ec3rDW7rvvVV19drL///vvF+sUXX1ysl+Ypv/nmm4vrTp8+vVi/4YYbivVm4+zN1kfvNA17RCyfYPGjXegFQBdxuiyQBGEHkiDsQBKEHUiCsANJ8BPXDliwYEGxPsinCe/atatYf+GFF4r1Zj+/veiii067J3QHe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9uQ+/vjjYr3ZOHqzOj9xHRzs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZk1uyZEm/W0CPsGcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ09u+/bt/W4BPdJ0z257ju3f2X7N9n7bt1XLp9veYfuN6nZa99sF0KrJfIw/IemHETFP0j9JWmN7nqQ7Je2MiLmSdlaPAQyopmGPiCMR8XJ1/5ik1yXNlrRU0ubqaZslXdutJgG077QO0NkelvRNSbslzYyII1XpXUkzG6yz2nbddn2Q5zwDznSTDrvtL0v6taQfRMSfxtciIiTFROtFxMaIqEVEbWhoqK1mAbRuUmG3/SWNBf1XEfGbavF7tmdV9VmSRrrTIoBOaDr05rFrBT8q6fWI+PG40jZJKyU9WN1u7UqH6Ko333yz3y2gRyYzzv5tSSskvWp7b7VsrcZC/rTtVZIOSVrWnRYBdELTsEfE7yU1mgngu51tB0C3cLoskARhB5Ig7EAShB1IgrADSfAT1+Quu+yyYn3s5EicCdizA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLMnd8kllxTrc+fOLdab/R6+VOfKRb3Fnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVr164t1letWtXy+o888khx3Xnz5hXrOD3s2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgicnMzz5H0i8lzZQUkjZGxE9tr5N0i6TR6qlrI+K5bjWK/rjuuuuK9S1bthTrO3bsaFhbt25dcd1NmzYV61OmTCnW8XmTOanmhKQfRsTLtr8i6SXbJ/8L/iQi/r177QHolMnMz35E0pHq/jHbr0ua3e3GAHTWaX1ntz0s6ZuSdleLbrX9iu3HbE9rsM5q23Xb9dHR0YmeAqAHJh1221+W9GtJP4iIP0n6maSvS5qvsT3/+onWi4iNEVGLiBrXHAP6Z1Jht/0ljQX9VxHxG0mKiPci4rOI+Iukn0u6tHttAmhX07DbtqRHJb0eET8et3zWuKd9T9K+zrcHoFMmczT+25JWSHrV9t5q2VpJy23P19hw3EFJ3+9Kh+irqVOnFutPP/10sX7XXXc1rG3YsKG4brOhOX4Ce3omczT+95I8QYkxdeBvCGfQAUkQdiAJwg4kQdiBJAg7kARhB5JwRPRsY7VaLer1es+2B2RTq9VUr9cnGipnzw5kQdiBJAg7kARhB5Ig7EAShB1IgrADSfR0nN32qKRD4xbNkHS0Zw2cnkHtbVD7kuitVZ3s7R8iYsLrv/U07F/YuF2PiFrfGigY1N4GtS+J3lrVq974GA8kQdiBJPod9o193n7JoPY2qH1J9NaqnvTW1+/sAHqn33t2AD1C2IEk+hJ221fY/oPtA7bv7EcPjdg+aPtV23tt9/XH99UceiO2941bNt32DttvVLcTzrHXp97W2T5cvXd7bV/Vp97m2P6d7dds77d9W7W8r+9doa+evG89/85u+2xJ/yvpXyS9LWmPpOUR8VpPG2nA9kFJtYjo+wkYtr8j6c+SfhkR/1gt+zdJH0TEg9U/lNMi4l8HpLd1kv7c72m8q9mKZo2fZlzStZJuVh/fu0Jfy9SD960fe/ZLJR2IiLci4rikLZKW9qGPgRcRuyR9cMripZI2V/c3a+x/lp5r0NtAiIgjEfFydf+YpJPTjPf1vSv01RP9CPtsSX8c9/htDdZ87yHpedsv2V7d72YmMDMijlT335U0s5/NTKDpNN69dMo04wPz3rUy/Xm7OED3RQsj4luSrpS0pvq4OpBi7DvYII2dTmoa716ZYJrxv+rne9fq9Oft6kfYD0uaM+7xV6tlAyEiDle3I5Ke0eBNRf3eyRl0q9uRPvfzV4M0jfdE04xrAN67fk5/3o+w75E01/bXbJ8j6QZJ2/rQxxfYnlIdOJHtKZIWa/Cmot4maWV1f6WkrX3s5XMGZRrvRtOMq8/vXd+nP4+Inv9JukpjR+TflHRXP3po0NdFkv6n+tvf794kPamxj3X/p7FjG6sk/b2knZLekPTfkqYPUG+PS3pV0isaC9asPvW2UGMf0V+RtLf6u6rf712hr568b5wuCyTBATogCcIOJEHYgSQIO5AEYQeSIOxAEoQdSOL/AX8cJNGdGc1bAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we we need to look into how to manipulate tensors, we'll start with slicing. \n"
      ],
      "metadata": {
        "id": "5hjxhuyj3pq_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "my_slice = train_images[10:100] #(== [10:100, :, :]) == [10:100, 0:28, 0:28] == [10:100, :28, :28]\n",
        "my_slice.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rWYVbDyQ3l42",
        "outputId": "492ecf96-09ee-4bd8-80ec-f6dac997ef17"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(90, 28, 28)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Can also slice just the 14x14 pixels in the bottom right corner of each image:"
      ],
      "metadata": {
        "id": "4brq1YhM4WlE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "my_slice = train_images[:, 14:, 14:]"
      ],
      "metadata": {
        "id": "060S0Yky35gR"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(my_slice[0], cmap = plt.cm.binary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "FLTGEzvb4MaB",
        "outputId": "46fc3a96-05bd-444b-dcd4-ae6f4403273f"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f4d182193d0>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMcUlEQVR4nO3da6hd9ZnH8e8vF6cxrU10gqSJjCKiBO2M5VBtOzhSHWKtGF/MC6UOcVqYN+NUS6EaVMq8G6iUFqZUxNrKVAx46VSkFzPaUgam2ngho8bbWC+xponUqdIKGnzmxd6W4yFqu9baK8f8vx84nL3WXv/zPOdwfmdd9l7nn6pC0sFvyYFuQNI4DLvUCMMuNcKwS40w7FIjlo1ZLEktWdL978sbb7wxYDd/mnXr1nUee8QRR/SqvXz58l7j1Y6nn36aF198Mft7btSwL1myhJUrV3Ye/8orrwzYzZ/m4osv7jx28+bNvWqvXbu213i1Y25u7m2f8zBeaoRhlxph2KVG9Ap7krOSPJbkySSXD9WUpOF1DnuSpcA3gE8BG4ALkmwYqjFJw+qzZ/8o8GRVPVVVrwFbgU3DtCVpaH3Cvg54bt7yrum6t0jyj0m2J9nuHXbSgTPz19mr6lrgWoClS5eadukA6bNnfx44at7y+uk6SYtQn7D/AjguyTFJDgHOB24fpi1JQ+t8GF9V+5JcDPwYWApcX1UPD9aZpEH1Omevqh8APxioF0kz5DvopEYYdqkRo97iesIJJ7B169bO4y+99NLOY+++++7OYwG2bNnSeewzzzzTq/aVV17ZeWyf+/B1cHHPLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNGPUW1xUrVnDSSSd1Hn/rrbd2Hnv77f3+PV6fmVivueaaXrUfe+yxzmP73tqrg4d7dqkRhl1qhGGXGmHYpUb0mcX1qCQ/SfJIkoeTXDJkY5KG1edq/D7gi1V1f5IPAPcl2VZVjwzUm6QBdd6zV9ULVXX/9PErwE72M4urpMVhkHP2JEcDJwP37Oe5P0zZvHfv3iHKSeqgd9iTvB+4Fbi0ql5e+HxVXVtVc1U1t2bNmr7lJHXUK+xJljMJ+o1VddswLUmahT5X4wN8C9hZVV8driVJs9Bnz/4J4O+BTyZ5cPpx9kB9SRpYn/nZ/wvIgL1ImiHfQSc1wrBLjUhVjVZsbm6utm/fPlq9IS1fvrzz2H379vWqvWxZ9zc6btu2rVft008/vdd4jWtubo7t27fv9/TaPbvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNWLUKZtfffVVduzY0Xn8zTff3Hnsvffe23ks9L9NtY8NGzZ0HnvaaacN2Iney9yzS40w7FIjDLvUCMMuNWKI6Z+WJnkgyR1DNCRpNobYs1/CZAZXSYtY37ne1gOfBq4bph1Js9J3z/414EvAG2+3wfwpm1966aWe5SR11Wdix3OAPVV13zttN3/K5tWrV3ctJ6mnvhM7npvkaWArkwkevztIV5IG1znsVbWlqtZX1dHA+cDdVXXhYJ1JGpSvs0uNGORGmKr6KfDTIb6WpNlwzy41wrBLjRj1fvbHH3+cjRs3dh6/e/fuAbsZz5Il/f6mrl279oDV1sHD3wSpEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdasSot7ju27fvPXub6tzcXOexV111Va/a5557bq/xErhnl5ph2KVGGHapEYZdakTfiR1XJbklyaNJdib52FCNSRpW36vxXwd+VFV/l+QQ4NABepI0A53DnuSDwGnARQBV9Rrw2jBtSRpan8P4Y4C9wLeTPJDkuiQrF240f8rmHrUk9dQn7MuAjwDfrKqTgd8Bly/caP6UzT1qSeqpT9h3Abuq6p7p8i1Mwi9pEeozZfNu4Lkkx09XnQE8MkhXkgbX92r8PwM3Tq/EPwX8Q/+WJM1Cr7BX1YOA5+LSe4DvoJMaYdilRox6P3tfp5xySuexl112Wa/amzZt6jzWaZO1GPhbKDXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSI0a9n/3YY4/l6quv7jx+48aNnceuWLGi81jpYOCeXWqEYZcaYdilRvSdsvkLSR5O8lCSm5K8b6jGJA2rc9iTrAM+D8xV1YnAUuD8oRqTNKy+h/HLgBVJljGZm/1X/VuSNAt95np7HrgaeBZ4AfhtVd25cLv5Uza//PLL3TuV1Eufw/jVwCYm87R/CFiZ5MKF282fsvmwww7r3qmkXvocxp8J/LKq9lbV68BtwMeHaUvS0PqE/Vng1CSHJgmTKZt3DtOWpKH1OWe/B7gFuB/4n+nXunagviQNrO+UzV8GvjxQL5JmyHfQSY0w7FIjRr3FddWqVZx33nljlpQ05Z5daoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGvGvYk1yfZE+Sh+atOzzJtiRPTD+vnm2bkvr6Y/bs3wHOWrDucuCuqjoOuGu6LGkRe9ewV9XPgN8sWL0JuGH6+AbAfxkrLXJdz9mPrKoXpo93A0e+3Ybzp2zeu3dvx3KS+up9ga6qCqh3eP4PUzavWbOmbzlJHXUN+6+TrAWYft4zXEuSZqFr2G8HNk8fbwa+P0w7kmblj3np7Sbgv4Hjk+xK8jngX4G/TfIEcOZ0WdIi9q5zvVXVBW/z1BkD9yJphnwHndQIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41ouuUzV9J8miSHUm+l2TVbNuU1FfXKZu3ASdW1YeBx4EtA/claWCdpmyuqjurat908efA+hn0JmlAQ5yzfxb44QBfR9IM9Qp7kiuAfcCN77CN87NLi0DnsCe5CDgH+Mx0jvb9cn52aXF414kd9yfJWcCXgL+pqt8P25KkWeg6ZfO/AR8AtiV5MMk1M+5TUk9dp2z+1gx6kTRDvoNOaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxqRd/jHsMMXS/YCz7zDJn8OvDhSO9a29sFY+y+qar//xnnUsL+bJNuras7a1rb28DyMlxph2KVGLLawX2tta1t7NhbVObuk2Vlse3ZJM2LYpUYsirAnOSvJY0meTHL5iHWPSvKTJI8keTjJJWPVntfD0iQPJLlj5LqrktyS5NEkO5N8bMTaX5j+vB9KclOS98243vVJ9iR5aN66w5NsS/LE9PPqEWt/Zfpz35Hke0lWzaL2Qgc87EmWAt8APgVsAC5IsmGk8vuAL1bVBuBU4J9GrP2mS4CdI9cE+Drwo6o6AfjLsXpIsg74PDBXVScCS4HzZ1z2O8BZC9ZdDtxVVccBd02Xx6q9DTixqj4MPA5smVHttzjgYQc+CjxZVU9V1WvAVmDTGIWr6oWqun/6+BUmv/DrxqgNkGQ98GngurFqTut+EDiN6QSdVfVaVf3fiC0sA1YkWQYcCvxqlsWq6mfAbxas3gTcMH18A3DeWLWr6s6q2jdd/Dmwfha1F1oMYV8HPDdveRcjBu5NSY4GTgbuGbHs15jMc//GiDUBjgH2At+enkJcl2TlGIWr6nngauBZ4AXgt1V15xi1Fziyql6YPt4NHHkAegD4LPDDMQothrAfcEneD9wKXFpVL49U8xxgT1XdN0a9BZYBHwG+WVUnA79jdoexbzE9N97E5A/Oh4CVSS4co/bbqcnrz6O/Bp3kCiankjeOUW8xhP154Kh5y+un60aRZDmToN9YVbeNVRf4BHBukqeZnLp8Msl3R6q9C9hVVW8exdzCJPxjOBP4ZVXtrarXgduAj49Ue75fJ1kLMP28Z8ziSS4CzgE+UyO92WUxhP0XwHFJjklyCJOLNbePUThJmJy37qyqr45R801VtaWq1lfV0Uy+57urapQ9XFXtBp5Lcvx01RnAI2PUZnL4fmqSQ6c//zM4MBcobwc2Tx9vBr4/VuEkZzE5fTu3qn4/Vl2q6oB/AGczuSr5v8AVI9b9ayaHbzuAB6cfZx+A7/904I6Ra/4VsH36vf8HsHrE2v8CPAo8BPw78GczrncTk+sDrzM5qvkccASTq/BPAP8JHD5i7SeZXKd683fumjF+7r5dVmrEYjiMlzQCwy41wrBLjTDsUiMMu9QIwy41wrBLjfh/T/KINbSSAtkAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Or the pixels in the centre:"
      ],
      "metadata": {
        "id": "hQPbY9MX4hRw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "my_slice = train_images[:, 7:-7, 7:-7] #Recalling that in python negative slices start counting backwards from the end."
      ],
      "metadata": {
        "id": "SanBRdd64NFx"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(my_slice[0], cmap = plt.cm.binary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "DAGXojKL4r6b",
        "outputId": "9fad57b5-442d-437e-9751-aadac2f1db7f"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f4d1818afd0>"
            ]
          },
          "metadata": {},
          "execution_count": 18
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANb0lEQVR4nO3db4xV9Z3H8c+HGdhWwA7sGsMfI0aNG4JdJJNq2w27omuoNYDJPlDriluS9cFqpSEhEDWNPjCbtDElsSlRsNXtRBNBthPTdkWgNptsteAfFsFB1qJiobCpCw19gNjvPriXDc4yyJ7fuWcufN+vhMz9953vd8b5eO4995z7c0QIwLlvzGgPAKAZhB1IgrADSRB2IAnCDiTR22Szvr6+mDp1auX63bt3V679+OOPK9cinxkzZhTV9/T01DPI/9PBgwd15MgRn+q+RsM+depUDQwMVK6/9tprK9cePny4ci3OTvYp/+bPyIMPPljUe8KECUX1VS1fvnzE+3gaDyRB2IEkCDuQRFHYbc+3PWR7j+0VdQ0FoH6Vw267R9L3JH1F0kxJt9qeWddgAOpVsmX/gqQ9EfFORByT9IykhfWMBaBuJWGfJun9k67va9/2Cbb/wfZW21s//PDDgnYASnR8B11EPBYR/RHRP2nSpE63AzCCkrB/IOmik65Pb98GoAuVhP1Xki63fYntcZJukTRYz1gA6lb5cNmIOG77bkn/KqlH0hMR8WZtkwGoVdGx8RHxE0k/qWkWAB3EEXRAEoQdSKLRU1zHjRuniy++uHL95MmTK9dyims1V199dVF9X19f5dotW7YU9R43blzl2jvuuKOo92h5+OGHR7yPLTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJRk9x7e3tLTpN9ZFHHqlcOzhY9vF4c+bMqVx7zz33FPUuMXv27KL6TZs2FdWPHz++cu2OHTuKeq9ataqo/lzDlh1IgrADSRB2IAnCDiRRsorrRba32N5p+03b99Y5GIB6leyNPy5pWUS8anuipG22N0bEzppmA1Cjylv2iNgfEa+2L/9e0i6dYhVXAN2hltfstmdIukrSy6e473+XbD506FAd7QBUUBx22xMkrZe0NCKODL//5CWbL7jggtJ2ACoqCrvtsWoFfSAinqtnJACdULI33pLWStoVEdWPYwXQiJIt+5cl/Z2kebZfb/+7saa5ANSsZH32f5PkGmcB0EEcQQckQdiBJBo9n73UokWLKtfOmzevqPfEiRMr177xxhtFvdesWVO5dtmyZUW9S85HLzVr1qyi+scff7ymSc4NbNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJnFWnuJY4//zzU/YuOT1Wkm677bai+jFj2J50C/5LAEkQdiAJwg4kQdiBJOpY/qnH9mu2n69jIACdUceW/V61VnAF0MVK13qbLumrksre3wHQcaVb9u9KWi7pjyM9gCWbge5QsrDjTZIORsS20z2OJZuB7lC6sOMC23slPaPWAo8/qmUqALWrHPaIWBkR0yNihqRbJG2OiNtrmwxArXifHUiilhNhIuLnkn5ex/cC0Bls2YEkCDuQRJrz2UfTQw89VFS/bdtp3908rZdeeqmo94svvlhUf8MNNxTVoz5s2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwimsDxo8fX1S/du3ayrWzZ88u6r1kyZKi+nnz5lWu7e/vL+p99913V661XdS7G7FlB5Ig7EAShB1IgrADSZQu7Nhne53tt2zvsv3FugYDUK/SvfGrJP0sIv7W9jhJ59UwE4AOqBx225+TNFfSnZIUEcckHatnLAB1K3kaf4mkQ5J+YPs122ts/583lFmyGegOJWHvlTRH0vcj4ipJRyWtGP4glmwGukNJ2PdJ2hcRL7evr1Mr/AC6UMmSzQckvW/7ivZN10naWctUAGpXujf+HkkD7T3x70j6+/KRAHRCUdgj4nVJZWcrAGgER9ABSRB2IAnOZz8LXHrppZVrn3rqqaLeixcvLqov6V86+9GjRyvXlv7cU6ZMKarvBLbsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATns5/jbr755qL6yy67rKh+6dKllWs3b95c1HvlypWVa999992i3vfff3/l2mnTphX1HglbdiAJwg4kQdiBJEqXbP6m7Tdt77D9tO3P1DUYgHpVDrvtaZK+Iak/ImZJ6pF0S12DAahX6dP4Xkmftd2r1trsvykfCUAnlKz19oGk70h6T9J+SYcj4oXhj2PJZqA7lDyNnyRpoVrrtE+VNN727cMfx5LNQHcoeRp/vaRfR8ShiPhI0nOSvlTPWADqVhL29yRdY/s821ZryeZd9YwFoG4lr9lflrRO0quS/qP9vR6raS4ANStdsvlbkr5V0ywAOogj6IAkCDuQBKe44rSuvPLKovr169dXrh0cHCzqXbLs8urVq4t6Dw0NVa4tPbV3JGzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAlHRGPN+vv7Y+vWrY31Q25jx46tXHv8+PGi3r291T8qYuPGjZVr77rrLg0NDflU97FlB5Ig7EAShB1I4lPDbvsJ2wdt7zjptsm2N9p+u/11UmfHBFDqTLbsP5Q0f9htKyRtiojLJW1qXwfQxT417BHxC0m/G3bzQklPti8/KWlRzXMBqFnV1+wXRsT+9uUDki4c6YEs2Qx0h+IddNF6o37EN+tZshnoDlXD/lvbUySp/fVgfSMB6ISqYR+UdGK5jcWSflzPOAA65Uzeenta0r9LusL2PttLJP2TpL+x/bak69vXAXSxTz2ANyJuHeGu62qeBUAHcQQdkARhB5JgyWac1vbt24vqn3322cq1r7zySlHv0tNUS8ycObNy7dy5cyvXTpw4ccT72LIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEpzPfhYYGhqqXLtq1aqi3hs2bCiqP3DgQFH9aBkzpmw7OGXKlFHrPeL37ch3BdB1CDuQBGEHkqi6ZPO3bb9le7vtDbb7OjsmgFJVl2zeKGlWRHxe0m5JK2ueC0DNKi3ZHBEvRMSJj+78paTpHZgNQI3qeM3+dUk/reH7AOigorDbvk/ScUkDp3kM67MDXaBy2G3fKekmSV9rr9F+SqzPDnSHSkfQ2Z4vabmkv4qIP9Q7EoBOqLpk86OSJkraaPt126s7PCeAQlWXbF7bgVkAdBBH0AFJEHYgCU5xPUMlp2oODIz4zuQZefTRRyvX7t27t6j32ay/v79y7QMPPFDUe8GCBUX1ncCWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5LwaT4Ytv5m9iFJ757mIX8m6b8aGofe9D4Xe18cEaf8GOdGw/5pbG+NiOqfOEBvetN7RDyNB5Ig7EAS3Rb2x+hNb3p3Rle9ZgfQOd22ZQfQIYQdSKIrwm57vu0h23tsr2iw70W2t9jeaftN2/c21fukGXpsv2b7+Yb79tleZ/st27tsf7HB3t9s/7532H7a9mc63O8J2wdt7zjptsm2N9p+u/11UoO9v93+vW+3vcF2Xyd6DzfqYbfdI+l7kr4iaaakW23PbKj9cUnLImKmpGsk/WODvU+4V9KuhntK0ipJP4uIP5f0F03NYHuapG9I6o+IWZJ6JN3S4bY/lDR/2G0rJG2KiMslbWpfb6r3RkmzIuLzknZLWtmh3p8w6mGX9AVJeyLinYg4JukZSQubaBwR+yPi1fbl36v1Bz+tid6SZHu6pK9KWtNUz3bfz0maq/YCnRFxLCL+u8EReiV91navpPMk/aaTzSLiF5J+N+zmhZKebF9+UtKipnpHxAsRcbx99ZeSpnei93DdEPZpkt4/6fo+NRi4E2zPkHSVpJcbbPtdtda5/2ODPSXpEkmHJP2g/RJije3xTTSOiA8kfUfSe5L2SzocES800XuYCyNif/vyAUkXjsIMkvR1ST9tolE3hH3U2Z4gab2kpRFxpKGeN0k6GBHbmug3TK+kOZK+HxFXSTqqzj2N/YT2a+OFav0PZ6qk8bZvb6L3SKL1/nPj70Hbvk+tl5JliwGeoW4I+weSLjrp+vT2bY2wPVatoA9ExHNN9ZX0ZUkLbO9V66XLPNs/aqj3Pkn7IuLEs5h1aoW/CddL+nVEHIqIjyQ9J+lLDfU+2W9tT5Gk9teDTTa3faekmyR9LRo62KUbwv4rSZfbvsT2OLV21gw20di21XrduisiHmmi5wkRsTIipkfEDLV+5s0R0cgWLiIOSHrf9hXtm66TtLOJ3mo9fb/G9nnt3/91Gp0dlIOSFrcvL5b046Ya256v1su3BRHxh6b6KiJG/Z+kG9XaK/mfku5rsO9fqvX0bbuk19v/bhyFn/+vJT3fcM/Zkra2f/Z/kTSpwd4PSnpL0g5J/yzpTzrc72m19g98pNazmiWS/lStvfBvS3pR0uQGe+9Raz/Vib+51U383jlcFkiiG57GA2gAYQeSIOxAEoQdSIKwA0kQdiAJwg4k8T/KH89MaF2UAQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Generally in tensorflow, and thus Keras, the first axis of a tensor is the samples axis - it tells us how many samples there are in the tensor. Due to memory-constraints, and the nature of parallelisation when using GPUs, we usually batch our data."
      ],
      "metadata": {
        "id": "Uza8I2US492B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_1 = train_images[:128]\n",
        "batch_2 = train_images[128:256]"
      ],
      "metadata": {
        "id": "xbr6F6aP4w3V"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "But we wouldn't want to type out batch_3...batch_4...batch_5, so we use a bit of maths to determine the nth batch."
      ],
      "metadata": {
        "id": "cS9PvVnP5WWl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "n = 3\n",
        "batch_n = train_images[128 * n:128 * (n+1)]"
      ],
      "metadata": {
        "id": "BAVo69lO5VZL"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.arange(3,128) + (128*(n+1))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mamHQerI6eZO",
        "outputId": "2f95f769-05c4-4700-d06e-0d9615ebaa1c"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527,\n",
              "       528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540,\n",
              "       541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553,\n",
              "       554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566,\n",
              "       567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579,\n",
              "       580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592,\n",
              "       593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605,\n",
              "       606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618,\n",
              "       619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631,\n",
              "       632, 633, 634, 635, 636, 637, 638, 639])"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Making tensors more concrete with common examples:\n",
        "vector data Rank-2 tensors of shape (samples, features) - i.e. data frame\n",
        "timeseries data Rank-3 tensors of shape (samples, timesteps, features)\n",
        "images - Rank-4 tensors of shape (samples, height, width, channels)\n",
        "video - Rank-5 tensors of shape (samples, frames, height, width, channels\n",
        "\n"
      ],
      "metadata": {
        "id": "qtOgJcqj6Bsy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tensor Operations\n",
        "\n",
        "In a previous MNIST implementation we stacked two dense neural layers on top of one another. We used a RELU activation in one, and a softmax in the second. The first layer takes a matrix as an input and returns another matrix - a new representation. Where W is a matrix, and b is a vector, the transformation is an affine transformation, the matrix multiplication of the input and W, plus b. As matrices are rank-2 tensors, we can see this as a tensor * tensor multiplication, followed by a matrix + vector addition of the matrix which is output by the tensor multiplciation."
      ],
      "metadata": {
        "id": "lY_NE1UU70Uo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The relu operation and the addition are element-wise operations - they are applied independently to each entry of the tensors. They can be imlemented naively as for-loops or they can be vectorised (they're open to parallelisation)"
      ],
      "metadata": {
        "id": "JkzsDtYq99J1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def naive_relu(x):\n",
        "  assert(len(x.shape) ==2)\n",
        "  x = x.copy()\n",
        "  for i in range(x.shape[0]):\n",
        "    for j in range(x.shape[1]):\n",
        "      x[i, j] = max(x[i, j], 0)\n",
        "  return x"
      ],
      "metadata": {
        "id": "xQzt79nB742X"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def naive_vector_addition(x, y ):\n",
        "  assert(len(x.shape)== 2)\n",
        "  assert(x.shape == y.shape)\n",
        "  x = x.copy()\n",
        "  for i in range(x.shape[0]):\n",
        "    for j in range(x.shape[1]):\n",
        "      x[i, j] += y[i, j]\n",
        "  return x"
      ],
      "metadata": {
        "id": "qOhs2p2p-MN6"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.array([\n",
        "              [2, 3, 1,2],\n",
        "              [3, 2, 1, 2]\n",
        "])\n",
        "y = x"
      ],
      "metadata": {
        "id": "BmQDx7dx-eo4"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "z = naive_vector_addition(x,y)\n"
      ],
      "metadata": {
        "id": "dLFChENt-qpg"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "naive_relu(x *-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qa0etIw4-sJh",
        "outputId": "be4f341f-ab76-4cab-b9b9-51b74fd37321"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 0, 0, 0],\n",
              "       [0, 0, 0, 0]])"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Or using the already-optimised numpy:"
      ],
      "metadata": {
        "id": "ReWIzgsL_nb7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "z = x + y\n",
        "z = np.maximum(z, 0.)"
      ],
      "metadata": {
        "id": "3LFeFd5Q_e7d"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "z"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5sycsFIy_rKV",
        "outputId": "21cfda69-8dd4-463a-c327-bdc031bda584"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[4., 6., 2., 4.],\n",
              "       [6., 4., 2., 4.]])"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This shows what the relu activation function does, either we take the maximum value of each element, or if that maximum value is below 0, we take 0. Now we can compare the speed of our naive implementations and their numpy counterparts"
      ],
      "metadata": {
        "id": "7tnlBD8H_03z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import time"
      ],
      "metadata": {
        "id": "aExdqlBx_v25"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.random.random((20, 100))\n",
        "y = np.random.random((20, 100))"
      ],
      "metadata": {
        "id": "v_Iyf3fMAE8u"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t0 = time.time()\n",
        "for _ in range(1000):\n",
        "  z = x + y\n",
        "  z = np.maximum(z, 0.)\n",
        "print(\"Took: {0: .2f} s\".format(time.time()-t0))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NsdThynaAJ4s",
        "outputId": "05febb82-503c-40f6-b426-e201fe75cba7"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Took:  0.02 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t0 = time.time()\n",
        "for _ in range(1000):\n",
        "  z = naive_vector_addition(x, y)\n",
        "  z = naive_relu(z)\n",
        "print(\"Took: {0: .2f} s\".format(time.time()- t0))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DILW2HdpAYBc",
        "outputId": "ac881b00-dcc0-4277-fbd4-6cdb1e05cbaf"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Took:  5.64 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "About 280* faster than the naive implementation! Next we'll look at broadcasting - which is an important means of extending tensor/matrix operations to objects which don't naturally have the same shape/dimensions, or shapes/dimensions that are compatible according to the rules of tensor/matrix operations, i.e. mxn matrix can only multiply with nxp matrices.\n",
        "\n"
      ],
      "metadata": {
        "id": "IRsZIHirAyj5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.random.random((32, 10))\n",
        "y = np.random.random((10, ))\n",
        "x.shape, y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EnKI45Y_gSM4",
        "outputId": "9363c709-3c48-4cf0-c54c-91aec9d4e8b6"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((32, 10), (10,))"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "What happens if we try to use our naive_vector_addition to add x + y? We draw an error, because the shapes of x and y are not the same. "
      ],
      "metadata": {
        "id": "TyOxE1D-g5OU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "naive_vector_addition(x, y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "id": "GqeqWwjygOCQ",
        "outputId": "c29f1109-1167-4804-aad6-affa4ce80b1d"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AssertionError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-36-31042d8307f2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnaive_vector_addition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-23-c7255ef86d85>\u001b[0m in \u001b[0;36mnaive_vector_addition\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mnaive_vector_addition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m   \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m   \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m   \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAssertionError\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "However, if we use numpy's implementation, the shape of y is broadcast to be compatible with the shape of x, so y is essentially replicated 32 times!"
      ],
      "metadata": {
        "id": "QxvVEa0bhB6r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x + y"
      ],
      "metadata": {
        "id": "UGsFzSKRg_Tw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y = np.expand_dims(y, axis = 0)\n",
        "y.shape # we can see that y is now shape (1, 10) whereas before it was (10,)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uS8K2xT_Avhv",
        "outputId": "12863203-2ecb-4d6a-d5dc-384090ca3322"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 10)"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y = np.concatenate([y]*32, axis = 0)"
      ],
      "metadata": {
        "id": "3JXbp2sDiUIm"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iPnu7DFIiaaX",
        "outputId": "9f7f0764-6652-459c-aa23-1a302984b6f6"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(32, 10)"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y = np.concatenate([y] * 32, axis = 0)"
      ],
      "metadata": {
        "id": "3PYr-QkMhapr"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x.shape, y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9MY3WYkDhuNc",
        "outputId": "f746aa2d-b706-469d-88bb-13494061a349"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((32, 10), (32, 10))"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def naive_add_matrix_and_vector(x, y):\n",
        "  assert len(x.shape) == 2\n",
        "  assert len(y.shape) ==1\n",
        "  assert x.shape[1] == y.shape[0]\n",
        "  x = x.copy()\n",
        "  for i in range(x.shape[0]):\n",
        "    for j in range(x.shape[1]):\n",
        "      x[i, j] += y[j]\n",
        "  return x"
      ],
      "metadata": {
        "id": "EKuEAPGQhpzz"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "naive_vector_addition(x, y) #now runs with our naive vector addition"
      ],
      "metadata": {
        "id": "b14SQC4bmHqu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "naive_vector_addition(x, y) == x + y #does the same thing (just in much less time!)"
      ],
      "metadata": {
        "id": "Pyjk7SRwmS2Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we can look at broadcasting with np.maximum()"
      ],
      "metadata": {
        "id": "LJyQtvJInDph"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.random.random((64, 3, 32, 10))\n",
        "y = np.random.random((32, 10))"
      ],
      "metadata": {
        "id": "6U2Nbkk5mMHg"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x.shape, y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-IUIcGE1mr0h",
        "outputId": "3ee9af61-d9e6-4edf-a523-3ad28b09f203"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((64, 3, 32, 10), (32, 10))"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(x), type(y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g6cOzb-Hmo0l",
        "outputId": "d6e93927-7af1-4220-ee72-7aea8bd2dae0"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(numpy.ndarray, numpy.ndarray)"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "z = np.maximum(x, y)"
      ],
      "metadata": {
        "id": "Tmki-ChVmu_b"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "z.shape == x.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MvJWLVXPm4hB",
        "outputId": "7cb87f1f-293f-477f-cee2-f2141b1382ec"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "IwDy1YcNm5Y5"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}