Building your Deep Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d7551fe0-42c6-43a7-96b6-734181361bb9",
   "metadata": {},
   "source": [
    "# Building your Deep Neural Network: Step by Step"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6deb5d2-a035-468d-b527-6c8e2a7720e7",
   "metadata": {},
   "source": [
    "## 1 - Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d47e52a7-affa-4fde-bb2c-3b01a61710e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "import numpy as np  # For numerical operations and matrix manipulations\n",
    "import h5py  # For working with HDF5 file formats (used in storing large datasets)\n",
    "import matplotlib.pyplot as plt  # For data visualization\n",
    "from dnn_utils_v2 import sigmoid, sigmoid_backward, relu, relu_backward  # Activation functions and their gradients\n",
    "\n",
    "# Set random seed for reproducibility\n",
    "np.random.seed(1)\n",
    "\n",
    "# Configure Matplotlib parameters for plots\n",
    "plt.rcParams['figure.figsize'] = (5.0, 4.0)  # Default plot size\n",
    "plt.rcParams['image.interpolation'] = 'nearest'  # No interpolation for image display\n",
    "plt.rcParams['image.cmap'] = 'gray'  # Set default color map to grayscale\n",
    "\n",
    "# Test case import - these functions likely contain predefined test data for validation\n",
    "from testCases_v4a import *  # Importing test cases for validation of neural network models\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59fed42e-7fdd-4ba2-894e-9cfff4582fc6",
   "metadata": {},
   "source": [
    "## 3 - Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf37af75-4267-492e-8891-69930ca140ea",
   "metadata": {},
   "source": [
    "### 3.1 - 2-layer Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9eb91fdd-f672-4f83-8920-d39f12178494",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np  # Import numpy for numerical computations\n",
    "\n",
    "# FUNCTION: initialize_parameters\n",
    "def initialize_parameters(n_x, n_h, n_y):\n",
    "    \"\"\"\n",
    "    Initialize the parameters of a two-layer neural network.\n",
    "    \n",
    "    Arguments:\n",
    "    n_x -- size of the input layer (number of features)\n",
    "    n_h -- size of the hidden layer (number of hidden units)\n",
    "    n_y -- size of the output layer (number of output classes)\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- Python dictionary containing:\n",
    "                  W1 -- weight matrix of shape (n_h, n_x)\n",
    "                  b1 -- bias vector of shape (n_h, 1)\n",
    "                  W2 -- weight matrix of shape (n_y, n_h)\n",
    "                  b2 -- bias vector of shape (n_y, 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    # Set a random seed for reproducibility\n",
    "    np.random.seed(1)\n",
    "    \n",
    "    # Initialize weight matrices with small random values and bias vectors with zeros\n",
    "    W1 = np.random.randn(n_h, n_x) * 0.01  # Weight matrix for layer 1\n",
    "    b1 = np.zeros((n_h, 1))  # Bias vector for layer 1\n",
    "    W2 = np.random.randn(n_y, n_h) * 0.01  # Weight matrix for layer 2\n",
    "    b2 = np.zeros((n_y, 1))  # Bias vector for layer 2\n",
    "\n",
    "    # Sanity check: Ensure the dimensions of the matrices and vectors are correct\n",
    "    assert W1.shape == (n_h, n_x), f\"W1 shape should be {(n_h, n_x)}, but got {W1.shape}\"\n",
    "    assert b1.shape == (n_h, 1), f\"b1 shape should be {(n_h, 1)}, but got {b1.shape}\"\n",
    "    assert W2.shape == (n_y, n_h), f\"W2 shape should be {(n_y, n_h)}, but got {W2.shape}\"\n",
    "    assert b2.shape == (n_y, 1), f\"b2 shape should be {(n_y, 1)}, but got {b2.shape}\"\n",
    "\n",
    "    # Store parameters in a dictionary\n",
    "    parameters = {\n",
    "        \"W1\": W1,\n",
    "        \"b1\": b1,\n",
    "        \"W2\": W2,\n",
    "        \"b2\": b2\n",
    "    }\n",
    "\n",
    "    return parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c75310a2-9d7f-4b20-b150-8bfe748f3af5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1 = [[ 0.01624345 -0.00611756 -0.00528172]\n",
      " [-0.01072969  0.00865408 -0.02301539]]\n",
      "b1 = [[0.]\n",
      " [0.]]\n",
      "W2 = [[ 0.01744812 -0.00761207]]\n",
      "b2 = [[0.]]\n"
     ]
    }
   ],
   "source": [
    "# Initialize the parameters with the given layer sizes\n",
    "parameters = initialize_parameters(3, 2, 1)\n",
    "\n",
    "# Print the values of the initialized parameters\n",
    "print(\"W1 =\", parameters[\"W1\"])  # Weight matrix for layer 1\n",
    "print(\"b1 =\", parameters[\"b1\"])  # Bias vector for layer 1\n",
    "print(\"W2 =\", parameters[\"W2\"])  # Weight matrix for layer 2\n",
    "print(\"b2 =\", parameters[\"b2\"])  # Bias vector for layer 2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62e19524-6c4a-4248-b571-cef3f596eb53",
   "metadata": {},
   "source": [
    "## 3.2 - L-layer Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aea8c4ef-74ba-4305-a274-f21745a71040",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np  # Ensure numpy is imported\n",
    "\n",
    "# FUNCTION: initialize_parameters_deep\n",
    "def initialize_parameters_deep(layer_dims):\n",
    "    \"\"\"\n",
    "    Initialize parameters for an L-layer deep neural network.\n",
    "    \n",
    "    Arguments:\n",
    "    layer_dims -- list containing the dimensions of each layer in the network (including input and output layers)\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- dictionary containing the initialized weights and biases for each layer:\n",
    "                  W1, b1, ..., WL, bL:\n",
    "                  Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n",
    "                  bl -- bias vector of shape (layer_dims[l], 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    np.random.seed(3)  # For reproducibility of random values\n",
    "    parameters = {}  # Dictionary to store the weight and bias matrices\n",
    "    L = len(layer_dims)  # Total number of layers in the network (input + hidden + output)\n",
    "\n",
    "    # Loop through each layer to initialize weights and biases\n",
    "    for l in range(1, L):\n",
    "        # Initialize weight matrix with small random values and bias vector with zeros\n",
    "        parameters[f'W{l}'] = np.random.randn(layer_dims[l], layer_dims[l-1]) * 0.01\n",
    "        parameters[f'b{l}'] = np.zeros((layer_dims[l], 1))\n",
    "\n",
    "        # Ensure the dimensions of weights and biases are correct\n",
    "        assert parameters[f'W{l}'].shape == (layer_dims[l], layer_dims[l-1]), f\"Shape mismatch in W{l}\"\n",
    "        assert parameters[f'b{l}'].shape == (layer_dims[l], 1), f\"Shape mismatch in b{l}\"\n",
    "\n",
    "    return parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "863d8224-85d9-4aaa-86d7-d79afef516b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1 = [[ 0.01788628  0.0043651   0.00096497 -0.01863493 -0.00277388]\n",
      " [-0.00354759 -0.00082741 -0.00627001 -0.00043818 -0.00477218]\n",
      " [-0.01313865  0.00884622  0.00881318  0.01709573  0.00050034]\n",
      " [-0.00404677 -0.0054536  -0.01546477  0.00982367 -0.01101068]]\n",
      "b1 = [[0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]]\n",
      "W2 = [[-0.01185047 -0.0020565   0.01486148  0.00236716]\n",
      " [-0.01023785 -0.00712993  0.00625245 -0.00160513]\n",
      " [-0.00768836 -0.00230031  0.00745056  0.01976111]]\n",
      "b2 = [[0.]\n",
      " [0.]\n",
      " [0.]]\n"
     ]
    }
   ],
   "source": [
    "# Initialize parameters for a deep neural network with layer sizes [5, 4, 3]\n",
    "parameters = initialize_parameters_deep([5, 4, 3])\n",
    "\n",
    "# Print initialized parameters for the first two layers\n",
    "print(\"W1 =\", parameters[\"W1\"])  # Weight matrix for layer 1\n",
    "print(\"b1 =\", parameters[\"b1\"])  # Bias vector for layer 1\n",
    "print(\"W2 =\", parameters[\"W2\"])  # Weight matrix for layer 2\n",
    "print(\"b2 =\", parameters[\"b2\"])  # Bias vector for layer 2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "363f01af-5062-4bbd-80e0-c15e2eb5df62",
   "metadata": {},
   "source": [
    "## 4 - Forward propagation module"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daccd3b3-96b0-472b-8787-716a9912704a",
   "metadata": {},
   "source": [
    "### 4.1 - Linear Forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f94fe359-8821-4da5-b537-cdc69795a2ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np  # Ensure numpy is imported\n",
    "\n",
    "# FUNCTION: linear_forward\n",
    "def linear_forward(A, W, b):\n",
    "    \"\"\"\n",
    "    Implement the linear part of a layer's forward propagation.\n",
    "\n",
    "    Arguments:\n",
    "    A -- Activations from the previous layer (or input data), of shape (size of previous layer, number of examples)\n",
    "    W -- Weights matrix, numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- Bias vector, numpy array of shape (size of the current layer, 1)\n",
    "\n",
    "    Returns:\n",
    "    Z -- Pre-activation parameter (linear combination), of shape (size of current layer, number of examples)\n",
    "    cache -- Tuple containing \"A\", \"W\", and \"b\" for efficient backward pass computation\n",
    "    \"\"\"\n",
    "    \n",
    "    # Calculate the pre-activation parameter Z using the formula: Z = W.A + b\n",
    "    Z = np.dot(W, A) + b\n",
    "\n",
    "    # Ensure that the shape of Z is as expected (size of current layer, number of examples)\n",
    "    assert Z.shape == (W.shape[0], A.shape[1]), \"Shape mismatch: Z has incorrect dimensions\"\n",
    "    \n",
    "    # Cache the inputs for backward propagation\n",
    "    cache = (A, W, b)\n",
    "    \n",
    "    return Z, cache\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ca2c076d-86fb-44b3-ba30-85c0930a9559",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z = [[ 3.26295337 -1.23429987]]\n"
     ]
    }
   ],
   "source": [
    "A, W, b = linear_forward_test_case()\n",
    "\n",
    "Z, linear_cache = linear_forward(A, W, b)\n",
    "\n",
    "print(\"Z =\", Z)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93e9e6f6-d712-483d-9e0d-6ba0d38e47d9",
   "metadata": {},
   "source": [
    "### 4.2 - Linear Activation Forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "20c721af-c021-4dfe-84c9-5fc9ceed4836",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FUNCTION: linear_activation_forward\n",
    "def linear_activation_forward(A_prev, W, b, activation):\n",
    "    \"\"\"\n",
    "    Implement the forward propagation for the linear -> activation layer.\n",
    "\n",
    "    Arguments:\n",
    "    A_prev -- Activations from previous layer (or input data), of shape (size of previous layer, number of examples)\n",
    "    W -- Weights matrix, numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- Bias vector, numpy array of shape (size of the current layer, 1)\n",
    "    activation -- The activation function to be used, either \"sigmoid\" or \"relu\"\n",
    "\n",
    "    Returns:\n",
    "    A -- The output of the activation function (post-activation value)\n",
    "    cache -- A tuple containing both linear and activation caches, stored for backpropagation\n",
    "    \"\"\"\n",
    "    \n",
    "    # Compute linear forward followed by the specified activation function\n",
    "    if activation == \"sigmoid\":\n",
    "        # Perform linear forward and apply sigmoid activation\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)  # Z is the pre-activation parameter\n",
    "        A, activation_cache = sigmoid(Z)                # A is the post-activation using sigmoid\n",
    "        \n",
    "    elif activation == \"relu\":\n",
    "        # Perform linear forward and apply ReLU activation\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)  # Z is the pre-activation parameter\n",
    "        A, activation_cache = relu(Z)                   # A is the post-activation using ReLU\n",
    "    \n",
    "    # Ensure the output's shape matches the expected dimensions\n",
    "    assert A.shape == (W.shape[0], A_prev.shape[1]), \"Shape mismatch: A has incorrect dimensions\"\n",
    "\n",
    "    # Store both linear cache and activation cache for backward pass\n",
    "    cache = (linear_cache, activation_cache)\n",
    "\n",
    "    return A, cache\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d18be8fb-8395-4eab-bfa0-7e4adf61018b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "With sigmoid: A = [[0.96890023 0.11013289]]\n",
      "With ReLU: A = [[3.43896131 0.        ]]\n"
     ]
    }
   ],
   "source": [
    "# Test case setup: Load sample data for A_prev, W, and b\n",
    "A_prev, W, b = linear_activation_forward_test_case()\n",
    "\n",
    "# Perform forward propagation with sigmoid activation\n",
    "A, linear_activation_cache = linear_activation_forward(A_prev, W, b, activation=\"sigmoid\")\n",
    "print(\"With sigmoid: A =\", A)\n",
    "\n",
    "# Perform forward propagation with ReLU activation\n",
    "A, linear_activation_cache = linear_activation_forward(A_prev, W, b, activation=\"relu\")\n",
    "print(\"With ReLU: A =\", A)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ded095c6-aa31-4658-ba06-fd44dd9a2a33",
   "metadata": {},
   "source": [
    "### 4.3 - L-Layer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "eb24eeb1-201d-4226-80eb-2ad6f4e87789",
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_forward(X, parameters):\n",
    "    \"\"\"\n",
    "    Implements forward propagation for the [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID computation.\n",
    "    \n",
    "    Arguments:\n",
    "    X -- input data, numpy array of shape (input size, number of examples)\n",
    "    parameters -- dictionary containing the initialized weights and biases\n",
    "    \n",
    "    Returns:\n",
    "    AL -- the output of the final layer (post-activation)\n",
    "    caches -- list of caches containing every cache from linear_activation_forward for backpropagation\n",
    "    \"\"\"\n",
    "    caches = []  # List to store all caches for backpropagation\n",
    "    A = X  # Set input data as activation for the first layer\n",
    "    L = len(parameters) // 2  # Number of layers in the network (W1, b1, ..., WL, bL)\n",
    "\n",
    "    # [LINEAR -> RELU]*(L-1): Forward propagate through the first (L-1) layers using ReLU activation\n",
    "    for l in range(1, L):\n",
    "        A_prev = A  # Activation from the previous layer\n",
    "        # Forward propagate through current layer l with ReLU activation\n",
    "        A, cache = linear_activation_forward(A_prev, parameters[f'W{l}'], parameters[f'b{l}'], activation=\"relu\")\n",
    "        caches.append(cache)  # Store the cache for backpropagation\n",
    "    \n",
    "    # LINEAR -> SIGMOID: Forward propagate through the last layer using Sigmoid activation\n",
    "    AL, cache = linear_activation_forward(A, parameters[f'W{L}'], parameters[f'b{L}'], activation=\"sigmoid\")\n",
    "    caches.append(cache)  # Store the cache for the final layer\n",
    "    \n",
    "    # Ensure the output has the correct shape\n",
    "    assert AL.shape == (1, X.shape[1]), f\"Expected AL to be of shape (1, {X.shape[1]})\"\n",
    "    \n",
    "    return AL, caches  # Return the final output and all caches for backpropagation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d070c898-9508-4c62-a523-16661065e41a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AL = [[0.03921668 0.70498921 0.19734387 0.04728177]]\n",
      "Length of caches list = 3\n"
     ]
    }
   ],
   "source": [
    "# Example usage of the L_model_forward function\n",
    "\n",
    "# Load the test case with input data X and initialized parameters\n",
    "X, parameters = L_model_forward_test_case_2hidden()\n",
    "\n",
    "# Perform forward propagation through the network\n",
    "AL, caches = L_model_forward(X, parameters)\n",
    "\n",
    "# Output the final activation (AL) and the number of cached layers\n",
    "print(f\"AL = {AL}\")\n",
    "print(f\"Length of caches list = {len(caches)}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9372c53a-494f-48c0-85dd-cdd15535cb74",
   "metadata": {},
   "source": [
    "## 5 - Cost function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "01cd8391-4d24-4d69-896f-1e62cc0dc6d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def compute_cost(AL, Y):\n",
    "    \"\"\"\n",
    "    Compute the cross-entropy cost for binary classification.\n",
    "\n",
    "    Arguments:\n",
    "    AL -- probability vector from the model's output, shape (1, number of examples)\n",
    "    Y -- true label vector (1 if true, 0 if false), shape (1, number of examples)\n",
    "\n",
    "    Returns:\n",
    "    cost -- cross-entropy cost as a scalar value\n",
    "    \"\"\"\n",
    "    \n",
    "    m = Y.shape[1]  # Number of examples\n",
    "    \n",
    "    # Compute the cross-entropy cost\n",
    "    cost = (-1 / m) * (np.dot(Y, np.log(AL).T) + np.dot(1 - Y, np.log(1 - AL).T))\n",
    "    \n",
    "    # Ensure the cost is a scalar (e.g., turn [[17]] into 17)\n",
    "    cost = np.squeeze(cost)\n",
    "    \n",
    "    # Validate the cost is a scalar value\n",
    "    assert cost.ndim == 0, \"Cost should be a scalar.\"\n",
    "\n",
    "    return cost\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6ece2414-4fe3-47d9-a6b2-96a807524ea0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost: 0.2797765635793422\n"
     ]
    }
   ],
   "source": [
    "# Example test case data (for illustration purposes)\n",
    "Y, AL = compute_cost_test_case()\n",
    "\n",
    "# Compute and display the cost\n",
    "cost = compute_cost(AL, Y)\n",
    "print(f\"Cost: {cost}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9539f25e-9657-4fec-8ef2-ef74e12b4208",
   "metadata": {},
   "source": [
    "## 6 - Backward propagation module"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ced6eb5a-1c44-4125-a807-30aaa0eafc3d",
   "metadata": {},
   "source": [
    "### 6.1 - Linear backward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0afa8907-78b4-450d-8816-ada656d498bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache):\n",
    "    \"\"\"\n",
    "    Implement the linear portion of backward propagation for a single layer.\n",
    "\n",
    "    Arguments:\n",
    "    dZ -- Gradient of the cost with respect to the linear output of the current layer.\n",
    "    cache -- Tuple containing (A_prev, W, b) from the forward propagation of the current layer.\n",
    "\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation of the previous layer (same shape as A_prev).\n",
    "    dW -- Gradient of the cost with respect to W of the current layer (same shape as W).\n",
    "    db -- Gradient of the cost with respect to b of the current layer (same shape as b).\n",
    "    \"\"\"\n",
    "    A_prev, W, _ = cache  # Unpacking the cache (b is unused here, so it's replaced with '_')\n",
    "    m = A_prev.shape[1]\n",
    "\n",
    "    # Calculate gradients\n",
    "    dW = (1 / m) * np.dot(dZ, A_prev.T)  # Gradient of W\n",
    "    db = (1 / m) * np.sum(dZ, axis=1, keepdims=True)  # Gradient of b\n",
    "    dA_prev = np.dot(W.T, dZ)  # Gradient of the activation from the previous layer\n",
    "\n",
    "    # Ensure the shapes are correct\n",
    "    assert dA_prev.shape == A_prev.shape, \"Shape of dA_prev is incorrect\"\n",
    "    assert dW.shape == W.shape, \"Shape of dW is incorrect\"\n",
    "    assert db.shape == (W.shape[0], 1), \"Shape of db is incorrect\"\n",
    "    \n",
    "    return dA_prev, dW, db\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9156e726-e0a0-4183-89ba-7a398dab9320",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dA_prev = [[-1.15171336  0.06718465 -0.3204696   2.09812712]\n",
      " [ 0.60345879 -3.72508701  5.81700741 -3.84326836]\n",
      " [-0.4319552  -1.30987417  1.72354705  0.05070578]\n",
      " [-0.38981415  0.60811244 -1.25938424  1.47191593]\n",
      " [-2.52214926  2.67882552 -0.67947465  1.48119548]]\n",
      "dW = [[ 0.07313866 -0.0976715  -0.87585828  0.73763362  0.00785716]\n",
      " [ 0.85508818  0.37530413 -0.59912655  0.71278189 -0.58931808]\n",
      " [ 0.97913304 -0.24376494 -0.08839671  0.55151192 -0.10290907]]\n",
      "db = [[-0.14713786]\n",
      " [-0.11313155]\n",
      " [-0.13209101]]\n"
     ]
    }
   ],
   "source": [
    "# Set up test inputs for backward propagation\n",
    "dZ, linear_cache = linear_backward_test_case()\n",
    "\n",
    "# Perform the backward linear step\n",
    "dA_prev, dW, db = linear_backward(dZ, linear_cache)\n",
    "\n",
    "# Display the results\n",
    "print(f\"dA_prev = {dA_prev}\")\n",
    "print(f\"dW = {dW}\")\n",
    "print(f\"db = {db}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28ca195e-c2e0-48b9-a98c-13f37fb1ad32",
   "metadata": {},
   "source": [
    "### 6.2 - Linear-Activation backward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3236d671-68a1-4783-830f-a5a0447a3732",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_activation_backward(dA, cache, activation):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the LINEAR->ACTIVATION layer.\n",
    "\n",
    "    Arguments:\n",
    "    dA -- Post-activation gradient for the current layer (l).\n",
    "    cache -- Tuple of values (linear_cache, activation_cache) for efficient backward propagation.\n",
    "    activation -- Activation function used in this layer: \"sigmoid\" or \"relu\".\n",
    "\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation of the previous layer (l-1).\n",
    "    dW -- Gradient of the cost with respect to W (current layer l).\n",
    "    db -- Gradient of the cost with respect to b (current layer l).\n",
    "    \"\"\"\n",
    "    linear_cache, activation_cache = cache\n",
    "\n",
    "    # Calculate dZ based on the activation function\n",
    "    if activation == \"relu\":\n",
    "        dZ = relu_backward(dA, activation_cache)\n",
    "    elif activation == \"sigmoid\":\n",
    "        dZ = sigmoid_backward(dA, activation_cache)\n",
    "    else:\n",
    "        raise ValueError(\"Invalid activation function specified.\")\n",
    "\n",
    "    # Perform linear backward step\n",
    "    dA_prev, dW, db = linear_backward(dZ, linear_cache)\n",
    "\n",
    "    return dA_prev, dW, db\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d2122598-438d-4d34-819a-8f33e82f6c80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Activation: sigmoid\n",
      "dA_prev = [[ 0.11017994  0.01105339]\n",
      " [ 0.09466817  0.00949723]\n",
      " [-0.05743092 -0.00576154]]\n",
      "dW = [[ 0.10266786  0.09778551 -0.01968084]]\n",
      "db = [[-0.05729622]]\n",
      "\n",
      "Activation: relu\n",
      "dA_prev = [[ 0.44090989  0.        ]\n",
      " [ 0.37883606  0.        ]\n",
      " [-0.2298228   0.        ]]\n",
      "dW = [[ 0.44513824  0.37371418 -0.10478989]]\n",
      "db = [[-0.20837892]]\n"
     ]
    }
   ],
   "source": [
    "# Set up test inputs for the linear activation backward function\n",
    "dAL, linear_activation_cache = linear_activation_backward_test_case()\n",
    "\n",
    "# Test backward propagation with sigmoid activation\n",
    "dA_prev, dW, db = linear_activation_backward(dAL, linear_activation_cache, activation=\"sigmoid\")\n",
    "print(\"Activation: sigmoid\")\n",
    "print(f\"dA_prev = {dA_prev}\")\n",
    "print(f\"dW = {dW}\")\n",
    "print(f\"db = {db}\\n\")\n",
    "\n",
    "# Test backward propagation with ReLU activation\n",
    "dA_prev, dW, db = linear_activation_backward(dAL, linear_activation_cache, activation=\"relu\")\n",
    "print(\"Activation: relu\")\n",
    "print(f\"dA_prev = {dA_prev}\")\n",
    "print(f\"dW = {dW}\")\n",
    "print(f\"db = {db}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a1287c6-c41f-4324-937a-7f3b481537a4",
   "metadata": {},
   "source": [
    "### 6.3 - L-Model Backward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "bd421858-0c81-49ae-9fba-91f7cc9951fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_backward(AL, Y, caches):\n",
    "    \"\"\"\n",
    "    Implements the backward propagation for the entire network, including the linear and activation layers.\n",
    "    \n",
    "    Arguments:\n",
    "    AL -- Probability vector from forward propagation (output layer)\n",
    "    Y -- True labels vector (same shape as AL)\n",
    "    caches -- List of caches from forward propagation, where each cache contains:\n",
    "              - The cache for the linear and activation parts of each layer\n",
    "    \n",
    "    Returns:\n",
    "    grads -- A dictionary containing gradients for each layer:\n",
    "             grads[f\"dA{l}\"] -> Gradient of the cost with respect to activations from layer l\n",
    "             grads[f\"dW{l}\"] -> Gradient of the cost with respect to weights from layer l\n",
    "             grads[f\"db{l}\"] -> Gradient of the cost with respect to biases from layer l\n",
    "    \"\"\"\n",
    "    \n",
    "    grads = {}  # Initialize dictionary to store gradients\n",
    "    L = len(caches)  # Number of layers in the neural network\n",
    "    m = AL.shape[1]  # Number of examples\n",
    "    Y = Y.reshape(AL.shape)  # Reshape Y to ensure the same shape as AL\n",
    "    \n",
    "    # Compute the gradient of the loss with respect to AL using binary cross-entropy loss\n",
    "    dAL = -(np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "\n",
    "    # Backpropagation for the final layer (layer L), which uses a sigmoid activation function\n",
    "    current_cache = caches[L - 1]  # Retrieve the cache for the last layer\n",
    "    grads[f\"dA{L-1}\"], grads[f\"dW{L}\"], grads[f\"db{L}\"] = linear_activation_backward(dAL, current_cache, activation=\"sigmoid\")\n",
    "\n",
    "    # Loop over the remaining layers in reverse order, from layer L-2 to layer 0 (which use ReLU activations)\n",
    "    for l in reversed(range(L - 1)):\n",
    "        current_cache = caches[l]  # Retrieve the cache for the current layer\n",
    "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[f\"dA{l+1}\"], current_cache, activation=\"relu\")\n",
    "        grads[f\"dA{l}\"] = dA_prev_temp  # Store gradient for activation\n",
    "        grads[f\"dW{l+1}\"] = dW_temp     # Store gradient for weights\n",
    "        grads[f\"db{l+1}\"] = db_temp     # Store gradient for biases\n",
    "\n",
    "    return grads\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "13d15e81-0c05-46f3-b525-9321e054f586",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dW1 = [[0.41010002 0.07807203 0.13798444 0.10502167]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.05283652 0.01005865 0.01777766 0.0135308 ]]\n",
      "db1 = [[-0.22007063]\n",
      " [ 0.        ]\n",
      " [-0.02835349]]\n",
      "dA1 = [[ 0.12913162 -0.44014127]\n",
      " [-0.14175655  0.48317296]\n",
      " [ 0.01663708 -0.05670698]]\n"
     ]
    }
   ],
   "source": [
    "# Test data for backward propagation\n",
    "AL, Y_assess, caches = L_model_backward_test_case()\n",
    "\n",
    "# Perform backward propagation using the computed AL and Y_assess\n",
    "grads = L_model_backward(AL, Y_assess, caches)\n",
    "\n",
    "# Function to print gradients\n",
    "print_grads(grads)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b86ad594-a9b1-4d35-8763-e07784fccade",
   "metadata": {},
   "source": [
    "### 6.4 - Update parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "bac02b42-2542-4915-90ef-1a7e751ee040",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(parameters, grads, learning_rate):\n",
    "    \"\"\"\n",
    "    Update parameters using gradient descent.\n",
    "\n",
    "    Arguments:\n",
    "    parameters -- Python dictionary containing parameters:\n",
    "                  \"W1\", \"b1\", ..., \"WL\", \"bL\" for each layer L in the network\n",
    "    grads -- Python dictionary containing gradients:\n",
    "             \"dW1\", \"db1\", ..., \"dWL\", \"dbL\" from backpropagation\n",
    "    learning_rate -- Scalar, the learning rate for gradient descent\n",
    "\n",
    "    Returns:\n",
    "    parameters -- Python dictionary with updated parameters after gradient descent:\n",
    "                  parameters[\"W1\"], parameters[\"b1\"], ..., parameters[\"WL\"], parameters[\"bL\"]\n",
    "    \"\"\"\n",
    "\n",
    "    L = len(parameters) // 2  # Number of layers in the neural network\n",
    "    \n",
    "    # Update parameters for each layer\n",
    "    for l in range(1, L + 1):\n",
    "        # Update weights and biases using gradient descent update rule\n",
    "        parameters[f\"W{l}\"] -= learning_rate * grads[f\"dW{l}\"]\n",
    "        parameters[f\"b{l}\"] -= learning_rate * grads[f\"db{l}\"]\n",
    "\n",
    "    return parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "ed684e30-7ee3-4abf-893b-103c79ea753e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1 = [[-0.59562069 -0.09991781 -2.14584584  1.82662008]\n",
      " [-1.76569676 -0.80627147  0.51115557 -1.18258802]\n",
      " [-1.0535704  -0.86128581  0.68284052  2.20374577]]\n",
      "b1 = [[-0.04659241]\n",
      " [-1.28888275]\n",
      " [ 0.53405496]]\n",
      "W2 = [[-0.55569196  0.0354055   1.32964895]]\n",
      "b2 = [[-0.84610769]]\n"
     ]
    }
   ],
   "source": [
    "# Test case to check update_parameters function\n",
    "parameters, grads = update_parameters_test_case()\n",
    "\n",
    "# Update parameters using gradient descent with a learning rate of 0.1\n",
    "parameters = update_parameters(parameters, grads, learning_rate=0.1)\n",
    "\n",
    "# Printing updated parameters for W1, b1, W2, and b2 using f-strings for clarity\n",
    "print(f\"W1 = {parameters['W1']}\")\n",
    "print(f\"b1 = {parameters['b1']}\")\n",
    "print(f\"W2 = {parameters['W2']}\")\n",
    "print(f\"b2 = {parameters['b2']}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99cc1e36-7384-431b-af6e-43d57218bd75",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}