3b_validate_and_compute_PQ.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook to check post processing pipeline. Here, inference is ran of the whole slide validation image data (dtv2) and PQ is computed for the given model\n",
    "\n",
    "USE THIS WHEN VALIDATING ON PATIENT LEVEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name experiment\n",
    "experiment_name = \"exp-1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 883,
     "status": "ok",
     "timestamp": 1578784778530,
     "user": {
      "displayName": "Md Hasib Zunair 1320262643",
      "photoUrl": "",
      "userId": "12069756592370329757"
     },
     "user_tz": 300
    },
    "id": "PqrxTSb8pEXX",
    "outputId": "28b38dc5-4bdd-45f8-af78-6f324c136b15"
   },
   "outputs": [],
   "source": [
    "# Import libs\n",
    "import os \n",
    "import time\n",
    "import cv2\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import skimage.draw\n",
    "import random\n",
    "import keras\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors\n",
    "import scipy.io as sio\n",
    "import scipy\n",
    "import scipy.ndimage\n",
    "from PIL import Image\n",
    "from skimage.transform import resize\n",
    "from scipy import ndimage as ndi\n",
    "from skimage.segmentation import watershed\n",
    "from skimage.feature import peak_local_max\n",
    "\n",
    "from PIL import Image, ImagePalette\n",
    "NUCLEI_PALETTE = ImagePalette.random()\n",
    "\n",
    "\n",
    "def create_directory(directory):\n",
    "    '''\n",
    "    Creates a new folder in the specified directory if the folder doesn't exist.\n",
    "    INPUT\n",
    "        directory: Folder to be created, called as \"folder/\".\n",
    "    OUTPUT\n",
    "        New folder in the current directory.\n",
    "    '''\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "\n",
    "    \n",
    "# Define paths\n",
    "dataset_name = \"test_images\" \n",
    "base_path = os.path.abspath(\".\")\n",
    "\n",
    "# Directory of images to run detection on\n",
    "TRAIN_IMAGES =  os.path.join(base_path, \"dataset\", \"data_processedv2\", \"val\", \"images/\")\n",
    "TRAIN_MASKS = os.path.join(base_path, \"dataset\", \"data_processedv2\", \"val\", \"masks/\")\n",
    "\n",
    "# Directory of images to run detection on\n",
    "VALID_IMAGES =  os.path.join(base_path, \"dataset\", \"data_processedv2\", \"val\", \"images/\")\n",
    "VALID_MASKS = os.path.join(base_path, \"dataset\", \"data_processedv2\", \"val\", \"masks/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_IMAGES, TRAIN_MASKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VALID_IMAGES, VALID_MASKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad(img, pad_size=96):\n",
    "    \"\"\"\n",
    "    Load image from a given path and pad it on the sides, so that eash side is divisible by 96 (network requirement)\n",
    "    if pad = True:\n",
    "        returns image as numpy.array, tuple with padding in pixels as(x_min_pad, y_min_pad, x_max_pad, y_max_pad)\n",
    "    else:\n",
    "        returns image as numpy.array\n",
    "    \"\"\"\n",
    "\n",
    "    if pad_size == 0:\n",
    "        return img\n",
    "\n",
    "    height, width = img.shape[:2]\n",
    "\n",
    "    if height % pad_size == 0:\n",
    "        y_min_pad = 0\n",
    "        y_max_pad = 0\n",
    "    else:\n",
    "        y_pad = pad_size - height % pad_size\n",
    "        y_min_pad = int(y_pad / 2)\n",
    "        y_max_pad = y_pad - y_min_pad\n",
    "\n",
    "    if width % pad_size == 0:\n",
    "        x_min_pad = 0\n",
    "        x_max_pad = 0\n",
    "    else:\n",
    "        x_pad = pad_size - width % pad_size\n",
    "        x_min_pad = int(x_pad / 2)\n",
    "        x_max_pad = x_pad - x_min_pad\n",
    "\n",
    "    img = cv2.copyMakeBorder(img, y_min_pad, y_max_pad, x_min_pad, x_max_pad, cv2.BORDER_REFLECT_101)\n",
    "\n",
    "    return img, (x_min_pad, y_min_pad, x_max_pad, y_max_pad)\n",
    "\n",
    "\n",
    "\n",
    "def unpad(img, pads):\n",
    "    \"\"\"\n",
    "    img: numpy array of the shape (height, width)\n",
    "    pads: (x_min_pad, y_min_pad, x_max_pad, y_max_pad)\n",
    "    @return padded image\n",
    "    \"\"\"\n",
    "    (x_min_pad, y_min_pad, x_max_pad, y_max_pad) = pads\n",
    "    height, width = img.shape[:2]\n",
    "\n",
    "    return img[y_min_pad:height - y_max_pad, x_min_pad:width - x_max_pad]\n",
    "\n",
    "\n",
    "\n",
    "def read_nuclei(path):\n",
    "    \"read raw data\"\n",
    "\n",
    "    # Load 4-channel image\n",
    "    img = skimage.io.imread(path)\n",
    "    \n",
    "    # input image\n",
    "    if len(img.shape) > 2:\n",
    "        img = img[:,:,:3]\n",
    "    # mask\n",
    "    else:\n",
    "        # do nothing\n",
    "        pass\n",
    "        \n",
    "    return img\n",
    "\n",
    "\n",
    "\n",
    "def save_nuclei(path, img):\n",
    "    \"save image\"\n",
    "    skimage.io.imsave(path, img)\n",
    "\n",
    "    \n",
    "def sliding_window(image, step, window):\n",
    "    x_loc = []\n",
    "    y_loc = []\n",
    "    cells = []\n",
    "    \n",
    "    for y in range(0, image.shape[0], step):\n",
    "        for x in range(0, image.shape[1], step):\n",
    "            cells.append(image[y:y + window[1], x:x + window[0]])\n",
    "            x_loc.append(x)\n",
    "            y_loc.append(y)\n",
    "    return x_loc, y_loc, cells\n",
    "\n",
    "\n",
    "def extract_patches(image, step, patch_size):\n",
    "    \n",
    "    patches = []\n",
    "    \n",
    "    # Get locations\n",
    "    x_pos, y_pos, cells = sliding_window(image, step, (patch_size[0], patch_size[1]))\n",
    "\n",
    "    for (x, y, cell) in zip(x_pos, y_pos, cells):\n",
    "\n",
    "        # Get patch\n",
    "        patch = image[y:y + patch_size[0], x:x + patch_size[0]]\n",
    "\n",
    "        # Get size\n",
    "        raw_dim = (patch.shape[1], patch.shape[0]) # W, H\n",
    "        #print(raw_dim)\n",
    "        #print(patch.shape)\n",
    "\n",
    "\n",
    "        if raw_dim != (patch_size[0], patch_size[1]):\n",
    "\n",
    "            # Resize to 64x64\n",
    "            #patch = cv2.resize(patch, (64, 64), interpolation = cv2.INTER_AREA)\n",
    "            patch, pad_locs = pad(patch, pad_size=patch_size[0])\n",
    "            \n",
    "            \n",
    "            # Do stuffffff\n",
    "            patches.append(patch)\n",
    "        \n",
    "        else:\n",
    "\n",
    "            # Do stuffffff\n",
    "            patches.append(patch)\n",
    "    \n",
    "    patches = np.array(patches)\n",
    "    \n",
    "    return patches\n",
    "    \n",
    "# Compute Panoptic quality metric for each image\n",
    "def Panoptic_quality(ground_truth_image,predicted_image):\n",
    "    TP = 0\n",
    "    FP = 0\n",
    "    FN = 0\n",
    "    sum_IOU = 0\n",
    "    matched_instances = {}# Create a dictionary to save ground truth indices in keys and predicted matched instances as velues\n",
    "                        # It will also save IOU of the matched instance in [indx][1]\n",
    "\n",
    "    # Find matched instances and save it in a dictionary\n",
    "    for i in np.unique(ground_truth_image):\n",
    "        if i == 0:\n",
    "            pass\n",
    "        else:\n",
    "            temp_image = np.array(ground_truth_image)\n",
    "            temp_image = temp_image == i\n",
    "            matched_image = temp_image * predicted_image\n",
    "        \n",
    "            for j in np.unique(matched_image):\n",
    "                if j == 0:\n",
    "                    pass\n",
    "                else:\n",
    "                    pred_temp = predicted_image == j\n",
    "                    intersection = sum(sum(temp_image*pred_temp))\n",
    "                    union = sum(sum(temp_image + pred_temp))\n",
    "                    IOU = intersection/union\n",
    "                    if IOU> 0.5:\n",
    "                        matched_instances [i] = j, IOU \n",
    "                        \n",
    "    # Compute TP, FP, FN and sum of IOU of the matched instances to compute Panoptic Quality               \n",
    "                        \n",
    "    pred_indx_list = np.unique(predicted_image)\n",
    "    pred_indx_list = np.array(pred_indx_list[1:])\n",
    "\n",
    "    # Loop on ground truth instances\n",
    "    for indx in np.unique(ground_truth_image):\n",
    "        if indx == 0:\n",
    "            pass\n",
    "        else:\n",
    "            if indx in matched_instances.keys():\n",
    "                pred_indx_list = np.delete(pred_indx_list, np.argwhere(pred_indx_list == [indx][0]))\n",
    "                TP = TP+1\n",
    "                sum_IOU = sum_IOU+matched_instances[indx][1]\n",
    "            else:\n",
    "                FN = FN+1\n",
    "    FP = len(np.unique(pred_indx_list))\n",
    "    PQ = sum_IOU/(TP+0.5*FP+0.5*FN)\n",
    "    \n",
    "    return PQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load a random image from the images folder\n",
    "image_fns = sorted(next(os.walk(TRAIN_IMAGES))[2])\n",
    "gt_fns = sorted(next(os.walk(TRAIN_MASKS))[2])\n",
    "\n",
    "\n",
    "idx = random.randrange(len(image_fns)) # 94 \n",
    "print(\"Index: \", idx)\n",
    "\n",
    "\n",
    "image = skimage.io.imread(os.path.join(TRAIN_IMAGES, image_fns[idx]))\n",
    "gt = skimage.io.imread(os.path.join(TRAIN_MASKS, gt_fns[idx]))\n",
    "\n",
    "assert image.shape[:2] == gt.shape, \"Wrong image or ground truth!\"\n",
    "assert image.dtype == gt.dtype, \"Wrong data types!\"\n",
    "\n",
    "print(image.shape, gt.shape)\n",
    "\n",
    "val1 = gt.flatten()\n",
    "print(\"Ground truth classes: \", np.unique(val1))\n",
    "\n",
    "#label_map = {'Epithelial':1,\n",
    "#             'Lymphocyte':2,\n",
    "#             'Macrophage':4,\n",
    "#             'Neutrophil':3,\n",
    "#            }\n",
    "\n",
    "# Stolen from https://stackoverflow.com/questions/16834861/create-own-colormap-using-matplotlib-and-plot-color-scale \n",
    "norm=plt.Normalize(0,4) # 5 classes including BG\n",
    "map_name = matplotlib.colors.LinearSegmentedColormap.from_list(\"\", [\"black\", \"red\",\"yellow\",\"blue\", \"green\"])\n",
    "#map_name = 'magma'\n",
    "\n",
    "f, axarr = plt.subplots(1,2, figsize=(16,16))\n",
    "\n",
    "# idx = 154 all classes\n",
    "axarr[0].imshow(image, cmap=map_name, norm=norm)\n",
    "axarr[1].imshow(gt, cmap=map_name, norm=norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import efficientnet.tfkeras\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "log_path = os.path.join(base_path, \"logs\", experiment_name)\n",
    "\n",
    "model = None\n",
    "model = load_model('{}/{}.h5'.format(log_path, experiment_name), compile=False)\n",
    "#model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SAME CODE BLOCK AS IN 6_inference.ipynb\n",
    "\n",
    "import numpy as np\n",
    "from skimage.transform import resize\n",
    "\n",
    "# Helper function for data visualization\n",
    "def visualize(**images):\n",
    "    \"\"\"Plot images in one row.\"\"\"\n",
    "    \n",
    "    norm=plt.Normalize(0,4) # 5 classes including BG\n",
    "    map_name = matplotlib.colors.LinearSegmentedColormap.from_list(\"\", [\"black\", \"red\",\"yellow\",\"blue\", \"green\"])\n",
    "    \n",
    "    n = len(images)\n",
    "    plt.figure(figsize=(18, 16))\n",
    "    for i, (name, image) in enumerate(images.items()):\n",
    "        plt.subplot(1, n, i + 1)\n",
    "        plt.xticks([])\n",
    "        plt.yticks([])\n",
    "        plt.title(' '.join(name.split('_')).title())\n",
    "        plt.imshow(image, cmap=map_name, norm=norm)\n",
    "    \n",
    "    plt.show()\n",
    "    \n",
    "    \n",
    "    \n",
    "def visualize__(**images):\n",
    "    \"\"\"Plot images in one row.\"\"\"\n",
    "    \n",
    "    norm=plt.Normalize(0,4) # 5 classes including BG\n",
    "    map_name = matplotlib.colors.LinearSegmentedColormap.from_list(\"\", [\"black\", \"red\",\"yellow\",\"blue\", \"green\"])\n",
    "\n",
    "\n",
    "    import matplotlib.gridspec as gridspec\n",
    "    gs1 = gridspec.GridSpec(2, 2)\n",
    "    gs1.update(wspace=0.0015, hspace=0.0015) # set the spacing between axes. \n",
    "\n",
    "    \n",
    "    n = len(images)\n",
    "    plt.figure(figsize=(18, 16))\n",
    "    for i, (name, image) in enumerate(images.items()):\n",
    "        plt.subplot(1, n, i + 1)\n",
    "        plt.xticks([])\n",
    "        plt.yticks([])\n",
    "        #plt.title(' '.join(name.split('_')).title())\n",
    "        plt.imshow(image, cmap=map_name, norm=norm)\n",
    "        \n",
    "    #plt.subplots_adjust(wspace=0, hspace=0.1, left=0, right=1, bottom=0, top=1)\n",
    "    plt.show()\n",
    "    \n",
    "    \n",
    "    \n",
    "def prep(img):\n",
    "    img = img.astype('float32')\n",
    "    img = (img > 0.5).astype(np.uint8)  # threshold\n",
    "    img = resize(img, (image_cols, image_rows), preserve_range=True)\n",
    "    return img\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def visualize_results(image, mask):\n",
    "    \n",
    "    f, axarr = plt.subplots(1,2, figsize=(16, 16))\n",
    "    \n",
    "    norm=plt.Normalize(0,4) # 5 classes including BG\n",
    "    map_name = matplotlib.colors.LinearSegmentedColormap.from_list(\"\", [\"black\", \"red\",\"yellow\",\"blue\", \"green\"])\n",
    "\n",
    "    axarr[0].imshow(image)\n",
    "    axarr[1].imshow(mask, cmap=map_name, norm=norm)\n",
    "\n",
    "\n",
    "    \n",
    "def vis_gray(image, mask):\n",
    "    \n",
    "    f, axarr = plt.subplots(1,2, figsize=(16, 16))\n",
    "    \n",
    "    axarr[0].imshow(image)\n",
    "    axarr[1].imshow(mask, cmap='gray')\n",
    "\n",
    "\n",
    "\n",
    "def predict(im):\n",
    "    \"\"\"Predict on patch\"\"\"\n",
    "    \n",
    "    im = np.expand_dims(im, axis=0)\n",
    "    \n",
    "    im = model.predict(im)\n",
    "    im = np.argmax(im.squeeze(), axis=-1)\n",
    " \n",
    "    #assert im.shape == (96, 96), \"Wrong shape, {}!\".format(im.shape)\n",
    "    \n",
    "    return im\n",
    "\n",
    "\n",
    "def instance_seg(image, footprint=np.ones((3, 3))):\n",
    "    distance = ndi.distance_transform_edt(image)\n",
    "    local_maxi = peak_local_max(distance, indices=False, footprint=footprint, labels=image)\n",
    "    markers = ndi.label(local_maxi)[0]\n",
    "    labels = watershed(-distance, markers, mask=image)\n",
    "    return labels    \n",
    "\n",
    "\n",
    "def whole_slide_predict(whole_image):\n",
    "    \n",
    "    #import pdb; pdb.set_trace()\n",
    "    \n",
    "    # If input image less than patch, infer on whole image\n",
    "    if whole_image.shape[0] < 96 or whole_image.shape[1] < 96:\n",
    "        \n",
    "        # Get size\n",
    "        raw_dim = (whole_image.shape[1], whole_image.shape[0]) # W, H\n",
    "        \n",
    "        # Resize to 64x64 for prediction\n",
    "        #whole_image_rs = cv2.resize(whole_image, (64, 64), interpolation = cv2.INTER_AREA)\n",
    "        whole_image_rs, pad_locs = pad(whole_image, pad_size=96)\n",
    "        \n",
    "        \n",
    "        # Infer\n",
    "        pred = predict(whole_image_rs)\n",
    "        \n",
    "        \n",
    "        # Resize back to original shape\n",
    "        #pred = cv2.resize(pred, raw_dim, interpolation = cv2.INTER_AREA)\n",
    "        pred = unpad(pred, pad_locs)\n",
    "        \n",
    "        # Change dtype for resizing back to original shape\n",
    "        pred = pred.astype(np.uint8)\n",
    "        \n",
    "      \n",
    "    else:\n",
    "        \n",
    "        # Get patch locations\n",
    "        x_pos, y_pos, cells = sliding_window(whole_image, 96, (96, 96)) \n",
    "\n",
    "        # Array for storing predictions\n",
    "        pred = np.zeros((whole_image.shape[0], whole_image.shape[1])).astype(np.uint8)\n",
    "\n",
    "        # Slide over each patch\n",
    "        for (x, y, cell) in zip(x_pos, y_pos, cells):\n",
    "\n",
    "            # Get patch\n",
    "            patch = whole_image[y:y + 96, x:x + 96]\n",
    "\n",
    "            # Get size\n",
    "            raw_dim = (patch.shape[1], patch.shape[0]) # W, H\n",
    "\n",
    "            # If less than patch size, resize and then run prediction\n",
    "            if raw_dim != (96, 96):\n",
    "\n",
    "\n",
    "                # Resize to 64x64\n",
    "                #patch_rs = cv2.resize(patch, (64, 64), interpolation = cv2.INTER_AREA)\n",
    "                patch_rs, pad_locs = pad(patch, pad_size=96)\n",
    "                \n",
    "                #print(patch.dtype, processed.dtype)\n",
    "                \n",
    "                assert patch.dtype == patch_rs.dtype, \"Wrong data type after resizing!\"\n",
    "\n",
    "                \n",
    "                # Infer\n",
    "                processed = predict(patch_rs)\n",
    "                \n",
    "                # Resize back to original shape\n",
    "                #processed = cv2.resize(processed, raw_dim, interpolation = cv2.INTER_AREA)\n",
    "                processed = unpad(processed, pad_locs)\n",
    "                \n",
    "                # Change dtype \n",
    "                processed = processed.astype(np.uint8)\n",
    "                \n",
    "                assert patch.shape[:2] == processed.shape, \"Wrong shape!\"\n",
    "                assert patch.dtype == processed.dtype, \"Wrong data type in prediction!\"\n",
    "\n",
    "            else:\n",
    "\n",
    "                \n",
    "                # Infer\n",
    "                processed = predict(patch)\n",
    "                \n",
    "                # Change dtype\n",
    "                processed = processed.astype(np.uint8)\n",
    "\n",
    "                #print(patch.dtype, processed.dtype)\n",
    "                \n",
    "\n",
    "                assert patch.shape[:2] == processed.shape, \"Wrong shape!\"\n",
    "                assert patch.dtype == processed.dtype, \"Wrong data type in prediction!\"\n",
    "\n",
    "\n",
    "            # Add in image variable\n",
    "            pred[y:y + 96, x:x + 96] = processed \n",
    "            processed = None\n",
    "\n",
    "    return pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute on arbitary images\n",
    "# Load a random image from the images folder\n",
    "image_fns = sorted(next(os.walk(VALID_IMAGES))[2])\n",
    "gt_fns = sorted(next(os.walk(VALID_MASKS))[2])\n",
    "\n",
    "\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    idx = random.randrange(len(image_fns))\n",
    "    #print(\"Index: \", idx)\n",
    "    \n",
    "    # Read image and ground truth\n",
    "    image = skimage.io.imread(os.path.join(VALID_IMAGES, image_fns[idx]))\n",
    "    gt = skimage.io.imread(os.path.join(VALID_MASKS, gt_fns[idx]))\n",
    "    \n",
    "    #print(image.dtype, gt.dtype)\n",
    "    \n",
    "    # Predict input image\n",
    "    pred = whole_slide_predict(image)\n",
    "    \n",
    "    # Post processing to refine predictions\n",
    "    pred_filt = cv2.medianBlur(pred.astype(np.uint8), 5)\n",
    "    \n",
    "    #print(image.shape, gt.shape, pred.shape)\n",
    "    \n",
    "    # Display image label and prediction\n",
    "    pq_g = Panoptic_quality(gt, gt)\n",
    "    pq = Panoptic_quality(gt, pred)\n",
    "    pq_f = Panoptic_quality(gt, pred_filt)\n",
    "    \n",
    "    #print(\"PQ of gt, pred and pred with medfil: \", pq_g, pq, pq_f)\n",
    "    \n",
    "    visualize(\n",
    "            image=image,\n",
    "            GT_mask = gt,\n",
    "            Predicted_mask = pred,\n",
    "            #Filtered_mask = pred_filt\n",
    "        )\n",
    "    \n",
    "    assert image.shape[:2] == pred.shape, \"Image missmatch\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Infer on one image for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mask_overlay(image, mask, color=(0, 255, 0)):\n",
    "    \"\"\"\n",
    "    Helper function to visualize mask on the top of the image\n",
    "    \"\"\"\n",
    "    \n",
    "    #epi_mask = np.where(mask != 1, img, 1)\n",
    "    \n",
    "    \n",
    "    mask = np.dstack((mask, mask, mask)) * np.array(color, (255, 0 , 0))\n",
    "    mask = mask.astype(np.uint8)\n",
    "    weighted_sum = cv2.addWeighted(mask, 0.5, image, 0.5, 0.)\n",
    "    img = image.copy()\n",
    "    ind = mask[:, :, 1] > 0    \n",
    "    img[ind] = weighted_sum[ind]    \n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_fns = sorted(next(os.walk(TRAIN_IMAGES))[2])\n",
    "gt_fns = sorted(next(os.walk(TRAIN_MASKS))[2])\n",
    " \n",
    "idx = random.randrange(len(image_fns))\n",
    "\n",
    "# Read image and ground truth\n",
    "image = skimage.io.imread(os.path.join(TRAIN_IMAGES, image_fns[idx]))\n",
    "gt = skimage.io.imread(os.path.join(TRAIN_MASKS, gt_fns[idx]))\n",
    "print(image.shape, gt.shape)\n",
    "\n",
    "# Predict input image\n",
    "pred = whole_slide_predict(image)\n",
    "#pred = predict(image)\n",
    "\n",
    "# Post processing to refine predictions\n",
    "pred_filt = cv2.medianBlur(pred.astype(np.uint8), 5)\n",
    "\n",
    "print(image.shape, gt.shape, pred.shape)\n",
    "\n",
    "# Display image label and prediction\n",
    "pq_g = Panoptic_quality(gt, gt)\n",
    "pq = Panoptic_quality(gt, pred)\n",
    "pq_f = Panoptic_quality(gt, pred_filt)\n",
    "\n",
    "\n",
    "print(\"PQ of gt, pred and pred with medfil: \", pq_g, pq, pq_f)\n",
    "\n",
    "visualize(\n",
    "        image=image,\n",
    "        GT_mask = gt,\n",
    "        Predicted_mask = pred,\n",
    "        Filtered_mask = pred_filt\n",
    "    )\n",
    "\n",
    "assert image.shape[:2] == pred.shape, \"Image missmatch\"\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inst_mask = instance_seg(pred_filt, np.ones((3, 3)))\n",
    "im = Image.fromarray(inst_mask.astype(np.uint8), mode='P')\n",
    "im.putpalette(NUCLEI_PALETTE)\n",
    "#im\n",
    "\n",
    "print(image.shape, pred.shape)\n",
    "print(\"Uniques predicted\", np.unique(pred))\n",
    "\n",
    "# Dummy mask\n",
    "zero_mask = np.zeros((pred_filt.shape[0], pred_filt.shape[1])).astype(np.uint8)\n",
    "# Overlay target class\n",
    "epi_mask = np.where(pred_filt != 1, zero_mask, 1)\n",
    "lym_mask = np.where(pred_filt != 2, zero_mask, 2)\n",
    "neu_mask = np.where(pred_filt != 3, zero_mask, 3)\n",
    "macro_mask = np.where(pred_filt != 4, zero_mask, 4)\n",
    "\n",
    "# Get uniques for (debugging)\n",
    "print(epi_mask.shape, lym_mask.shape, neu_mask.shape, macro_mask.shape)\n",
    "\n",
    "\n",
    "# Get instances for each class using watershed\n",
    "epi_mask = instance_seg(epi_mask)\n",
    "lym_mask = instance_seg(lym_mask)\n",
    "neu_mask = instance_seg(neu_mask)\n",
    "macro_mask = instance_seg(macro_mask)\n",
    "print(epi_mask.shape, lym_mask.shape, neu_mask.shape, macro_mask.shape)\n",
    "\n",
    "\n",
    "\n",
    "# Add color to instances\n",
    "epi_mask = Image.fromarray(epi_mask.astype(np.uint8), mode='P')\n",
    "epi_mask.putpalette(NUCLEI_PALETTE)\n",
    "\n",
    "lym_mask = Image.fromarray(lym_mask.astype(np.uint8), mode='P')\n",
    "lym_mask.putpalette(NUCLEI_PALETTE)\n",
    "\n",
    "neu_mask = Image.fromarray(neu_mask.astype(np.uint8), mode='P')\n",
    "neu_mask.putpalette(NUCLEI_PALETTE)\n",
    "\n",
    "macro_mask = Image.fromarray(macro_mask.astype(np.uint8), mode='P')\n",
    "macro_mask.putpalette(NUCLEI_PALETTE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_inst(**images):\n",
    "    \"\"\"Plot images in one row.\"\"\"\n",
    "    \n",
    "    norm=plt.Normalize(0,4) # 5 classes including BG\n",
    "    map_name = matplotlib.colors.LinearSegmentedColormap.from_list(\"\", [\"black\", \"red\",\"yellow\",\"blue\", \"green\"])\n",
    "\n",
    "    \n",
    "    n = len(images)\n",
    "    plt.figure(figsize=(30, 24))\n",
    "    for i, (name, image) in enumerate(images.items()):\n",
    "        \n",
    "        if name == \"GT_mask\" or name == \"Predicted_mask\":\n",
    "            plt.subplot(1, n, i + 1)\n",
    "            plt.xticks([])\n",
    "            plt.yticks([])\n",
    "            plt.title(' '.join(name.split('_')).title())\n",
    "            plt.imshow(image, cmap=map_name, norm=norm)\n",
    "        else:\n",
    "            plt.subplot(1, n, i + 1)\n",
    "            plt.xticks([])\n",
    "            plt.yticks([])\n",
    "            plt.title(' '.join(name.split('_')).title())\n",
    "            plt.imshow(image)\n",
    "    \n",
    "    plt.savefig(\"others/result.png\", bbox_inches = 'tight', pad_inches = 0, dpi=300)\n",
    "    plt.show()\n",
    "    \n",
    "    \n",
    "visualize_inst(\n",
    "        image=image,\n",
    "        #GT_mask = gt,\n",
    "        Predicted_mask = pred_filt,\n",
    "        Epithelial_mask = epi_mask,\n",
    "        Lymphocyte_mask = lym_mask,\n",
    "        Macrophage_mask = neu_mask,\n",
    "        Neutrophil_mask = macro_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#norm=plt.Normalize(0,4) # 5 classes including BG\n",
    "#map_name = matplotlib.colors.LinearSegmentedColormap.from_list(\"\", [\"black\", \"red\",\"yellow\",\"blue\", \"green\"])\n",
    "#plt.figure(figsize=(18, 16))\n",
    "#plt.imshow(np.hstack([image, mask_overlay(image, gt)]))\n",
    "\n",
    "#plt.imshow(image)\n",
    "#x = plt.imshow(pred, cmap=map_name, norm=norm, alpha=0.5) # interpolation='none'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute PQ on WS validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_fns = sorted(next(os.walk(VALID_IMAGES))[2])\n",
    "gt_fns = sorted(next(os.walk(VALID_MASKS))[2])\n",
    "\n",
    "pqs = []\n",
    "pqs_f = []\n",
    "pqs_gold = []\n",
    "\n",
    "\n",
    "for i in range(len(image_fns)):\n",
    "    \n",
    "    # Read image and ground truth\n",
    "    image = skimage.io.imread(os.path.join(VALID_IMAGES, image_fns[i]))\n",
    "    gt = skimage.io.imread(os.path.join(VALID_MASKS, gt_fns[i]))\n",
    "    \n",
    "    # Predict input image\n",
    "    pred = whole_slide_predict(image)\n",
    "    \n",
    "    # Post processing to refine predictions\n",
    "    pred_filt = cv2.medianBlur(pred.astype(np.uint8), 5)\n",
    "    \n",
    "    print(image.shape, gt.shape, pred.shape)\n",
    "    \n",
    "    # Display image label and prediction\n",
    "    pq_g = Panoptic_quality(gt, gt)\n",
    "    pq = Panoptic_quality(gt, pred)\n",
    "    pq_f = Panoptic_quality(gt, pred_filt)\n",
    "    \n",
    "    \n",
    "    print(\"PQ of gt, pred and pred with medfil: \", pq_g, pq, pq_f)\n",
    "    \n",
    "    visualize(\n",
    "            image=image,\n",
    "            GT_mask = gt,\n",
    "            Predicted_mask = pred,\n",
    "            Filtered_mask = pred_filt\n",
    "        )\n",
    "    \n",
    "    \n",
    "    pqs_gold.append(pq_g)\n",
    "    pqs.append(pq)\n",
    "    pqs_f.append(pq_f)\n",
    "    \n",
    "    assert image.shape[:2] == pred.shape, \"Image missmatch\"\n",
    "    \n",
    "\n",
    "pqs = np.mean(np.array(pqs))\n",
    "pqs_f = np.mean(np.array(pqs_f))\n",
    "pqs_gold = np.mean(np.array(pqs_gold))\n",
    "\n",
    "print(\"Mean PQ gold standard: \", pqs_gold)\n",
    "print(\"Mean PQ without filer: \", pqs)\n",
    "print(\"Mean PQ without filer: \", pqs_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pqs = []\n",
    "#pqs_f = []\n",
    "#pqs_gold = []\n",
    "\n",
    "#for i in tqdm(range(len(image_fns))):\n",
    "    \n",
    "    # Read image and ground truth\n",
    "#    image = skimage.io.imread(os.path.join(IMAGES_DEST, image_fns[i]))\n",
    "#    gt = skimage.io.imread(os.path.join(MASKS_DEST, gt_fns[i]))\n",
    "    \n",
    "    # Predict input image\n",
    "#    pred = whole_slide_predict(image)\n",
    "    \n",
    "    # Post processing to refine predictions\n",
    "#    pred_filt = cv2.medianBlur(pred.astype(np.uint8), 5)\n",
    "#    pred_filt = pred_filt.astype(np.int64)\n",
    "    \n",
    "    #print(image.shape, gt.shape, pred.shape)\n",
    "    \n",
    "    # Display image label and prediction\n",
    "    #visualize(\n",
    "    #        image=image,\n",
    "    #        GT_mask = gt,\n",
    "    #        Predicted_mask = pred\n",
    "    #    )\n",
    "    \n",
    "#    pq_gold = Panoptic_quality(gt, gt)\n",
    "#    pq = Panoptic_quality(gt, pred)\n",
    "#    pqf = Panoptic_quality(gt, pred_filt)\n",
    "    \n",
    "#    pqs_gold.append(pq_gold)\n",
    "#    pqs.append(pq)\n",
    "#    pqs_f.append(pqf)\n",
    "    \n",
    "\n",
    "#pqs = np.mean(np.array(pqs))\n",
    "#pqs_f = np.mean(np.array(pqs_f))\n",
    "#pqs_gold = np.mean(np.array(pqs_gold))\n",
    "\n",
    "#print(\"Mean PQ gold standard: \", pqs_gold)\n",
    "#print(\"Mean PQ without filer: \", pqs)\n",
    "#print(\"Mean PQ without filer: \", pqs_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "multiseg_camvid.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}