inspect_tree_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mask R-CNN - Inspect Tree Trained Model\n",
    "\n",
    "Code and visualizations to test, debug, and evaluate the Mask R-CNN model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import random\n",
    "import math\n",
    "import re\n",
    "import time\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "# Root directory of the project\n",
    "MRCNN_DIR = os.path.abspath(\"../Mask_RCNN/\")\n",
    "ROOT_DIR = os.path.abspath(\"./\")\n",
    "\n",
    "# Import Mask RCNN\n",
    "sys.path.append(MRCNN_DIR)  # To find local version of the library\n",
    "from mrcnn import utils\n",
    "from mrcnn import visualize\n",
    "from mrcnn.visualize import display_images\n",
    "import mrcnn.model as modellib\n",
    "from mrcnn.model import log\n",
    "\n",
    "import tree\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Directory to save logs and trained model\n",
    "LOGS_DIR = os.path.join(ROOT_DIR, \"logs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset directory\n",
    "DATASET_DIR = os.path.join(ROOT_DIR, \"data/mrcnn\")\n",
    "\n",
    "# Inference Configuration\n",
    "config = tree.TreeInferenceConfig()\n",
    "config.display()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook Preferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Device to load the neural network on.\n",
    "# Useful if you're training a model on the same \n",
    "# machine, in which case use CPU and leave the\n",
    "# GPU for training.\n",
    "DEVICE = \"/cpu:0\"  # /cpu:0 or /gpu:0\n",
    "\n",
    "# Inspect the model in training or inference modes\n",
    "# values: 'inference' or 'training'\n",
    "# Only inference mode is supported right now\n",
    "TEST_MODE = \"inference\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ax(rows=1, cols=1, size=16):\n",
    "    \"\"\"Return a Matplotlib Axes array to be used in\n",
    "    all visualizations in the notebook. Provide a\n",
    "    central point to control graph sizes.\n",
    "    \n",
    "    Adjust the size attribute to control how big to render images\n",
    "    \"\"\"\n",
    "    fig, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n",
    "    fig.tight_layout()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Validation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load validation dataset\n",
    "dataset = nucleus.NucleusDataset()\n",
    "dataset.load_nucleus(DATASET_DIR, \"val\")\n",
    "dataset.prepare()\n",
    "\n",
    "print(\"Images: {}\\nClasses: {}\".format(len(dataset.image_ids), dataset.class_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Create model in inference mode\n",
    "with tf.device(DEVICE):\n",
    "    model = modellib.MaskRCNN(mode=\"inference\",\n",
    "                              model_dir=LOGS_DIR,\n",
    "                              config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Path to a specific weights file\n",
    "# weights_path = \"/path/to/mask_rcnn_nucleus.h5\"\n",
    "\n",
    "# Or, load the last model you trained\n",
    "weights_path = model.find_last()\n",
    "\n",
    "# Load weights\n",
    "print(\"Loading weights \", weights_path)\n",
    "model.load_weights(weights_path, by_name=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "image_id = random.choice(dataset.image_ids)\n",
    "image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n",
    "    modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)\n",
    "info = dataset.image_info[image_id]\n",
    "print(\"image ID: {}.{} ({}) {}\".format(info[\"source\"], info[\"id\"], image_id, \n",
    "                                       dataset.image_reference(image_id)))\n",
    "print(\"Original image shape: \", modellib.parse_image_meta(image_meta[np.newaxis,...])[\"original_image_shape\"][0])\n",
    "\n",
    "# Run object detection\n",
    "results = model.detect_molded(np.expand_dims(image, 0), np.expand_dims(image_meta, 0), verbose=1)\n",
    "\n",
    "# Display results\n",
    "r = results[0]\n",
    "log(\"gt_class_id\", gt_class_id)\n",
    "log(\"gt_bbox\", gt_bbox)\n",
    "log(\"gt_mask\", gt_mask)\n",
    "\n",
    "# Compute AP over range 0.5 to 0.95 and print it\n",
    "utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask,\n",
    "                       r['rois'], r['class_ids'], r['scores'], r['masks'],\n",
    "                       verbose=1)\n",
    "\n",
    "visualize.display_differences(\n",
    "    image,\n",
    "    gt_bbox, gt_class_id, gt_mask,\n",
    "    r['rois'], r['class_ids'], r['scores'], r['masks'],\n",
    "    dataset.class_names, ax=get_ax(),\n",
    "    show_box=False, show_mask=False,\n",
    "    iou_threshold=0.5, score_threshold=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Display predictions only\n",
    "# visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], \n",
    "#                             dataset.class_names, r['scores'], ax=get_ax(1),\n",
    "#                             show_bbox=False, show_mask=False,\n",
    "#                             title=\"Predictions\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display Ground Truth only\n",
    "# visualize.display_instances(image, gt_bbox, gt_mask, gt_class_id, \n",
    "#                             dataset.class_names, ax=get_ax(1),\n",
    "#                             show_bbox=False, show_mask=False,\n",
    "#                             title=\"Ground Truth\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute AP on Batch of Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def compute_batch_ap(dataset, image_ids, verbose=1):\n",
    "    APs = []\n",
    "    for image_id in image_ids:\n",
    "        # Load image\n",
    "        image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n",
    "            modellib.load_image_gt(dataset, config,\n",
    "                                   image_id, use_mini_mask=False)\n",
    "        # Run object detection\n",
    "        results = model.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0)\n",
    "        # Compute AP over range 0.5 to 0.95\n",
    "        r = results[0]\n",
    "        ap = utils.compute_ap_range(\n",
    "            gt_bbox, gt_class_id, gt_mask,\n",
    "            r['rois'], r['class_ids'], r['scores'], r['masks'],\n",
    "            verbose=0)\n",
    "        APs.append(ap)\n",
    "        if verbose:\n",
    "            info = dataset.image_info[image_id]\n",
    "            meta = modellib.parse_image_meta(image_meta[np.newaxis,...])\n",
    "            print(\"{:3} {}   AP: {:.2f}\".format(\n",
    "                meta[\"image_id\"][0], meta[\"original_image_shape\"][0], ap))\n",
    "    return APs\n",
    "\n",
    "# Run on validation set\n",
    "limit = 5\n",
    "APs = compute_batch_ap(dataset, dataset.image_ids[:limit])\n",
    "print(\"Mean AP overa {} images: {:.4f}\".format(len(APs), np.mean(APs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step by Step Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 1: Region Proposal Network\n",
    "\n",
    "The Region Proposal Network (RPN) runs a lightweight binary classifier on a lot of boxes (anchors) over the image and returns object/no-object scores. Anchors with high *objectness* score (positive anchors) are passed to the stage two to be classified.\n",
    "\n",
    "Often, even positive anchors don't cover objects fully. So the RPN also regresses a refinement (a delta in location and size) to be applied to the anchors to shift it and resize it a bit to the correct boundaries of the object."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.a RPN Targets\n",
    "\n",
    "The RPN targets are the training values for the RPN. To generate the targets, we start with a grid of anchors that cover the full image at different scales, and then we compute the IoU of the anchors with ground truth object. Positive anchors are those that have an IoU >= 0.7 with any ground truth object, and negative anchors are those that don't cover any object by more than 0.3 IoU. Anchors in between (i.e. cover an object by IoU >= 0.3 but < 0.7) are considered neutral and excluded from training.\n",
    "\n",
    "To train the RPN regressor, we also compute the shift and resizing needed to make the anchor cover the ground truth object completely."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get anchors and convert to pixel coordinates\n",
    "anchors = model.get_anchors(image.shape)\n",
    "anchors = utils.denorm_boxes(anchors, image.shape[:2])\n",
    "log(\"anchors\", anchors)\n",
    "\n",
    "# Generate RPN trainig targets\n",
    "# target_rpn_match is 1 for positive anchors, -1 for negative anchors\n",
    "# and 0 for neutral anchors.\n",
    "target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(\n",
    "    image.shape, anchors, gt_class_id, gt_bbox, model.config)\n",
    "log(\"target_rpn_match\", target_rpn_match)\n",
    "log(\"target_rpn_bbox\", target_rpn_bbox)\n",
    "\n",
    "positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]\n",
    "negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]\n",
    "neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]\n",
    "positive_anchors = anchors[positive_anchor_ix]\n",
    "negative_anchors = anchors[negative_anchor_ix]\n",
    "neutral_anchors = anchors[neutral_anchor_ix]\n",
    "log(\"positive_anchors\", positive_anchors)\n",
    "log(\"negative_anchors\", negative_anchors)\n",
    "log(\"neutral anchors\", neutral_anchors)\n",
    "\n",
    "# Apply refinement deltas to positive anchors\n",
    "refined_anchors = utils.apply_box_deltas(\n",
    "    positive_anchors,\n",
    "    target_rpn_bbox[:positive_anchors.shape[0]] * model.config.RPN_BBOX_STD_DEV)\n",
    "log(\"refined_anchors\", refined_anchors, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display positive anchors before refinement (dotted) and\n",
    "# after refinement (solid).\n",
    "visualize.draw_boxes(\n",
    "    image, ax=get_ax(),\n",
    "    boxes=positive_anchors,\n",
    "    refined_boxes=refined_anchors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.b RPN Predictions\n",
    "\n",
    "Here we run the RPN graph and display its predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run RPN sub-graph\n",
    "pillar = model.keras_model.get_layer(\"ROI\").output  # node to start searching from\n",
    "\n",
    "# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10\n",
    "nms_node = model.ancestor(pillar, \"ROI/rpn_non_max_suppression:0\")\n",
    "if nms_node is None:\n",
    "    nms_node = model.ancestor(pillar, \"ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0\")\n",
    "if nms_node is None: #TF 1.9-1.10\n",
    "    nms_node = model.ancestor(pillar, \"ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0\")\n",
    "\n",
    "rpn = model.run_graph(image[np.newaxis], [\n",
    "    (\"rpn_class\", model.keras_model.get_layer(\"rpn_class\").output),\n",
    "    (\"pre_nms_anchors\", model.ancestor(pillar, \"ROI/pre_nms_anchors:0\")),\n",
    "    (\"refined_anchors\", model.ancestor(pillar, \"ROI/refined_anchors:0\")),\n",
    "    (\"refined_anchors_clipped\", model.ancestor(pillar, \"ROI/refined_anchors_clipped:0\")),\n",
    "    (\"post_nms_anchor_ix\", nms_node),\n",
    "    (\"proposals\", model.keras_model.get_layer(\"ROI\").output),\n",
    "], image_metas=image_meta[np.newaxis])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show top anchors by score (before refinement)\n",
    "limit = 100\n",
    "sorted_anchor_ids = np.argsort(rpn['rpn_class'][:,:,1].flatten())[::-1]\n",
    "visualize.draw_boxes(image, boxes=anchors[sorted_anchor_ids[:limit]], ax=get_ax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Show top anchors with refinement. Then with clipping to image boundaries\n",
    "limit = 50\n",
    "ax = get_ax(1, 2)\n",
    "pre_nms_anchors = utils.denorm_boxes(rpn[\"pre_nms_anchors\"][0], image.shape[:2])\n",
    "refined_anchors = utils.denorm_boxes(rpn[\"refined_anchors\"][0], image.shape[:2])\n",
    "refined_anchors_clipped = utils.denorm_boxes(rpn[\"refined_anchors_clipped\"][0], image.shape[:2])\n",
    "visualize.draw_boxes(image, boxes=pre_nms_anchors[:limit],\n",
    "                     refined_boxes=refined_anchors[:limit], ax=ax[0])\n",
    "visualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[:limit], ax=ax[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Show refined anchors after non-max suppression\n",
    "limit = 50\n",
    "ixs = rpn[\"post_nms_anchor_ix\"][:limit]\n",
    "visualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[ixs], ax=get_ax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show final proposals\n",
    "# These are the same as the previous step (refined anchors \n",
    "# after NMS) but with coordinates normalized to [0, 1] range.\n",
    "limit = 50\n",
    "# Convert back to image coordinates for display\n",
    "# h, w = config.IMAGE_SHAPE[:2]\n",
    "# proposals = rpn['proposals'][0, :limit] * np.array([h, w, h, w])\n",
    "visualize.draw_boxes(\n",
    "    image, ax=get_ax(),\n",
    "    refined_boxes=utils.denorm_boxes(rpn['proposals'][0, :limit], image.shape[:2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 2: Proposal Classification\n",
    "\n",
    "This stage takes the region proposals from the RPN and classifies them."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.a Proposal Classification\n",
    "\n",
    "Run the classifier heads on proposals to generate class propbabilities and bounding box regressions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get input and output to classifier and mask heads.\n",
    "mrcnn = model.run_graph([image], [\n",
    "    (\"proposals\", model.keras_model.get_layer(\"ROI\").output),\n",
    "    (\"probs\", model.keras_model.get_layer(\"mrcnn_class\").output),\n",
    "    (\"deltas\", model.keras_model.get_layer(\"mrcnn_bbox\").output),\n",
    "    (\"masks\", model.keras_model.get_layer(\"mrcnn_mask\").output),\n",
    "    (\"detections\", model.keras_model.get_layer(\"mrcnn_detection\").output),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get detection class IDs. Trim zero padding.\n",
    "det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)\n",
    "det_count = np.where(det_class_ids == 0)[0][0]\n",
    "det_class_ids = det_class_ids[:det_count]\n",
    "detections = mrcnn['detections'][0, :det_count]\n",
    "\n",
    "print(\"{} detections: {}\".format(\n",
    "    det_count, np.array(dataset.class_names)[det_class_ids]))\n",
    "\n",
    "captions = [\"{} {:.3f}\".format(dataset.class_names[int(c)], s) if c > 0 else \"\"\n",
    "            for c, s in zip(detections[:, 4], detections[:, 5])]\n",
    "visualize.draw_boxes(\n",
    "    image, \n",
    "    refined_boxes=utils.denorm_boxes(detections[:, :4], image.shape[:2]),\n",
    "    visibilities=[2] * len(detections),\n",
    "    captions=captions, title=\"Detections\",\n",
    "    ax=get_ax())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.c Step by Step Detection\n",
    "\n",
    "Here we dive deeper into the process of processing the detections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Proposals are in normalized coordinates\n",
    "proposals = mrcnn[\"proposals\"][0]\n",
    "\n",
    "# Class ID, score, and mask per proposal\n",
    "roi_class_ids = np.argmax(mrcnn[\"probs\"][0], axis=1)\n",
    "roi_scores = mrcnn[\"probs\"][0, np.arange(roi_class_ids.shape[0]), roi_class_ids]\n",
    "roi_class_names = np.array(dataset.class_names)[roi_class_ids]\n",
    "roi_positive_ixs = np.where(roi_class_ids > 0)[0]\n",
    "\n",
    "# How many ROIs vs empty rows?\n",
    "print(\"{} Valid proposals out of {}\".format(np.sum(np.any(proposals, axis=1)), proposals.shape[0]))\n",
    "print(\"{} Positive ROIs\".format(len(roi_positive_ixs)))\n",
    "\n",
    "# Class counts\n",
    "print(list(zip(*np.unique(roi_class_names, return_counts=True))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display a random sample of proposals.\n",
    "# Proposals classified as background are dotted, and\n",
    "# the rest show their class and confidence score.\n",
    "limit = 200\n",
    "ixs = np.random.randint(0, proposals.shape[0], limit)\n",
    "captions = [\"{} {:.3f}\".format(dataset.class_names[c], s) if c > 0 else \"\"\n",
    "            for c, s in zip(roi_class_ids[ixs], roi_scores[ixs])]\n",
    "visualize.draw_boxes(\n",
    "    image,\n",
    "    boxes=utils.denorm_boxes(proposals[ixs], image.shape[:2]),\n",
    "    visibilities=np.where(roi_class_ids[ixs] > 0, 2, 1),\n",
    "    captions=captions, title=\"ROIs Before Refinement\",\n",
    "    ax=get_ax())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply Bounding Box Refinement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Class-specific bounding box shifts.\n",
    "roi_bbox_specific = mrcnn[\"deltas\"][0, np.arange(proposals.shape[0]), roi_class_ids]\n",
    "log(\"roi_bbox_specific\", roi_bbox_specific)\n",
    "\n",
    "# Apply bounding box transformations\n",
    "# Shape: [N, (y1, x1, y2, x2)]\n",
    "refined_proposals = utils.apply_box_deltas(\n",
    "    proposals, roi_bbox_specific * config.BBOX_STD_DEV)\n",
    "log(\"refined_proposals\", refined_proposals)\n",
    "\n",
    "# Show positive proposals\n",
    "# ids = np.arange(roi_boxes.shape[0])  # Display all\n",
    "limit = 5\n",
    "ids = np.random.randint(0, len(roi_positive_ixs), limit)  # Display random sample\n",
    "captions = [\"{} {:.3f}\".format(dataset.class_names[c], s) if c > 0 else \"\"\n",
    "            for c, s in zip(roi_class_ids[roi_positive_ixs][ids], roi_scores[roi_positive_ixs][ids])]\n",
    "visualize.draw_boxes(\n",
    "    image, ax=get_ax(),\n",
    "    boxes=utils.denorm_boxes(proposals[roi_positive_ixs][ids], image.shape[:2]),\n",
    "    refined_boxes=utils.denorm_boxes(refined_proposals[roi_positive_ixs][ids], image.shape[:2]),\n",
    "    visibilities=np.where(roi_class_ids[roi_positive_ixs][ids] > 0, 1, 0),\n",
    "    captions=captions, title=\"ROIs After Refinement\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filter Low Confidence Detections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove boxes classified as background\n",
    "keep = np.where(roi_class_ids > 0)[0]\n",
    "print(\"Keep {} detections:\\n{}\".format(keep.shape[0], keep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove low confidence detections\n",
    "keep = np.intersect1d(keep, np.where(roi_scores >= config.DETECTION_MIN_CONFIDENCE)[0])\n",
    "print(\"Remove boxes below {} confidence. Keep {}:\\n{}\".format(\n",
    "    config.DETECTION_MIN_CONFIDENCE, keep.shape[0], keep))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Per-Class Non-Max Suppression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply per-class non-max suppression\n",
    "pre_nms_boxes = refined_proposals[keep]\n",
    "pre_nms_scores = roi_scores[keep]\n",
    "pre_nms_class_ids = roi_class_ids[keep]\n",
    "\n",
    "nms_keep = []\n",
    "for class_id in np.unique(pre_nms_class_ids):\n",
    "    # Pick detections of this class\n",
    "    ixs = np.where(pre_nms_class_ids == class_id)[0]\n",
    "    # Apply NMS\n",
    "    class_keep = utils.non_max_suppression(pre_nms_boxes[ixs], \n",
    "                                            pre_nms_scores[ixs],\n",
    "                                            config.DETECTION_NMS_THRESHOLD)\n",
    "    # Map indicies\n",
    "    class_keep = keep[ixs[class_keep]]\n",
    "    nms_keep = np.union1d(nms_keep, class_keep)\n",
    "    print(\"{:22}: {} -> {}\".format(dataset.class_names[class_id][:20], \n",
    "                                   keep[ixs], class_keep))\n",
    "\n",
    "keep = np.intersect1d(keep, nms_keep).astype(np.int32)\n",
    "print(\"\\nKept after per-class NMS: {}\\n{}\".format(keep.shape[0], keep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show final detections\n",
    "ixs = np.arange(len(keep))  # Display all\n",
    "# ixs = np.random.randint(0, len(keep), 10)  # Display random sample\n",
    "captions = [\"{} {:.3f}\".format(dataset.class_names[c], s) if c > 0 else \"\"\n",
    "            for c, s in zip(roi_class_ids[keep][ixs], roi_scores[keep][ixs])]\n",
    "visualize.draw_boxes(\n",
    "    image,\n",
    "    boxes=utils.denorm_boxes(proposals[keep][ixs], image.shape[:2]),\n",
    "    refined_boxes=utils.denorm_boxes(refined_proposals[keep][ixs], image.shape[:2]),\n",
    "    visibilities=np.where(roi_class_ids[keep][ixs] > 0, 1, 0),\n",
    "    captions=captions, title=\"Detections after NMS\",\n",
    "    ax=get_ax())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 3: Generating Masks\n",
    "\n",
    "This stage takes the detections (refined bounding boxes and class IDs) from the previous layer and runs the mask head to generate segmentation masks for every instance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.a Mask Targets\n",
    "\n",
    "These are the training targets for the mask branch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "limit = 8\n",
    "display_images(np.transpose(gt_mask[..., :limit], [2, 0, 1]), cmap=\"Blues\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.b Predicted Masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get predictions of mask head\n",
    "mrcnn = model.run_graph([image], [\n",
    "    (\"detections\", model.keras_model.get_layer(\"mrcnn_detection\").output),\n",
    "    (\"masks\", model.keras_model.get_layer(\"mrcnn_mask\").output),\n",
    "])\n",
    "\n",
    "# Get detection class IDs. Trim zero padding.\n",
    "det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)\n",
    "det_count = np.where(det_class_ids == 0)[0][0]\n",
    "det_class_ids = det_class_ids[:det_count]\n",
    "\n",
    "print(\"{} detections: {}\".format(\n",
    "    det_count, np.array(dataset.class_names)[det_class_ids]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Masks\n",
    "det_boxes = utils.denorm_boxes(mrcnn[\"detections\"][0, :, :4], image.shape[:2])\n",
    "det_mask_specific = np.array([mrcnn[\"masks\"][0, i, :, :, c] \n",
    "                              for i, c in enumerate(det_class_ids)])\n",
    "det_masks = np.array([utils.unmold_mask(m, det_boxes[i], image.shape)\n",
    "                      for i, m in enumerate(det_mask_specific)])\n",
    "log(\"det_mask_specific\", det_mask_specific)\n",
    "log(\"det_masks\", det_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_images(det_mask_specific[:4] * 255, cmap=\"Blues\", interpolation=\"none\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_images(det_masks[:4] * 255, cmap=\"Blues\", interpolation=\"none\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize Activations\n",
    "\n",
    "In some cases it helps to look at the output from different layers and visualize them to catch issues and odd patterns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Get activations of a few sample layers\n",
    "activations = model.run_graph([image], [\n",
    "    (\"input_image\",        tf.identity(model.keras_model.get_layer(\"input_image\").output)),\n",
    "    (\"res2c_out\",          model.keras_model.get_layer(\"res2c_out\").output),\n",
    "    (\"res3c_out\",          model.keras_model.get_layer(\"res3c_out\").output),\n",
    "    (\"rpn_bbox\",           model.keras_model.get_layer(\"rpn_bbox\").output),\n",
    "    (\"roi\",                model.keras_model.get_layer(\"ROI\").output),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backbone feature map\n",
    "display_images(np.transpose(activations[\"res2c_out\"][0,:,:,:4], [2, 0, 1]), cols=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('dap05')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "bf136276df7b73f579d00ffc03546cf27d68eba258d4f2ea564dd7fb6a9dd1fb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}