proper fast rcnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "**Feature Extraction**\n",
    "\n",
    "We begin with an image and a set of bounding boxes along with its label as defined below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "image = torch.zeros((1, 3, 800, 800)).float()\n",
    "\n",
    "bbox = torch.FloatTensor([[20, 30, 400, 500], [300, 400, 500, 600]]) # [y1, x1, y2, x2] format\n",
    "labels = torch.LongTensor([6, 8]) # 0 represents background\n",
    "sub_sample = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Generate anchor boxes**\n",
    "\n",
    "Create anchor boxes at all the locations in the feature map."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "ratios = [0.5, 1, 2]\n",
    "anchor_scales = [8, 16, 32]\n",
    "\n",
    "anchor_base = np.zeros((len(ratios) * len(anchor_scales), 4), dtype=np.float32)\n",
    "\n",
    "print(anchor_base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 4)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anchor_base.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.0 8.0\n"
     ]
    }
   ],
   "source": [
    "ctr_y = sub_sample /2.\n",
    "ctr_x = sub_sample /2.\n",
    "\n",
    "print(ctr_y, ctr_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(ratios)):\n",
    "    for j in range(len(anchor_scales)):\n",
    "        h = sub_sample * anchor_scales[j] * np.sqrt(ratios[i])\n",
    "        w = sub_sample * anchor_scales[j] * np.sqrt(1./ ratios[i])\n",
    "\n",
    "        index = i * len(anchor_scales) + j\n",
    "\n",
    "        anchor_base[index, 0] = ctr_y - h / 2.\n",
    "        anchor_base[index, 1] = ctr_x - w / 2.\n",
    "        anchor_base[index, 2] = ctr_y + h / 2.\n",
    "        anchor_base[index, 3] = ctr_x + w / 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -37.254833,  -82.50967 ,   53.254833,   98.50967 ],\n",
       "       [ -82.50967 , -173.01933 ,   98.50967 ,  189.01933 ],\n",
       "       [-173.01933 , -354.03867 ,  189.01933 ,  370.03867 ],\n",
       "       [ -56.      ,  -56.      ,   72.      ,   72.      ],\n",
       "       [-120.      , -120.      ,  136.      ,  136.      ],\n",
       "       [-248.      , -248.      ,  264.      ,  264.      ],\n",
       "       [ -82.50967 ,  -37.254833,   98.50967 ,   53.254833],\n",
       "       [-173.01933 ,  -82.50967 ,  189.01933 ,   98.50967 ],\n",
       "       [-354.03867 , -173.01933 ,  370.03867 ,  189.01933 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anchor_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_size = (800//16)\n",
    "ctr_x = np.arange(16, (fe_size+1) * 16, 16)\n",
    "ctr_y = np.arange(16, (fe_size+1) * 16, 16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr = np.empty((len(ctr_x) * len(ctr_y), 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "for x in range(len(ctr_x)):\n",
    "    for y in range(len(ctr_y)):\n",
    "        ctr[index, 1] = ctr_x[x] - 8\n",
    "        ctr[index, 0] = ctr_y[y] - 8\n",
    "        index +=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500, 4)\n"
     ]
    }
   ],
   "source": [
    "anchors = np.zeros(((fe_size * fe_size * 9), 4))\n",
    "index = 0\n",
    "for c in ctr:\n",
    "    ctr_y, ctr_x = c\n",
    "    for i in range(len(ratios)):\n",
    "        for j in range(len(anchor_scales)):\n",
    "            h = sub_sample * anchor_scales[j] * np.sqrt(ratios[i])\n",
    "            w = sub_sample * anchor_scales[j] * np.sqrt(1./ ratios[i])\n",
    "            anchors[index, 0] = ctr_y - h / 2.\n",
    "            anchors[index, 1] = ctr_x - w / 2.\n",
    "            anchors[index, 2] = ctr_y + h / 2.\n",
    "            anchors[index, 3] = ctr_x + w / 2.\n",
    "            index += 1\n",
    "print(anchors.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Assign the labels and the location of the object with respect to the anchor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbox = np.asarray([[20, 30, 400, 500], [300, 400, 500, 600]], dtype=np.float32) # [y1, x1, y2, x2] format\n",
    "labels = np.asarray([6, 8], dtype=np.int8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Indices of the valid anchor boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940,)\n"
     ]
    }
   ],
   "source": [
    "index_inside = np.where(\n",
    "        (anchors[:, 0] >= 0) &\n",
    "        (anchors[:, 1] >= 0) &\n",
    "        (anchors[:, 2] <= 800) &\n",
    "        (anchors[:, 3] <= 800)\n",
    "    )[0]\n",
    "print(index_inside.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Empty label with all fill -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940,)\n"
     ]
    }
   ],
   "source": [
    "label = np.empty((len(index_inside), ), dtype=np.int32)\n",
    "label.fill(-1)\n",
    "print(label.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Array of valid anchor boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 4)\n"
     ]
    }
   ],
   "source": [
    "valid_anchor_boxes = anchors[index_inside]\n",
    "print(valid_anchor_boxes.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For every box calculate the iou with the every ground truth object. \n",
    "\n",
    "- Find the max of x1 and y1 in both the boxes (xn1, yn1)\n",
    "- Find the min of x2 and y2 in both the boxes (xn2, yn2)\n",
    "- Now both the boxes are intersecting only\n",
    " if (xn1 < xn2) and (yn2 < yn1)\n",
    "      - iou_area will be (xn2 - xn1) * (yn2 - yn1)\n",
    " else\n",
    "      - iuo_area will be 0\n",
    "- similarly calculate area for anchor box and ground truth object\n",
    "- iou = iou_area/(anchor_box_area + ground_truth_area - iou_area)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 20.  30. 400. 500.]\n",
      " [300. 400. 500. 600.]]\n",
      "(8940, 2)\n"
     ]
    }
   ],
   "source": [
    "ious = np.empty((len(valid_anchor_boxes), 2), dtype=np.float32)\n",
    "ious.fill(0)\n",
    "print(bbox)\n",
    "for num1, i in enumerate(valid_anchor_boxes):\n",
    "    ya1, xa1, ya2, xa2 = i  \n",
    "    anchor_area = (ya2 - ya1) * (xa2 - xa1)\n",
    "    for num2, j in enumerate(bbox):\n",
    "        yb1, xb1, yb2, xb2 = j\n",
    "        box_area = (yb2- yb1) * (xb2 - xb1)\n",
    "        inter_x1 = max([xb1, xa1])\n",
    "        inter_y1 = max([yb1, ya1])\n",
    "        inter_x2 = min([xb2, xa2])\n",
    "        inter_y2 = min([yb2, ya2])\n",
    "        if (inter_x1 < inter_x2) and (inter_y1 < inter_y2):\n",
    "            iter_area = (inter_y2 - inter_y1) * \\\n",
    "(inter_x2 - inter_x1)\n",
    "            iou = iter_area / \\\n",
    "(anchor_area+ box_area - iter_area)            \n",
    "        else:\n",
    "            iou = 0.\n",
    "        ious[num1, num2] = iou\n",
    "print(ious.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Considering the scenarios of a and b, we need to find two things here\n",
    "- the highest iou for each gt_box and its corresponding anchor box\n",
    "- the highest iou for each anchor box and its corresponding ground truth box"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**case 1:** the highest iou for every ground truth box and the corresponding anchor box."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2262 5620]\n",
      "[0.68130493 0.61035156]\n"
     ]
    }
   ],
   "source": [
    "gt_argmax_ious = ious.argmax(axis=0)\n",
    "print(gt_argmax_ious)\n",
    "gt_max_ious = ious[gt_argmax_ious, np.arange(ious.shape[1])]\n",
    "print(gt_max_ious)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**case 2:** the highest iou for every anchor box and the corresponding ground truth box."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940,)\n",
      "[0 0 0 ... 0 0 0]\n",
      "[0.06811669 0.07083762 0.07083762 ... 0.         0.         0.        ]\n"
     ]
    }
   ],
   "source": [
    "argmax_ious = ious.argmax(axis=1)\n",
    "print(argmax_ious.shape)\n",
    "print(argmax_ious)\n",
    "max_ious = ious[np.arange(len(index_inside)), argmax_ious]\n",
    "print(max_ious)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have three arrays\n",
    "- argmax_ious — Tells which ground truth object has max iou with each anchor.\n",
    "- max_ious — Tells the max_iou with ground truth object with each anchor.\n",
    "- gt_argmax_ious — Tells the anchors with the highest Intersection-over-Union (IoU) overlap with a ground-truth box."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets put thresholds to some variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_iou_threshold  = 0.7\n",
    "neg_iou_threshold = 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "case 1: Assign a negative class to the anchor box which has a max iou as less than 0.3 for that class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "label[max_ious < neg_iou_threshold] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "case 2: Assign a positive class to the anchor boxes that has the highest ious for the ground truth objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "label[gt_argmax_ious] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "case 3: Assign positive class to the anchor boxes that have iou values as greater than positive threshold as 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "label[max_ious > pos_iou_threshold] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Training RPN** \n",
    "\n",
    "The Faster_R-CNN paper phrases as follows \n",
    "\n",
    "*Each mini-batch arises from a single image that contains many positive and negitive example anchors, but this will bias towards negitive samples as they are dominate. Instead, we randomly sample 256 anchors in an image to compute the loss function of a mini-batch, where the sampled positive and negative anchors have a ratio of up to 1:1. If there are fewer than 128 positive samples in an image, we pad the mini-batch with negitive ones..*\n",
    "\n",
    "From this we can derive two variable as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_ratio = 0.5\n",
    "n_sample = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_pos = pos_ratio * n_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- postive boxes sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_index = np.where(label == 1)[0]\n",
    "if len(pos_index) > n_pos:\n",
    "    disable_index = np.random.choice(pos_index, size=(len(pos_index) - n_pos), replace=False)\n",
    "    label[disable_index] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- negative sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_neg = n_sample * np.sum(label == 1)\n",
    "neg_index = np.where(label == 0)[0]\n",
    "if len(neg_index) > n_neg:\n",
    "    disable_index = np.random.choice(neg_index, size=(len(neg_index) - n_neg), replace = False)\n",
    "    label[disable_index] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Assigning locations to anchor boxes**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now lets assign the locations to each anchor box with the ground truth object which has maximum iou. Note, we will assign anchor locs to all the valid anchor boxes irrespective of its label, later when we are calculating the losses, we can remove them with simple filters.\n",
    "\n",
    "We already know which ground truth object has high iou with each anchor box, Now we need to find the locations of ground truth with respect to the anchor box location. Faster_R-CNN uses the following parametrizion for this."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$t_{x} = (x - x_{a})/w_{a}$$\n",
    "$$t_{y} = (y - y_{a})/h_{a}$$\n",
    "$$t_{w} = log(w/ w_a)$$\n",
    "$$t_{h} = log(h/ h_a)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each anchor box, find the groundtruth object which has the maximum iou."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 20.  30. 400. 500.]\n",
      " [ 20.  30. 400. 500.]\n",
      " [ 20.  30. 400. 500.]\n",
      " ...\n",
      " [ 20.  30. 400. 500.]\n",
      " [ 20.  30. 400. 500.]\n",
      " [ 20.  30. 400. 500.]]\n"
     ]
    }
   ],
   "source": [
    "max_iou_bbox = bbox[argmax_ious]\n",
    "print(max_iou_bbox)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Inorder to find t_{x}, t_{y}, t_{w}, t_{h}, we need to convert the y1, x1, y2, x2 format of valid anchor boxes and associated ground truth boxes with max iou to ctr_y, ctr_x , h, w format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "height = valid_anchor_boxes[:, 2] - valid_anchor_boxes[:, 0]\n",
    "width = valid_anchor_boxes[:, 3] - valid_anchor_boxes[:, 1]\n",
    "ctr_y = valid_anchor_boxes[:, 0] + 0.5 * height\n",
    "ctr_x = valid_anchor_boxes[:, 1] + 0.5 * width\n",
    "\n",
    "base_height = max_iou_bbox[:, 2] - max_iou_bbox[:, 0]\n",
    "base_width = max_iou_bbox[:, 3] - max_iou_bbox[:, 1]\n",
    "base_ctr_y = max_iou_bbox[:, 0] + 0.5 * base_height\n",
    "base_ctr_x = max_iou_bbox[:, 1] + 0.5 * base_width"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use the formulaes to find the locs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.5855728   2.30914558  0.7415674   1.64727602]\n",
      " [ 0.49718446  2.30914558  0.7415674   1.64727602]\n",
      " [ 0.40879611  2.30914558  0.7415674   1.64727602]\n",
      " ...\n",
      " [-2.50801936 -5.29225232  0.7415674   1.64727602]\n",
      " [-2.59640771 -5.29225232  0.7415674   1.64727602]\n",
      " [-2.68479606 -5.29225232  0.7415674   1.64727602]]\n"
     ]
    }
   ],
   "source": [
    "eps = np.finfo(height.dtype).eps\n",
    "height = np.maximum(height, eps)\n",
    "width = np.maximum(width, eps)\n",
    "dy = (base_ctr_y - ctr_y) / height\n",
    "dx = (base_ctr_x - ctr_x) / width\n",
    "dh = np.log(base_height / height)\n",
    "dw = np.log(base_width / width)\n",
    "anchor_locs = np.vstack((dy, dx, dh, dw)).transpose()\n",
    "print(anchor_locs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8940, 4)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anchor_locs.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- final labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "anchor_labels = np.empty((len(anchors),), dtype=label.dtype)\n",
    "anchor_labels.fill(-1)\n",
    "anchor_labels[index_inside] = label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- final locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "anchor_locations = np.empty((len(anchors),) + anchors.shape[1:], dtype=anchor_locs.dtype)\n",
    "anchor_locations.fill(0)\n",
    "anchor_locations[index_inside, :] = anchor_locs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The final two matrices are\n",
    "- anchor_locations [N, 4] — [22500, 4]\n",
    "- anchor_labels [N,] — [22500]\n",
    "\n",
    "These are used as targets to the RPN network. We will see how this RPN network is designed in the next section."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Region Proposal Network**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "mid_channels = 512\n",
    "in_channels = 512 # depends on the output feature map. in vgg 16 it is equal to 512\n",
    "n_anchor = 9 # Number of anchors at each location\n",
    "conv1 = nn.Conv2d(in_channels, mid_channels, 3, 1, 1)\n",
    "reg_layer = nn.Conv2d(mid_channels, n_anchor *4, 1, 1, 0)\n",
    "cls_layer = nn.Conv2d(mid_channels, n_anchor *2, 1, 1, 0) \n",
    "## I will be going to use softmax here. you can equally use sigmoid if u replace 2 with 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The paper tells that they initialized these layers with zero mean and 0.01 standard deviation for weights and zeros for base. Lets do that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# conv sliding layer\n",
    "conv1.weight.data.normal_(0, 0.01)\n",
    "conv1.bias.data.zero_()\n",
    "# Regression layer\n",
    "reg_layer.weight.data.normal_(0, 0.01)\n",
    "reg_layer.bias.data.zero_()\n",
    "# classification layer\n",
    "cls_layer.weight.data.normal_(0, 0.01)\n",
    "cls_layer.bias.data.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the outputs we got in the feature extraction state should be sent to this network to predict locations of objects with repect to the anchor and the objectness score assoiciated with it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_map = torch.rand(1, 512, 50, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 18, 50, 50]) torch.Size([1, 36, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "x = conv1(out_map) # out_map is obtained in section 1\n",
    "pred_anchor_locs = reg_layer(x)\n",
    "pred_cls_scores = cls_layer(x)\n",
    "print(pred_cls_scores.shape, pred_anchor_locs.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 22500, 4])\n",
      "tensor([[[[-9.5024e-02,  2.6759e-02, -3.9740e-02,  ...,  9.7388e-02,\n",
      "           -1.8628e-02,  1.6635e-02],\n",
      "          [-4.4591e-02,  1.9843e-02,  8.6184e-02,  ...,  4.7052e-02,\n",
      "            5.3647e-02, -6.3169e-02],\n",
      "          [-1.0754e-02,  6.6458e-02,  7.8701e-02,  ...,  5.9399e-02,\n",
      "           -6.6282e-03, -6.6840e-02],\n",
      "          ...,\n",
      "          [-1.7168e-02,  7.0088e-02,  9.8154e-02,  ...,  2.7229e-03,\n",
      "           -2.5998e-02, -2.3750e-02],\n",
      "          [ 1.3076e-02,  1.2077e-01,  4.2411e-02,  ..., -2.2580e-02,\n",
      "           -4.0996e-02, -5.0846e-02],\n",
      "          [-4.6171e-03, -6.5945e-02,  4.9031e-02,  ..., -3.8430e-02,\n",
      "           -1.6058e-02, -5.7551e-02]],\n",
      "\n",
      "         [[-5.4386e-02,  2.0342e-02,  3.5286e-02,  ...,  7.9171e-02,\n",
      "           -8.2211e-03, -6.5353e-02],\n",
      "          [ 1.2870e-01,  3.8972e-02, -5.9430e-02,  ...,  3.3565e-02,\n",
      "            3.0402e-02, -6.4990e-02],\n",
      "          [-3.9643e-02,  6.9098e-02,  1.3554e-03,  ..., -5.3457e-02,\n",
      "            7.3035e-03, -1.5339e-03],\n",
      "          ...,\n",
      "          [-5.3760e-02,  9.5125e-02,  5.5999e-02,  ..., -5.0412e-02,\n",
      "            5.1013e-03,  1.7992e-02],\n",
      "          [-4.7007e-02,  6.3877e-02,  1.6125e-02,  ..., -3.4017e-02,\n",
      "            4.5600e-02, -4.4867e-02],\n",
      "          [-4.3686e-02, -3.6433e-02,  5.1490e-02,  ..., -6.2649e-02,\n",
      "           -8.0817e-02,  8.2004e-03]],\n",
      "\n",
      "         [[ 2.3407e-03,  5.8183e-02, -9.8463e-02,  ...,  1.4873e-01,\n",
      "            2.5081e-02,  1.7409e-02],\n",
      "          [-3.2096e-02,  1.0811e-01,  2.9551e-02,  ..., -3.4148e-02,\n",
      "           -2.0430e-02, -1.6546e-02],\n",
      "          [ 3.5759e-04,  1.2383e-01, -3.5414e-03,  ..., -2.9842e-02,\n",
      "            3.1279e-02, -8.5570e-04],\n",
      "          ...,\n",
      "          [-3.6464e-02,  7.4557e-02, -2.2040e-02,  ...,  2.2805e-02,\n",
      "            1.9604e-02, -6.6639e-02],\n",
      "          [-2.4853e-02,  5.0360e-02,  4.9095e-02,  ...,  5.4442e-02,\n",
      "           -4.6431e-03,  2.5863e-02],\n",
      "          [-5.0896e-02,  2.1683e-02,  6.2947e-02,  ..., -3.9798e-02,\n",
      "           -1.0578e-01, -7.9377e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-7.8005e-02,  2.6927e-02, -4.2675e-02,  ...,  9.6271e-02,\n",
      "            1.7625e-02, -1.4268e-02],\n",
      "          [-4.3068e-05,  1.2338e-01,  2.9165e-02,  ..., -4.5399e-03,\n",
      "           -4.9577e-02, -4.6132e-02],\n",
      "          [ 7.6636e-03,  1.9689e-01, -2.4628e-02,  ...,  9.0567e-02,\n",
      "           -3.4973e-02, -6.0283e-02],\n",
      "          ...,\n",
      "          [ 3.2817e-02,  5.2584e-02, -5.8724e-02,  ..., -4.1832e-02,\n",
      "           -8.1515e-03, -1.3289e-02],\n",
      "          [-3.5560e-03,  3.9606e-02, -4.1017e-02,  ...,  1.3915e-02,\n",
      "           -3.6118e-02, -4.2673e-02],\n",
      "          [-8.3192e-02,  2.1792e-03,  2.3366e-02,  ..., -1.2487e-01,\n",
      "           -8.7513e-02, -3.8125e-02]],\n",
      "\n",
      "         [[-3.8368e-02,  3.4422e-02,  2.0273e-02,  ...,  2.7447e-02,\n",
      "            8.7294e-03,  1.0604e-02],\n",
      "          [-4.5843e-02,  3.0366e-02,  5.2734e-02,  ..., -2.4238e-02,\n",
      "            3.0832e-02, -1.5016e-02],\n",
      "          [ 3.8437e-02,  8.7803e-02,  1.0495e-02,  ..., -8.5241e-02,\n",
      "           -2.7050e-02, -2.5959e-02],\n",
      "          ...,\n",
      "          [ 1.3504e-02,  3.9866e-02, -1.6693e-02,  ..., -8.1649e-02,\n",
      "           -1.7333e-02, -1.7116e-01],\n",
      "          [ 4.1456e-02,  6.5401e-02, -9.5080e-02,  ...,  1.5960e-02,\n",
      "            4.5507e-02,  1.0813e-02],\n",
      "          [-2.8680e-02,  4.8069e-02, -2.5767e-02,  ..., -5.2559e-02,\n",
      "           -5.2322e-02, -1.3202e-02]],\n",
      "\n",
      "         [[ 2.8047e-02,  4.4260e-02, -3.0670e-03,  ...,  2.5614e-02,\n",
      "            1.3117e-02, -6.1150e-02],\n",
      "          [ 6.9166e-02,  9.0508e-02, -6.3651e-02,  ...,  2.3477e-02,\n",
      "           -1.7786e-02, -1.3745e-01],\n",
      "          [ 8.2188e-02,  6.9599e-02, -1.7511e-02,  ..., -3.4405e-02,\n",
      "           -1.1457e-01, -4.1198e-02],\n",
      "          ...,\n",
      "          [-6.5272e-03,  4.9363e-02, -3.2277e-02,  ..., -6.8543e-02,\n",
      "            1.0324e-02,  1.4711e-02],\n",
      "          [ 7.7350e-02,  6.7626e-02, -1.9132e-02,  ..., -7.2963e-02,\n",
      "           -1.4162e-01, -5.8240e-02],\n",
      "          [-3.5798e-02,  4.4784e-02, -3.4442e-02,  ..., -5.4457e-02,\n",
      "           -8.3550e-02, -3.1805e-02]]]], grad_fn=<CopyBackwards>)\n",
      "torch.Size([1, 22500])\n",
      "torch.Size([1, 22500, 2])\n"
     ]
    }
   ],
   "source": [
    "pred_anchor_locs = pred_anchor_locs.permute(0, 2, 3, 1).contiguous().view(1, -1, 4)\n",
    "print(pred_anchor_locs.shape)\n",
    "#Out: torch.Size([1, 22500, 4])\n",
    "pred_cls_scores = pred_cls_scores.permute(0, 2, 3, 1).contiguous()\n",
    "print(pred_cls_scores)\n",
    "#Out torch.Size([1, 50, 50, 18])\n",
    "objectness_score = pred_cls_scores.view(1, 50, 50, 9, 2)[:, :, :, :, 1].contiguous().view(1, -1)\n",
    "print(objectness_score.shape)\n",
    "#Out torch.Size([1, 22500])\n",
    "pred_cls_scores  = pred_cls_scores.view(1, -1, 2)\n",
    "print(pred_cls_scores.shape)\n",
    "# Out torch.size([1, 22500, 2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Generating region proposals from the RPN to feed to the Fast RCNN**\n",
    "\n",
    "This is the non-max suppression step, where we choose the best non-overlapping anchor boxes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "nms_thresh = 0.7\n",
    "n_train_pre_nms = 12000\n",
    "n_train_post_nms = 2000\n",
    "n_test_pre_nms = 6000\n",
    "n_test_post_nms = 300\n",
    "min_size = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. convert the loc predictions from the rpn network to bbox [y1, x1, y2, x2] format."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the reverse operations of what we have done while assigning ground truth to anchor boxes .This operation decodes predictions by un-parameterizing them and offseting to image. the formulas are as follows\n",
    "$$x = (w_{a} * ctrx_{p}) + ctrx_{a}$$\n",
    "$$y = (h_{a} * ctrx_{p}) + ctrx_{a}$$\n",
    "$$h = np.exp(h_{p}) * h_{a}$$\n",
    "$$w = np.exp(w_{p}) * w_{a}$$\n",
    "and later convert to y1, x1, y2, x2 format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "convert all the anchors possible from the feature map, from y1, x1, y2, x2 into ctr_x, ctr_y, h, w format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "anc_height = anchors[:, 2] - anchors[:, 0]\n",
    "anc_width = anchors[:, 3] - anchors[:, 1]\n",
    "anc_ctr_y = anchors[:, 0] + 0.5 * anc_height\n",
    "anc_ctr_x = anchors[:, 1] + 0.5 * anc_width"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Convert predictions locs using above formulas. before that convert the pred_anchor_locs and objectness_score to numpy array\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_anchor_locs_numpy = pred_anchor_locs[0].data.numpy()\n",
    "objectness_score_numpy = objectness_score[0].data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22500, 4)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_anchor_locs_numpy.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pred_anchor_locs has x, y, h, w normalized by anchor box. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "dy = pred_anchor_locs_numpy[:, 0::4]\n",
    "dx = pred_anchor_locs_numpy[:, 1::4]\n",
    "dh = pred_anchor_locs_numpy[:, 2::4]\n",
    "dw = pred_anchor_locs_numpy[:, 3::4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "convert this normalized pred_anchor_locs back into real coords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr_y = dy * anc_height[:, np.newaxis] + anc_ctr_y[:, np.newaxis]\n",
    "ctr_x = dx * anc_width[:, np.newaxis] + anc_ctr_x[:, np.newaxis]\n",
    "h = np.exp(dh) * anc_height[:, np.newaxis]\n",
    "w = np.exp(dw) * anc_width[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "convert [ctr_x, ctr_y, h, w] to [y1, x1, y2, x2] format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "roi = np.zeros(pred_anchor_locs_numpy.shape)\n",
    "roi[:, 0::4] = ctr_y - 0.5 * h\n",
    "roi[:, 1::4] = ctr_x - 0.5 * w\n",
    "roi[:, 2::4] = ctr_y + 0.5 * h\n",
    "roi[:, 3::4] = ctr_x + 0.5 * w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22500, 4)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -28.92410141,  -89.62584035,   51.93897971,  101.67660377],\n",
       "       [ -94.47754683, -177.5039333 ,   82.77957668,  165.72836979],\n",
       "       [-167.19772599, -437.67890807,  204.49474029,  381.83549529],\n",
       "       ...,\n",
       "       [ 709.55375141,  757.09631736,  879.15302588,  846.27818025],\n",
       "       [ 597.94354323,  700.97313758,  983.30521569,  869.78544069],\n",
       "       [ 417.03526588,  625.57564561, 1103.40453522,  979.89805367]])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. clip the predicted boxes to the image,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.           0.          51.93897971 101.67660377]\n",
      " [  0.           0.          82.77957668 165.72836979]\n",
      " [  0.           0.         204.49474029 381.83549529]\n",
      " ...\n",
      " [709.55375141 757.09631736 800.         800.        ]\n",
      " [597.94354323 700.97313758 800.         800.        ]\n",
      " [417.03526588 625.57564561 800.         800.        ]]\n"
     ]
    }
   ],
   "source": [
    "img_size = (800, 800) #Image size\n",
    "roi[:, slice(0, 4, 2)] = np.clip(\n",
    "            roi[:, slice(0, 4, 2)], 0, img_size[0])\n",
    "roi[:, slice(1, 4, 2)] = np.clip(\n",
    "    roi[:, slice(1, 4, 2)], 0, img_size[1])\n",
    "print(roi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove predicted boxes with either height or width < threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500,)\n"
     ]
    }
   ],
   "source": [
    "hs = roi[:, 2] - roi[:, 0]\n",
    "ws = roi[:, 3] - roi[:, 1]\n",
    "keep = np.where((hs >= min_size) & (ws >= min_size))[0]\n",
    "roi = roi[keep, :]\n",
    "score = objectness_score_numpy[keep]\n",
    "print(score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "now all the chosen boxes have the min size of 16."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[14079  3342 12474 ... 10441 21412 13168]\n"
     ]
    }
   ],
   "source": [
    "order = score.ravel().argsort()[::-1]\n",
    "print(order)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take top pre_nms_topN (e.g. 12000 while training and 300 while testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500, 4)\n",
      "[[  0.           0.          51.93897971 101.67660377]\n",
      " [  0.           0.          82.77957668 165.72836979]\n",
      " [  0.           0.         204.49474029 381.83549529]\n",
      " ...\n",
      " [709.55375141 757.09631736 800.         800.        ]\n",
      " [597.94354323 700.97313758 800.         800.        ]\n",
      " [417.03526588 625.57564561 800.         800.        ]]\n"
     ]
    }
   ],
   "source": [
    "order = order[:n_train_pre_nms]\n",
    "#roi = roi[order, :]\n",
    "print(roi.shape)\n",
    "print(roi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Non-max suppression**\n",
    "\n",
    "Pseudo Code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Take all the roi boxes [roi_array]\n",
    "- Find the areas of all the boxes [roi_area]\n",
    "- Take the indexes of order the probability score in descending order [order_array]\n",
    "keep = []\n",
    "while order_array.size > 0:\n",
    "  - take the first element in order_array and append that to keep  \n",
    "  - Find the area with all other boxes\n",
    "  - Find the index of all the boxes which have high overlap with this box\n",
    "  - Remove them from order array\n",
    "  - Iterate this till we get the order_size to zero (while loop)\n",
    "- Ouput the keep variable which tells what indexes to consider."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = roi[:, 0]\n",
    "x1 = roi[:, 1]\n",
    "y2 = roi[:, 2]\n",
    "x2 = roi[:, 3]\n",
    "area = (x2 - x1 + 1) * (y2 - y1 + 1)\n",
    "keep = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22500,)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "order.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([14079,  3342, 12474, ..., 10441, 21412, 13168])"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'thresh' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-125-a8ab645c88e7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0minter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mw\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0movr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minter\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0marea\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0marea\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0minter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0minds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0movr\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mthresh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0morder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minds\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0mkeep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkeep\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn_train_post_nms\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# while training/testing , use accordingly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'thresh' is not defined"
     ]
    }
   ],
   "source": [
    "while order.size > 0:\n",
    "    i = order[0]\n",
    "    xx1 = np.maximum(x1[i], x1[order[1:]])\n",
    "    yy1 = np.maximum(y1[i], y1[order[1:]])\n",
    "    xx2 = np.minimum(x2[i], x2[order[1:]])\n",
    "    yy2 = np.minimum(y2[i], y2[order[1:]])\n",
    "    w = np.maximum(0.0, xx2 - xx1 + 1)\n",
    "    h = np.maximum(0.0, yy2 - yy1 + 1)\n",
    "    inter = w * h\n",
    "    ovr = inter / (area[i] + area[order[1:]] - inter)\n",
    "    inds = np.where(ovr <= thresh)[0]\n",
    "    order = order[inds + 1]\n",
    "keep = keep[:n_train_post_nms] # while training/testing , use accordingly\n",
    "roi = roi[keep] # the final region proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}