experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import skimage\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from torchvision.models._utils import IntermediateLayerGetter\n",
    "\n",
    "from models.modeling.deeplab import *\n",
    "from dataloader.talk2car import *\n",
    "\n",
    "from PIL import Image\n",
    "from skimage.transform import resize\n",
    "\n",
    "from models.model import JointModel\n",
    "\n",
    "from utils.im_processing import *\n",
    "from utils.metrics import *\n",
    "\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Args:\n",
    "    lr = 3e-4\n",
    "    batch_size = 64\n",
    "    num_workers = 4\n",
    "    image_encoder = \"deeplabv3_plus\"\n",
    "    num_layers = 1\n",
    "    num_encoder_layers = 1\n",
    "    dropout = 0.25\n",
    "    skip_conn = False\n",
    "    model_path = \"./saved_model/talk2car/baseline_drop_0.25_bs_64_el_1_sl_40_bce_0.49785.pth\"\n",
    "    loss = \"bce\"\n",
    "    dataroot = \"/ssd_scratch/cvit/kanishk/\"\n",
    "    glove_path = \"/ssd_scratch/cvit/kanishk/glove/\"\n",
    "    dataset = \"talk2car\"\n",
    "    task = \"talk2car\"\n",
    "    split = \"val\"\n",
    "    seq_len = 40\n",
    "    image_dim = 448\n",
    "    mask_dim = 448\n",
    "    mask_thresh = 0.3\n",
    "    area_thresh = 0.4\n",
    "    topk = 10\n",
    "    metric = \"pointing_game\"\n",
    "\n",
    "args = Args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda being used with 2 GPUs!!\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "n_gpu = torch.cuda.device_count()\n",
    "print(f'{device} being used with {n_gpu} GPUs!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing dataset\n",
      "Length of dataset: 1163\n"
     ]
    }
   ],
   "source": [
    "print(\"Initializing dataset\")\n",
    "\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "to_tensor = transforms.ToTensor()\n",
    "resize = transforms.Resize((args.image_dim, args.image_dim))\n",
    "\n",
    "\n",
    "val_dataset = Talk2Car(\n",
    "    root=args.dataroot,\n",
    "    split=args.split,\n",
    "    transform=transforms.Compose([resize, to_tensor, normalize]),\n",
    "    mask_transform=transforms.Compose([ResizeAnnotation(args.mask_dim)]),\n",
    "    glove_path=args.glove_path\n",
    ")\n",
    "\n",
    "val_loader = DataLoader(\n",
    "    val_dataset, shuffle=True, batch_size=1, num_workers=0, pin_memory=True\n",
    ")\n",
    "\n",
    "data_len = val_dataset.__len__()\n",
    "print(f'Length of dataset: {data_len}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "return_layers = {\"layer2\": \"layer2\", \"layer3\": \"layer3\", \"layer4\": \"layer4\"}\n",
    "\n",
    "model = DeepLab(num_classes=21, backbone=\"resnet\", output_stride=16)\n",
    "model.load_state_dict(torch.load(\"./models/deeplab-resnet.pth.tar\")[\"state_dict\"])\n",
    "\n",
    "image_encoder = IntermediateLayerGetter(model.backbone, return_layers)\n",
    "\n",
    "for param in image_encoder.parameters():\n",
    "    param.requires_grad_(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_channels = 2048\n",
    "out_channels = 512\n",
    "stride = 2\n",
    "\n",
    "joint_model = JointModel(\n",
    "    in_channels=in_channels,\n",
    "    out_channels=out_channels,\n",
    "    stride=stride,\n",
    "    num_layers=args.num_layers,\n",
    "    num_encoder_layers=args.num_encoder_layers,\n",
    "    dropout=args.dropout,\n",
    "    skip_conn=args.skip_conn,\n",
    "    mask_dim=args.mask_dim,\n",
    ")\n",
    "\n",
    "if n_gpu > 1:\n",
    "    image_encoder = nn.DataParallel(image_encoder)\n",
    "    joint_model = nn.DataParallel(joint_model)\n",
    "\n",
    "state_dict = torch.load(args.model_path)\n",
    "if \"state_dict\" in state_dict:\n",
    "    state_dict = state_dict[\"state_dict\"]\n",
    "joint_model.load_state_dict(state_dict) \n",
    "\n",
    "joint_model.to(device)\n",
    "image_encoder.to(device)\n",
    "\n",
    "image_encoder.eval();\n",
    "joint_model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_mask_IOU(masks, target, thresh=0.3):\n",
    "    assert(target.shape[-2:] == masks.shape[-2:])\n",
    "    temp = ((masks>thresh) * target)\n",
    "    intersection = temp.sum()\n",
    "    union = (((masks>thresh) + target) - temp).sum()\n",
    "    return intersection, union\n",
    "\n",
    "def meanIOU(m, gt, t):\n",
    "    temp = ((m > t)*gt)\n",
    "    inter = temp.sum()\n",
    "    union = ((m > t) + gt - temp).sum()\n",
    "    return inter/union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection(command, key_words):\n",
    "    count = 0\n",
    "    words = []\n",
    "    for key in key_words:\n",
    "        if key in command:\n",
    "            words.append(key)\n",
    "            count += 1\n",
    "    return count, words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "recall_at_k for val split\n",
      "K=1: Accuracy:0.49785038693035255, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878786\n",
      "K=5: Accuracy:0.5184866723989682, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878806\n",
      "K=10: Accuracy:0.5270851246775581, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878817\n",
      "K=50: Accuracy:0.5528804815133276, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878806\n",
      "K=100: Accuracy:0.5692175408426483, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878795\n",
      "K=500: Accuracy:0.6448839208942391, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878797\n",
      "K=1000: Accuracy:0.6964746345657782, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878803\n",
      "K=5000: Accuracy:0.8632846087704213, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.1702530485187879\n"
     ]
    }
   ],
   "source": [
    "args.metric = \"recall_at_k\"\n",
    "args.topk = 5000\n",
    "\n",
    "args.mask_thresh = 0.3\n",
    "\n",
    "image_encoder.eval()\n",
    "joint_model.eval()\n",
    "\n",
    "print(f'{args.metric} for {args.split} split')\n",
    "\n",
    "exp_params = []\n",
    "\n",
    "if args.metric == \"intersection_at_t\":\n",
    "    exp_params = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]\n",
    "elif args.metric == \"recall_at_k\":\n",
    "    exp_params = [1, 5, 10, 50, 100, 500, 1000, 5000]\n",
    "else:\n",
    "    exp_params = [0]\n",
    "\n",
    "for param in exp_params:\n",
    "    total_inter = 0\n",
    "    total_union = 0\n",
    "\n",
    "    total_accuracy = 0\n",
    "\n",
    "    mean_IOU = 0\n",
    "    total_accuracy = 0\n",
    "\n",
    "    data_len = len(val_loader)\n",
    "\n",
    "    n_iter = 0\n",
    "\n",
    "    base_correct = 0\n",
    "\n",
    "    for step, batch in enumerate(val_loader):\n",
    "\n",
    "        img = batch[\"image\"].cuda(non_blocking=True)\n",
    "\n",
    "        phrase = batch[\"phrase\"].cuda(non_blocking=True)\n",
    "        phrase_mask = batch[\"phrase_mask\"].cuda(non_blocking=True)\n",
    "        index = batch[\"index\"]\n",
    "\n",
    "        gt_mask = batch[\"seg_mask\"]\n",
    "        gt_mask = gt_mask.squeeze(dim=1)\n",
    "\n",
    "        _, h, w = gt_mask.shape\n",
    "\n",
    "        if gt_mask[0, w//2, h//2] > 0:\n",
    "            base_correct += 1\n",
    "\n",
    "        batch_size = img.shape[0]\n",
    "        img_mask = torch.ones(batch_size, 14 * 14, dtype=torch.int64).cuda(\n",
    "            non_blocking=True\n",
    "        )\n",
    "\n",
    "        orig_phrase = batch[\"orig_phrase\"][0]\n",
    "        phrase_len = len(orig_phrase.split())\n",
    "\n",
    "        with torch.no_grad():\n",
    "            img = image_encoder(img)\n",
    "\n",
    "        output_mask = joint_model(img, phrase, img_mask, phrase_mask)\n",
    "        output_mask = output_mask.detach().cpu()\n",
    "\n",
    "        inter, union = compute_batch_IOU(output_mask, gt_mask, args.mask_thresh)\n",
    "\n",
    "        total_inter += inter.sum().item()\n",
    "        total_union += union.sum().item()\n",
    "\n",
    "        accuracy = 0\n",
    "        if args.metric == \"pointing_game\":\n",
    "            accuracy += pointing_game(output_mask, gt_mask)\n",
    "        elif args.metric == \"intersection_at_t\":\n",
    "            accuracy += intersection_at_t(output_mask, gt_mask, args.mask_thresh, param)\n",
    "        elif args.metric == \"recall_at_k\":\n",
    "            accuracy += recall_at_k(output_mask, gt_mask, param)\n",
    "        elif args.metric == \"dice_score\":\n",
    "            accuracy += dice_score(output_mask, gt_mask, args.mask_thresh)\n",
    "        total_accuracy += accuracy\n",
    "\n",
    "        score = 0 if union.item() == 0 else inter.item() / union.item()\n",
    "\n",
    "        mean_IOU += score\n",
    "\n",
    "        total_score = total_inter / total_union\n",
    "\n",
    "    overall_IOU = total_inter / total_union\n",
    "    mean_IOU = mean_IOU / data_len\n",
    "    final_accuracy = total_accuracy / data_len\n",
    "\n",
    "    if args.metric == \"pointing_game\":\n",
    "        center_accuracy = base_correct / data_len\n",
    "        print(f'Center Accuracy: {center_accuracy}')\n",
    "\n",
    "    if args.metric == \"intersection_at_t\":\n",
    "        print(f\"Area_Thresh={param}: Accuracy:{final_accuracy}, Overall_IOU: {overall_IOU}, Mean_IOU: {mean_IOU}\")\n",
    "    elif args.metric ==\"recall_at_k\":\n",
    "        print(f\"K={param}: Accuracy:{final_accuracy}, Overall_IOU: {overall_IOU}, Mean_IOU: {mean_IOU}\")\n",
    "    else:\n",
    "        print(f\"Pointing Game Accuracy:{final_accuracy}, Overall_IOU: {overall_IOU}, Mean_IOU: {mean_IOU}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Command Length Analysis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Center Accuracy: 0.0025795356835769563\n",
      "pointing_game for val split\n",
      "Accuracy:0.49785038693035255, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878797, Total: 0, \n"
     ]
    }
   ],
   "source": [
    "args.metric = \"pointing_game\"\n",
    "metric = args.metric\n",
    "\n",
    "args.topk = 200\n",
    "\n",
    "args.mask_thresh = 0.3\n",
    "\n",
    "image_encoder.eval()\n",
    "joint_model.eval()\n",
    "\n",
    "total_inter = 0\n",
    "total_union = 0\n",
    "\n",
    "total_accuracy = 0\n",
    "\n",
    "result_map = {\n",
    "    '0-10': {},\n",
    "    '10-20': {},\n",
    "    '20-': {},\n",
    "}\n",
    "\n",
    "area_thresh = args.metric\n",
    "\n",
    "mean_IOU = 0\n",
    "total_accuracy = 0\n",
    "\n",
    "data_len = len(val_loader)\n",
    "\n",
    "n_iter = 0\n",
    "\n",
    "base_correct = 0\n",
    "\n",
    "for step, batch in enumerate(val_loader):\n",
    "\n",
    "    img = batch[\"image\"].cuda(non_blocking=True)\n",
    "\n",
    "    phrase = batch[\"phrase\"].cuda(non_blocking=True)\n",
    "    phrase_mask = batch[\"phrase_mask\"].cuda(non_blocking=True)\n",
    "    index = batch[\"index\"]\n",
    "\n",
    "    gt_mask = batch[\"seg_mask\"]\n",
    "    gt_mask = gt_mask.squeeze(dim=1)\n",
    "    \n",
    "    _, h, w = gt_mask.shape\n",
    "    \n",
    "    if gt_mask[0, w//2, h//2] > 0:\n",
    "        base_correct += 1\n",
    "\n",
    "    batch_size = img.shape[0]\n",
    "    img_mask = torch.ones(batch_size, 14 * 14, dtype=torch.int64).cuda(\n",
    "        non_blocking=True\n",
    "    )\n",
    "    \n",
    "    orig_phrase = batch[\"orig_phrase\"][0]\n",
    "    phrase_len = len(orig_phrase.split())\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        img = image_encoder(img)\n",
    "\n",
    "    output_mask = joint_model(img, phrase, img_mask, phrase_mask)\n",
    "    output_mask = output_mask.detach().cpu()\n",
    "    \n",
    "    # count, inter_words = intersection(orig_phrase)\n",
    "\n",
    "    inter, union = compute_batch_IOU(output_mask, gt_mask, args.mask_thresh)\n",
    "\n",
    "    total_inter += inter.sum().item()\n",
    "    total_union += union.sum().item()\n",
    "\n",
    "    accuracy = 0\n",
    "    if args.metric == \"pointing_game\":\n",
    "        accuracy += pointing_game(output_mask, gt_mask)\n",
    "    elif args.metric == \"recall_at_k\":\n",
    "        accuracy += recall_at_k(output_mask, gt_mask, args.topk)\n",
    "    elif args.metric == \"dice_score\":\n",
    "        accuracy += dice_score(output_mask, gt_mask, args.mask_thresh)\n",
    "    total_accuracy += accuracy\n",
    "\n",
    "    if phrase_len < 10:\n",
    "        if metric not in result_map['0-10']:\n",
    "            result_map['0-10'][metric] = []\n",
    "        result_map['0-10'][metric].append(accuracy)\n",
    "    elif phrase_len < 20:\n",
    "        if metric not in result_map['10-20']:\n",
    "            result_map['10-20'][metric] = []\n",
    "        result_map['10-20'][metric].append(accuracy)\n",
    "    else:\n",
    "        if metric not in result_map['20-']:\n",
    "            result_map['20-'][metric] = []\n",
    "        result_map['20-'][metric].append(accuracy)\n",
    "        \n",
    "    score = 0 if union.item() == 0 else inter.item() / union.item()\n",
    "\n",
    "    mean_IOU += score\n",
    "\n",
    "    total_score = total_inter / total_union\n",
    "\n",
    "overall_IOU = total_inter / total_union\n",
    "mean_IOU = mean_IOU / data_len\n",
    "final_accuracy = total_accuracy / data_len\n",
    "\n",
    "if args.metric == \"pointing_game\":\n",
    "    center_accuracy = base_correct / data_len\n",
    "    print(f'Center Accuracy: {center_accuracy}')\n",
    "\n",
    "print(f'{args.metric} for {args.split} split')\n",
    "print(f\"Accuracy:{final_accuracy}, Overall_IOU: {overall_IOU}, Mean_IOU: {mean_IOU}, Total: {n_iter}, \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results at mask_thresh: 0.3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pointing_game</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phrase_len</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0-10</th>\n",
       "      <td>0.520958</td>\n",
       "      <td>501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10-20</th>\n",
       "      <td>0.485520</td>\n",
       "      <td>587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20-</th>\n",
       "      <td>0.440000</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            pointing_game  count\n",
       "phrase_len                      \n",
       "0-10             0.520958    501\n",
       "10-20            0.485520    587\n",
       "20-              0.440000     75"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f'Results at mask_thresh: {args.mask_thresh}')\n",
    "\n",
    "refined_map = {}\n",
    "for key1 in result_map:\n",
    "    refined_map[key1] = {}\n",
    "    total_len = 0\n",
    "    for key2 in result_map[key1]:\n",
    "        total_len += len(result_map[key1][key2])\n",
    "        accuracy = torch.tensor(result_map[key1][key2])\n",
    "        refined_map[key1][key2] = accuracy.mean().item()\n",
    "    refined_map[key1]['count'] = total_len // len(result_map[key1])\n",
    "\n",
    "df_result = pd.DataFrame.from_dict(refined_map,orient='index')\n",
    "df_result.index.name = 'phrase_len'\n",
    "\n",
    "df_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Command Type Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_words = [\"stop\", \"slow\", \"wait\", \"park\", \"speed\", \"change\", \"turn\", \"follow\", \"u-turn\", \"straight\", \"pull\", \"switch\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pointing_game for val split\n",
      "Center Accuracy: 0.0025795356835769563\n",
      "Accuracy:0.49785038693035255, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.170253048518788, Total: 0, \n"
     ]
    }
   ],
   "source": [
    "args.metric = \"pointing_game\"\n",
    "metric = args.metric\n",
    "\n",
    "args.topk = 200\n",
    "\n",
    "args.mask_thresh = 0.3\n",
    "\n",
    "image_encoder.eval()\n",
    "joint_model.eval()\n",
    "\n",
    "total_inter = 0\n",
    "total_union = 0\n",
    "\n",
    "total_accuracy = 0\n",
    "\n",
    "result_map = {key: {} for key in action_words}\n",
    "\n",
    "mean_IOU = 0\n",
    "total_accuracy = 0\n",
    "\n",
    "data_len = len(val_loader)\n",
    "\n",
    "n_iter = 0\n",
    "\n",
    "base_correct = 0\n",
    "\n",
    "for step, batch in enumerate(val_loader):\n",
    "\n",
    "    img = batch[\"image\"].cuda(non_blocking=True)\n",
    "\n",
    "    phrase = batch[\"phrase\"].cuda(non_blocking=True)\n",
    "    phrase_mask = batch[\"phrase_mask\"].cuda(non_blocking=True)\n",
    "    index = batch[\"index\"]\n",
    "    \n",
    "    gt_mask = batch[\"seg_mask\"]\n",
    "    gt_mask = gt_mask.squeeze(dim=1)\n",
    "    \n",
    "    _, h, w = gt_mask.shape\n",
    "    \n",
    "    if gt_mask[0, w//2, h//2] > 0:\n",
    "        base_correct += 1\n",
    "\n",
    "    batch_size = img.shape[0]\n",
    "    img_mask = torch.ones(batch_size, 14 * 14, dtype=torch.int64).cuda(\n",
    "        non_blocking=True\n",
    "    )\n",
    "    \n",
    "    orig_phrase = batch[\"orig_phrase\"][0]\n",
    "    phrase_len = len(orig_phrase.split())\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        img = image_encoder(img)\n",
    "\n",
    "    output_mask = joint_model(img, phrase, img_mask, phrase_mask)\n",
    "    output_mask = output_mask.detach().cpu()\n",
    "    \n",
    "    count, inter_words = intersection(orig_phrase, action_words)\n",
    "\n",
    "    inter, union = compute_batch_IOU(output_mask, gt_mask, args.mask_thresh)\n",
    "\n",
    "    total_inter += inter.sum().item()\n",
    "    total_union += union.sum().item()\n",
    "\n",
    "    accuracy = 0\n",
    "    if args.metric == \"pointing_game\":\n",
    "        accuracy += pointing_game(output_mask, gt_mask)\n",
    "    elif args.metric == \"intersection_at_t\":\n",
    "        accuracy += intersection_at_t(output_mask, gt_mask, args.mask_thresh, args.area_thresh)\n",
    "    elif args.metric == \"recall_at_k\":\n",
    "        accuracy += recall_at_k(output_mask, gt_mask, args.topk)\n",
    "    elif args.metric == \"dice_score\":\n",
    "        accuracy += dice_score(output_mask, gt_mask, args.mask_thresh)\n",
    "    total_accuracy += accuracy\n",
    "\n",
    "    for word in action_words:\n",
    "        if word in inter_words:\n",
    "            if metric not in result_map[word]:\n",
    "                result_map[word][metric] = []\n",
    "            result_map[word][metric].append(accuracy)\n",
    "\n",
    "    score = 0 if union.item() == 0 else inter.item() / union.item()\n",
    "\n",
    "    mean_IOU += score\n",
    "\n",
    "    total_score = total_inter / total_union\n",
    "\n",
    "overall_IOU = total_inter / total_union\n",
    "mean_IOU = mean_IOU / data_len\n",
    "final_accuracy = total_accuracy / data_len\n",
    "\n",
    "print(f'{args.metric} for {args.split} split')\n",
    "\n",
    "if args.metric == \"pointing_game\":\n",
    "    center_accuracy = base_correct / data_len\n",
    "    print(f'Center Accuracy: {center_accuracy}')\n",
    "\n",
    "print(f\"Accuracy:{final_accuracy}, Overall_IOU: {overall_IOU}, Mean_IOU: {mean_IOU}, Total: {n_iter}, \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results at mask_thresh: 0.3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pointing_game</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>action</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>stop</th>\n",
       "      <td>0.510067</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slow</th>\n",
       "      <td>0.565789</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wait</th>\n",
       "      <td>0.477273</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>park</th>\n",
       "      <td>0.438127</td>\n",
       "      <td>299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>speed</th>\n",
       "      <td>0.538462</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>change</th>\n",
       "      <td>0.416667</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>turn</th>\n",
       "      <td>0.506073</td>\n",
       "      <td>247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>follow</th>\n",
       "      <td>0.488889</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>u-turn</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>straight</th>\n",
       "      <td>0.384615</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pull</th>\n",
       "      <td>0.507042</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>switch</th>\n",
       "      <td>0.562500</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          pointing_game  count\n",
       "action                        \n",
       "stop           0.510067    149\n",
       "slow           0.565789     76\n",
       "wait           0.477273     44\n",
       "park           0.438127    299\n",
       "speed          0.538462     13\n",
       "change         0.416667     24\n",
       "turn           0.506073    247\n",
       "follow         0.488889    135\n",
       "u-turn         0.333333     18\n",
       "straight       0.384615     39\n",
       "pull           0.507042    142\n",
       "switch         0.562500     16"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f'Results at mask_thresh: {args.mask_thresh}')\n",
    "\n",
    "refined_map = {}\n",
    "for key1 in result_map:\n",
    "    refined_map[key1] = {}\n",
    "    total_len = 0\n",
    "    for key2 in result_map[key1]:\n",
    "        total_len += len(result_map[key1][key2])\n",
    "        accuracy = torch.tensor(result_map[key1][key2])\n",
    "        refined_map[key1][key2] = accuracy.mean().item()\n",
    "    if len(result_map[key1]) > 0:\n",
    "        refined_map[key1]['count'] = total_len // len(result_map[key1])\n",
    "\n",
    "df_result = pd.DataFrame.from_dict(refined_map,orient='index')\n",
    "df_result.index.name = 'action'\n",
    "\n",
    "df_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Temporal Vs Non-Temporal Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "temporal_words = [\"once\", \"then\", \"when\", \"sometime\", \"minutes\", \"possible\", \"safe\", \"wait\", \"while\", \"check\", \"before\", \"until\", \"after\", \"soon\", \"but\", \"slow\", \"follow\", \"u-turn\", \"you turn\",]\n",
    "action_words = [\"stop\", \"slow\", \"wait\", \"park\", \"speed\", \"change\", \"turn\", \"follow\", \"u-turn\", \"straight\", \"pull\", \"switch\", \"pick\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pointing_game for val split\n",
      "Center Accuracy: 0.005159071367153913\n",
      "Accuracy:0.49785038693035255, Overall_IOU: 0.19882105059992924, Mean_IOU: 0.17025304851878814, Total: 0, \n"
     ]
    }
   ],
   "source": [
    "args.metric = \"pointing_game\"\n",
    "metric = args.metric\n",
    "\n",
    "args.topk = 200\n",
    "\n",
    "args.mask_thresh = 0.3\n",
    "args.area_thresh = 0.4\n",
    "\n",
    "image_encoder.eval()\n",
    "joint_model.eval()\n",
    "\n",
    "total_inter = 0\n",
    "total_union = 0\n",
    "\n",
    "total_accuracy = 0\n",
    "\n",
    "area_thresh = args.metric\n",
    "\n",
    "result_map = {\n",
    "    'non-temporal': {},\n",
    "    'temporal': {},\n",
    "}\n",
    "\n",
    "mean_IOU = 0\n",
    "total_accuracy = 0\n",
    "\n",
    "data_len = len(val_loader)\n",
    "\n",
    "n_iter = 0\n",
    "\n",
    "for step, batch in enumerate(val_loader):\n",
    "\n",
    "    img = batch[\"image\"].cuda(non_blocking=True)\n",
    "\n",
    "    phrase = batch[\"phrase\"].cuda(non_blocking=True)\n",
    "    phrase_mask = batch[\"phrase_mask\"].cuda(non_blocking=True)\n",
    "    index = batch[\"index\"]\n",
    "\n",
    "    gt_mask = batch[\"seg_mask\"]\n",
    "    gt_mask = gt_mask.squeeze(dim=1)\n",
    "    \n",
    "    _, h, w = gt_mask.shape\n",
    "    \n",
    "    if gt_mask[0, w//2, h//2] > 0:\n",
    "        base_correct += 1\n",
    "\n",
    "    batch_size = img.shape[0]\n",
    "    img_mask = torch.ones(batch_size, 14 * 14, dtype=torch.int64).cuda(\n",
    "        non_blocking=True\n",
    "    )\n",
    "    \n",
    "    orig_phrase = batch[\"orig_phrase\"][0]\n",
    "    phrase_len = len(orig_phrase.split())\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        img = image_encoder(img)\n",
    "\n",
    "    output_mask = joint_model(img, phrase, img_mask, phrase_mask)\n",
    "    output_mask = output_mask.detach().cpu()\n",
    "    \n",
    "    count_temp, inter_words = intersection(orig_phrase, temporal_words)\n",
    "    count_action, inter_words = intersection(orig_phrase, action_words)\n",
    "\n",
    "    inter, union = compute_batch_IOU(output_mask, gt_mask, args.mask_thresh)\n",
    "\n",
    "    total_inter += inter.sum().item()\n",
    "    total_union += union.sum().item()\n",
    "\n",
    "    accuracy = 0\n",
    "    if args.metric == \"pointing_game\":\n",
    "        accuracy += pointing_game(output_mask, gt_mask)\n",
    "    elif args.metric == \"intersection_at_t\":\n",
    "        accuracy += intersection_at_t(output_mask, gt_mask, args.mask_thresh, area_thresh)\n",
    "    elif args.metric == \"recall_at_k\":\n",
    "        accuracy += recall_at_k(output_mask, gt_mask, args.topk)\n",
    "    elif args.metric == \"dice_score\":\n",
    "        accuracy += dice_score(output_mask, gt_mask, args.mask_thresh)\n",
    "    total_accuracy += accuracy\n",
    "\n",
    "    if count_temp > 0 or count_action > 1:\n",
    "        if area_thresh not in result_map['temporal']:\n",
    "            result_map['temporal'][area_thresh] = []\n",
    "        result_map['temporal'][area_thresh].append(accuracy)\n",
    "    else:\n",
    "        if area_thresh not in result_map['non-temporal']:\n",
    "            result_map['non-temporal'][area_thresh] = []\n",
    "        result_map['non-temporal'][area_thresh].append(accuracy)\n",
    "\n",
    "    score = 0 if union.item() == 0 else inter.item() / union.item()\n",
    "\n",
    "    mean_IOU += score\n",
    "\n",
    "    total_score = total_inter / total_union\n",
    "\n",
    "overall_IOU = total_inter / total_union\n",
    "mean_IOU = mean_IOU / data_len\n",
    "final_accuracy = total_accuracy / data_len\n",
    "\n",
    "print(f'{args.metric} for {args.split} split')\n",
    "\n",
    "if args.metric == \"pointing_game\":\n",
    "    center_accuracy = base_correct / data_len\n",
    "    print(f'Center Accuracy: {center_accuracy}')\n",
    "\n",
    "print(f\"Accuracy:{final_accuracy}, Overall_IOU: {overall_IOU}, Mean_IOU: {mean_IOU}, Total: {n_iter}, \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results at mask_thresh: 0.3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pointing_game</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>command_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>non-temporal</th>\n",
       "      <td>0.493789</td>\n",
       "      <td>644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temporal</th>\n",
       "      <td>0.502890</td>\n",
       "      <td>519</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              pointing_game  count\n",
       "command_type                      \n",
       "non-temporal       0.493789    644\n",
       "temporal           0.502890    519"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f'Results at mask_thresh: {args.mask_thresh}')\n",
    "\n",
    "refined_map = {}\n",
    "for key1 in result_map:\n",
    "    refined_map[key1] = {}\n",
    "    total_len = 0\n",
    "    for key2 in result_map[key1]:\n",
    "        total_len += len(result_map[key1][key2])\n",
    "        accuracy = torch.tensor(result_map[key1][key2])\n",
    "        refined_map[key1][key2] = accuracy.mean().item()\n",
    "    refined_map[key1]['count'] = total_len // len(result_map[key1])\n",
    "\n",
    "df_result = pd.DataFrame.from_dict(refined_map,orient='index')\n",
    "df_result.index.name = 'command_type'\n",
    "\n",
    "df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}