evaluation/percentage plot-single network_BL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is used to generate the TPER curves across various datasets for a given network.\n",
    "\n",
    "The format of the input file is:\n",
    "```\n",
    "image_name_1, target_1, predicted_1\n",
    "image_name_2, target_2, predicted_2\n",
    ".\n",
    ".\n",
    ".\n",
    "image_name_n, target_n, predicted_n\n",
    "```\n",
    "\n",
    "The plot from this set of cells is a Area Under the Curve plot. As a demo the txt files generated for the BL network are added to this repo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "import numpy as np\n",
    "import plotly.graph_objects as go\n",
    "from numpy import trapz\n",
    "from scipy.integrate import simps\n",
    "\n",
    "network_name = \"BL\"\n",
    "\n",
    "def load_data(path):\n",
    "    f = open(path, \"r\")\n",
    "    dm =[]\n",
    "    for x in f:\n",
    "        x=x.replace('\\n','').split(',')\n",
    "        x =[float(i) for i in x]\n",
    "        # print(x)\n",
    "        dm.append(x)\n",
    "\n",
    "    dm = np.asarray(dm)\n",
    "\n",
    "    if dm.shape[1]==4:\n",
    "        dm = np.delete(dm, 1, 1) # remove the extra error column if present\n",
    "    \n",
    "    return dm\n",
    "\n",
    "def get_percent_error(dm):\n",
    "    error =[]\n",
    "    for i,j,k in dm:\n",
    "        error.append(abs(j-k)+0.001/j+0.001) # added the 0.001 to avoid zero count error in the deno\n",
    "    return error\n",
    "\n",
    "# tetas =\n",
    " \n",
    "def get_teta_and_er(dm,tetas):\n",
    "    error = get_percent_error(dm)\n",
    "    fin =[]\n",
    "\n",
    "    for t in tetas:\n",
    "        cnt_t =0\n",
    "        for i in error:\n",
    "            if i>=t:\n",
    "                cnt_t+=1\n",
    "        fin.append([t,cnt_t/dm.shape[0]])\n",
    "        \n",
    "    return np.asarray(fin)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'BL_txt\\\\jhu_unbinned_tar_pred_error.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_18864/2831177877.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mtetas\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m21\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdm\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mload_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnetwork_name\u001b[0m\u001b[1;33m+\u001b[0m \u001b[1;34mr\"_txt\\jhu_unbinned_tar_pred_error.txt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mbl_jhu\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_teta_and_er\u001b[0m\u001b[1;31m# .txtbl_jhu = np.asarray(bl_jhu)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0marea_bl_jhu\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrapz\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbl_jhu\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbl_jhu\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_18864/1836753539.py\u001b[0m in \u001b[0;36mload_data\u001b[1;34m(path)\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mload_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"r\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[0mdm\u001b[0m \u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'BL_txt\\\\jhu_unbinned_tar_pred_error.txt'"
     ]
    }
   ],
   "source": [
    "\n",
    "tetas = np.linspace(0,100,21)\n",
    "dm = load_data(path = network_name+ r\"_txt\\jhu_unbinned_tar_pred_error.txt\")\n",
    "bl_jhu = get_teta_and_er# .txtbl_jhu = np.asarray(bl_jhu)\n",
    "area_bl_jhu = trapz(y=bl_jhu[:,1], x=bl_jhu[:,0])\n",
    "\n",
    "# tetas = np.linspace(0,100,21)\n",
    "dm = load_data(path = network_name+ r\"_txt\\nwpu_unbinned_tar_pred_error.txt\")\n",
    "bl_nwpu = get_teta_and_er(dm,tetas= tetas)\n",
    "# bl_jhu = np.asarray(bl_jhu)\n",
    "area_bl_nwpu = trapz(y=bl_nwpu[:,1], x=bl_nwpu[:,0])\n",
    "\n",
    "# tetas = np.linspace(0,100,21)\n",
    "dm = load_data(path =  network_name+  r\"_txt\\ucf_unbinned_tar_pred_error.txt\")\n",
    "bl_ucf = get_teta_and_er(dm,tetas= tetas)\n",
    "# bl_jhu = np.asarray(bl_jhu)\n",
    "area_bl_ucf = trapz(y=bl_ucf[:,1], x=bl_ucf[:,0])\n",
    "\n",
    "dm = load_data(path = network_name+   r\"_txt\\sta_unbinned_tar_pred_error.txt\")\n",
    "bl_sta = get_teta_and_er(dm,tetas= tetas)\n",
    "# bl_jhu = np.asarray(bl_jhu)\n",
    "area_bl_sta = trapz(y=bl_sta[:,1], x=bl_sta[:,0])\n",
    "\n",
    "dm = load_data(path = network_name+  r\"_txt\\stb_unbinned_tar_pred_error.txt\")\n",
    "bl_stb = get_teta_and_er(dm,tetas= tetas)\n",
    "# bl_jhu = np.asarray(bl_jhu)\n",
    "area_bl_stb = trapz(y=bl_stb[:,1], x=bl_stb[:,0])\n",
    "\n",
    "\n",
    "fig = go.Figure(layout=go.Layout(#title=go.layout.Title(text=\"Error Ratio of the Model BL over all the datasets based on the Percentage error\"),\n",
    "                                 xaxis=go.layout.XAxis(title=go.layout.xaxis.Title(text=\"Percentage Error Threshold -- θ\")),\n",
    "                                 yaxis=go.layout.YAxis(title=go.layout.yaxis.Title(text=\"Error Ratio for a given θ of the model\"))))\n",
    "lis =[area_bl_jhu]*(bl_jhu.shape[0])\n",
    "print(len(lis))\n",
    "\n",
    "\n",
    "# new_x = np.linspace(0,max(x),num=int(max(x)))\n",
    "\n",
    "\n",
    "fig.add_trace(go.Scatter(x=bl_ucf[:,0], y=bl_ucf[:,1], #opacity=0.3,\n",
    "                         mode='lines', line=dict(width=4),name=\"UCF,    AUC: \"+str(abs(round(area_bl_ucf,2))))\n",
    "              )\n",
    "fig.add_trace(go.Scatter(x=bl_sta[:,0], y=bl_sta[:,1], #opacity=0.3,\n",
    "                         mode='lines', line=dict(width=4),name=\"STA,    AUC: \"+str(abs(round(area_bl_sta,2))))\n",
    "              )\n",
    "fig.add_trace(go.Scatter(x=bl_nwpu[:,0], y=bl_nwpu[:,1],#opacity=0.3,\n",
    "                         mode='lines', line=dict(width=4),name=\"NWPU, AUC: \"+str(abs(round(area_bl_nwpu,2))))\n",
    "              \n",
    "              ) # fill down to xaxis\n",
    "\n",
    "fig.add_trace(go.Scatter(x=bl_jhu[:,0], y=bl_jhu[:,1],#opacity=0.3,\n",
    "                         mode='lines', line=dict(width=4),name=\"JHU,    AUC: \"+str(abs(round(area_bl_jhu,2))))\n",
    "              ) # fill down to xaxis\n",
    "\n",
    "\n",
    "fig.add_trace(go.Scatter(x=bl_stb[:,0], y=bl_stb[:,1], #opacity=0.3,\n",
    "                         mode='lines', line=dict(width=4),name=\"STB,    AUC: \"+str(abs(round(area_bl_stb,2))))\n",
    "              )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "fig.update_layout(plot_bgcolor='rgba(0,0,0,0)',xaxis=dict(showgrid=True),\n",
    "                  yaxis=dict(showgrid=True,gridcolor='rgb(211,211,211)'),\n",
    "                  legend=dict(yanchor=\"top\",xanchor=\"right\",x= 0.99,y=0.99), \n",
    "                  font=dict(size=15,color=\"#000000\"),margin=dict(l=20 , r=20, t=40, b=20))\n",
    "\n",
    "fig.update_layout(\n",
    "    xaxis = dict(\n",
    "        tickangle = 75,\n",
    "        tickmode = 'array',\n",
    "        tickvals = [1,10,20,30,40,50,60,70,80,90,100],\n",
    "        ticktext = [\"1\",\"10\",\"20\",\"30\",\"40\",\"50\",\"60\",\"70\",\"80\",\"90\",\"100+\"]\n",
    "    )\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600\n",
    ")\n",
    "# fig.update_layout(hovermode=\"x\")\n",
    "fig.write_image(\"auc_bl.jpg\",scale=4)\n",
    "# fig.write_html(\"bl.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "760acb839ed56255c4d8baa0996141e5d9016d8f567c53d1a1d5d86c45970246"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}