fDBD_demo_on_openood.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "In this notebook, we demo implementation and performance on OpenOOD Benchmark [1]\n",
        "\n",
        "Ref: [1] https://github.com/Jingkang50/OpenOOD/tree/main"
      ],
      "metadata": {
        "id": "sXtASgVQT8pZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d0YB13dB9dki",
        "outputId": "f51bdbbd-24fc-44b2-cf52-1e5b050950da"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tue May 28 04:07:25 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   65C    P8              10W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. Install OpenOOD with pip and make necessary preparation"
      ],
      "metadata": {
        "id": "k4KGROHCk5b-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip show OpenOOD"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pkjkmG164vfO",
        "outputId": "657133c2-10cc-43cd-986a-668941ec04fd"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[33mWARNING: Package(s) not found: OpenOOD\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "FZnYwXFgjBcE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8abf64f1-eb27-4bad-d22b-6bef7f20d7e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/Jingkang50/OpenOOD\n",
            "  Cloning https://github.com/Jingkang50/OpenOOD to /tmp/pip-req-build-7yz7l7p3\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/Jingkang50/OpenOOD /tmp/pip-req-build-7yz7l7p3\n",
            "  Resolved https://github.com/Jingkang50/OpenOOD to commit 18c6f5174a2f518e2a8e819ffb1cd1914bcf12e0\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (2.3.0+cu121)\n",
            "Requirement already satisfied: torchvision>=0.13 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (0.18.0+cu121)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (1.2.2)\n",
            "Collecting json5 (from openood==1.5)\n",
            "  Downloading json5-0.9.25-py3-none-any.whl (30 kB)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (3.7.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (1.11.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (4.66.4)\n",
            "Requirement already satisfied: pyyaml>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (6.0.1)\n",
            "Collecting pre-commit (from openood==1.5)\n",
            "  Downloading pre_commit-3.7.1-py2.py3-none-any.whl (204 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m204.3/204.3 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: opencv-python>=4.4.0.46 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (4.8.0.76)\n",
            "Requirement already satisfied: imgaug>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (0.4.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (2.0.3)\n",
            "Collecting diffdist>=0.1 (from openood==1.5)\n",
            "  Downloading diffdist-0.1.tar.gz (4.6 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: Cython>=0.29.30 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (3.0.10)\n",
            "Collecting faiss-gpu>=1.7.2 (from openood==1.5)\n",
            "  Downloading faiss_gpu-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (85.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 MB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: gdown>=4.7.1 in /usr/local/lib/python3.10/dist-packages (from openood==1.5) (5.1.0)\n",
            "Collecting libmr>=0.1.9 (from openood==1.5)\n",
            "  Downloading libmr-0.1.9.zip (39 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from gdown>=4.7.1->openood==1.5) (4.12.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from gdown>=4.7.1->openood==1.5) (3.14.0)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.10/dist-packages (from gdown>=4.7.1->openood==1.5) (2.31.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from imgaug>=0.4.0->openood==1.5) (1.16.0)\n",
            "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.10/dist-packages (from imgaug>=0.4.0->openood==1.5) (1.25.2)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from imgaug>=0.4.0->openood==1.5) (9.4.0)\n",
            "Requirement already satisfied: scikit-image>=0.14.2 in /usr/local/lib/python3.10/dist-packages (from imgaug>=0.4.0->openood==1.5) (0.19.3)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.10/dist-packages (from imgaug>=0.4.0->openood==1.5) (2.31.6)\n",
            "Requirement already satisfied: Shapely in /usr/local/lib/python3.10/dist-packages (from imgaug>=0.4.0->openood==1.5) (2.0.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->openood==1.5) (4.11.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->openood==1.5) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->openood==1.5) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->openood==1.5) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->openood==1.5) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.13.1->openood==1.5)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->openood==1.5) (2.3.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->openood==1.5)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m74.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (1.2.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (4.51.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (1.4.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (24.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (3.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->openood==1.5) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->openood==1.5) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->openood==1.5) (2024.1)\n",
            "Collecting cfgv>=2.0.0 (from pre-commit->openood==1.5)\n",
            "  Downloading cfgv-3.4.0-py2.py3-none-any.whl (7.2 kB)\n",
            "Collecting identify>=1.0.0 (from pre-commit->openood==1.5)\n",
            "  Downloading identify-2.5.36-py2.py3-none-any.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.0/99.0 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nodeenv>=0.11.1 (from pre-commit->openood==1.5)\n",
            "  Downloading nodeenv-1.8.0-py2.py3-none-any.whl (22 kB)\n",
            "Collecting virtualenv>=20.10.0 (from pre-commit->openood==1.5)\n",
            "  Downloading virtualenv-20.26.2-py3-none-any.whl (3.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.9/3.9 MB\u001b[0m \u001b[31m83.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->openood==1.5) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->openood==1.5) (3.5.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from nodeenv>=0.11.1->pre-commit->openood==1.5) (67.7.2)\n",
            "Requirement already satisfied: tifffile>=2019.7.26 in /usr/local/lib/python3.10/dist-packages (from scikit-image>=0.14.2->imgaug>=0.4.0->openood==1.5) (2024.5.10)\n",
            "Requirement already satisfied: PyWavelets>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-image>=0.14.2->imgaug>=0.4.0->openood==1.5) (1.6.0)\n",
            "Collecting distlib<1,>=0.3.7 (from virtualenv>=20.10.0->pre-commit->openood==1.5)\n",
            "  Downloading distlib-0.3.8-py2.py3-none-any.whl (468 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.9/468.9 kB\u001b[0m \u001b[31m42.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: platformdirs<5,>=3.9.1 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.10.0->pre-commit->openood==1.5) (4.2.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->gdown>=4.7.1->openood==1.5) (2.5)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->openood==1.5) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown>=4.7.1->openood==1.5) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown>=4.7.1->openood==1.5) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown>=4.7.1->openood==1.5) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown>=4.7.1->openood==1.5) (2024.2.2)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown>=4.7.1->openood==1.5) (1.7.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->openood==1.5) (1.3.0)\n",
            "Building wheels for collected packages: openood, diffdist, libmr\n",
            "  Building wheel for openood (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for openood: filename=openood-1.5-py3-none-any.whl size=256506 sha256=26dcf22c12331fbfa90b09fe85e5233099d7b4ada7c936b406063248b2ac2e5f\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-dxqob_2k/wheels/c1/8c/95/c9073d7126855900f36c727dbefa70e125ca290a55d3deca0f\n",
            "  Building wheel for diffdist (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for diffdist: filename=diffdist-0.1-py3-none-any.whl size=6536 sha256=8ea0692206a45ffb7de860f4dd7de806dea6c473c1a2b0604ea1bb7bcb07c301\n",
            "  Stored in directory: /root/.cache/pip/wheels/f1/29/f6/5331a14ab74e769799b76eb32460f447c7feb7375a16b07854\n",
            "  Building wheel for libmr (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for libmr: filename=libmr-0.1.9-cp310-cp310-linux_x86_64.whl size=549303 sha256=21695ba8cc298f498dd14a6c11729180b1c8364f8ab18ccbe97711d8e7422299\n",
            "  Stored in directory: /root/.cache/pip/wheels/8a/e0/45/ec5f4f802b034150d6f5735922408cc0278bf85582b2a1a954\n",
            "Successfully built openood diffdist libmr\n",
            "Installing collected packages: faiss-gpu, distlib, diffdist, virtualenv, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nodeenv, libmr, json5, identify, cfgv, pre-commit, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, openood\n",
            "Successfully installed cfgv-3.4.0 diffdist-0.1 distlib-0.3.8 faiss-gpu-1.7.2 identify-2.5.36 json5-0.9.25 libmr-0.1.9 nodeenv-1.8.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 openood-1.5 pre-commit-3.7.1 virtualenv-20.26.2\n"
          ]
        }
      ],
      "source": [
        "!pip install git+https://github.com/Jingkang50/OpenOOD"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# necessary imports\n",
        "import torch\n",
        "\n",
        "from openood.evaluation_api import Evaluator\n",
        "from openood.networks import ResNet18_32x32 #just a wrapper around the ResNet\n",
        "from openood.networks.ash_net import ASHNet"
      ],
      "metadata": {
        "id": "dZ3_c6oHkfsb"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# download our pre-trained CIFAR-10 classifier\n",
        "!gdown 1byGeYxM_PlLjT72wZsMQvP6popJeWBgt\n",
        "!unzip cifar10_res18_v1.5.zip"
      ],
      "metadata": {
        "id": "Ye0PofseaDiJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9b51d945-6ebf-43d8-bdcc-80e8c2d9f161"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1byGeYxM_PlLjT72wZsMQvP6popJeWBgt\n",
            "From (redirected): https://drive.google.com/uc?id=1byGeYxM_PlLjT72wZsMQvP6popJeWBgt&confirm=t&uuid=af74a0b7-fa53-4990-b12d-5fe08edf39a8\n",
            "To: /content/cifar10_res18_v1.5.zip\n",
            "100% 375M/375M [00:09<00:00, 38.6MB/s]\n",
            "Archive:  cifar10_res18_v1.5.zip\n",
            "   creating: cifar10_resnet18_32x32_base_e100_lr0.1_default/\n",
            "   creating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s2/\n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s2/best_epoch99_acc0.9450.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s2/config.yml  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s2/best.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s2/last_epoch100_acc0.9420.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s2/log.txt  \n",
            "   creating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s1/\n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s1/config.yml  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s1/best.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s1/best_epoch95_acc0.9500.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s1/last_epoch100_acc0.9490.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s1/log.txt  \n",
            "   creating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/\n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/config.yml  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best_epoch96_acc0.9470.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/last_epoch100_acc0.9430.ckpt  \n",
            "  inflating: cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/log.txt  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# load the model\n",
        "net = ResNet18_32x32(num_classes=10)\n",
        "net.load_state_dict(\n",
        "    torch.load('./cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt')\n",
        ")\n",
        "net.cuda()\n",
        "net.eval()"
      ],
      "metadata": {
        "id": "9e6mehWBa5Td",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e527a3b-92f2-47be-9224-0dc049fcbcee"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ResNet18_32x32(\n",
              "  (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  (layer1): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential()\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential()\n",
              "    )\n",
              "  )\n",
              "  (layer2): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential(\n",
              "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential()\n",
              "    )\n",
              "  )\n",
              "  (layer3): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential(\n",
              "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential()\n",
              "    )\n",
              "  )\n",
              "  (layer4): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential(\n",
              "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (shortcut): Sequential()\n",
              "    )\n",
              "  )\n",
              "  (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
              "  (fc): Linear(in_features=512, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "2. SOTA Baseline"
      ],
      "metadata": {
        "id": "awILVNEjcjF3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "knn_evaluator = Evaluator(\n",
        "    net,\n",
        "    id_name='cifar10',                     # the target ID dataset\n",
        "    data_root='./data',                    # change if necessary\n",
        "    config_root=None,                      # see notes above\n",
        "    preprocessor=None,                     # default preprocessing for the target ID dataset\n",
        "    postprocessor_name='knn',              # the postprocessor to use\n",
        "    postprocessor=None,                    # if you want to use your own postprocessor\n",
        "    batch_size=200,                        # for certain methods the results can be slightly affected by batch size\n",
        "    shuffle=False,\n",
        "    num_workers=2)                         # could use more num_workers outside colab"
      ],
      "metadata": {
        "id": "lZRgkhPimiDh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "983ce9ba-47b6-45cf-abcb-75e6efdc7a59"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1XKzBdWCqg3vPoj-D32YixJyJJ0hL63gP\n",
            "From (redirected): https://drive.google.com/uc?id=1XKzBdWCqg3vPoj-D32YixJyJJ0hL63gP&confirm=t&uuid=a20bab61-f7ec-484a-a460-44409751cea2\n",
            "To: /content/data/benchmark_imglist.zip\n",
            "100%|██████████| 27.7M/27.7M [00:01<00:00, 18.7MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cifar10 needs download:\n",
            "./data/images_classic/cifar10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1Co32RiiWe16lTaiOU6JMMnyUYS41IlO1\n",
            "From (redirected): https://drive.google.com/uc?id=1Co32RiiWe16lTaiOU6JMMnyUYS41IlO1&confirm=t&uuid=5178b001-0fab-422c-9259-d2c6b4bffb51\n",
            "To: /content/data/images_classic/cifar10/cifar10.zip\n",
            "100%|██████████| 143M/143M [00:02<00:00, 53.3MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cifar100 needs download:\n",
            "./data/images_classic/cifar100\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1PGKheHUsf29leJPPGuXqzLBMwl8qMF8_\n",
            "From (redirected): https://drive.google.com/uc?id=1PGKheHUsf29leJPPGuXqzLBMwl8qMF8_&confirm=t&uuid=5d396f46-8a73-4301-8ba8-491b5ef054e9\n",
            "To: /content/data/images_classic/cifar100/cifar100.zip\n",
            "100%|██████████| 141M/141M [00:01<00:00, 80.7MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tin needs download:\n",
            "./data/images_classic/tin\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1PZ-ixyx52U989IKsMA2OT-24fToTrelC\n",
            "From (redirected): https://drive.google.com/uc?id=1PZ-ixyx52U989IKsMA2OT-24fToTrelC&confirm=t&uuid=d8ae713f-763b-4237-8458-bf7beb3e1fc4\n",
            "To: /content/data/images_classic/tin/tin.zip\n",
            "100%|██████████| 237M/237M [00:02<00:00, 106MB/s] \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mnist needs download:\n",
            "./data/images_classic/mnist\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1CCHAGWqA1KJTFFswuF9cbhmB-j98Y1Sb\n",
            "From (redirected): https://drive.google.com/uc?id=1CCHAGWqA1KJTFFswuF9cbhmB-j98Y1Sb&confirm=t&uuid=8ec930b0-031a-4594-926d-8618ce074915\n",
            "To: /content/data/images_classic/mnist/mnist.zip\n",
            "100%|██████████| 47.2M/47.2M [00:01<00:00, 40.7MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "svhn needs download:\n",
            "./data/images_classic/svhn\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1DQfc11HOtB1nEwqS4pWUFp8vtQ3DczvI\n",
            "To: /content/data/images_classic/svhn/svhn.zip\n",
            "100%|██████████| 19.0M/19.0M [00:00<00:00, 136MB/s] \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "texture needs download:\n",
            "./data/images_classic/texture\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1OSz1m3hHfVWbRdmMwKbUzoU8Hg9UKcam\n",
            "From (redirected): https://drive.google.com/uc?id=1OSz1m3hHfVWbRdmMwKbUzoU8Hg9UKcam&confirm=t&uuid=3331b3ae-ec7e-4fc4-bb58-e2b27115df1b\n",
            "To: /content/data/images_classic/texture/texture.zip\n",
            "100%|██████████| 626M/626M [00:08<00:00, 74.8MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "places365 needs download:\n",
            "./data/images_classic/places365\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1Ec-LRSTf6u5vEctKX9vRp9OA6tqnJ0Ay\n",
            "From (redirected): https://drive.google.com/uc?id=1Ec-LRSTf6u5vEctKX9vRp9OA6tqnJ0Ay&confirm=t&uuid=3af181f2-93f4-4ee1-bfaa-fd1f7f55b656\n",
            "To: /content/data/images_classic/places365/places365.zip\n",
            "100%|██████████| 497M/497M [00:07<00:00, 63.5MB/s]\n",
            "Setup: 100%|██████████| 250/250 [00:45<00:00,  5.48it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Starting automatic parameter search...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [00:02<00:00,  2.26it/s]\n",
            "100%|██████████| 5/5 [00:02<00:00,  2.30it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparam: [50], auroc: 0.905803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [00:02<00:00,  2.38it/s]\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.66it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparam: [100], auroc: 0.9042669999999999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [00:03<00:00,  1.34it/s]\n",
            "100%|██████████| 5/5 [00:02<00:00,  2.14it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparam: [200], auroc: 0.902765\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [00:02<00:00,  2.19it/s]\n",
            "100%|██████████| 5/5 [00:02<00:00,  2.06it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparam: [500], auroc: 0.9000844999999998\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [00:02<00:00,  2.05it/s]\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.50it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparam: [1000], auroc: 0.8976035\n",
            "Final hyperparam: 50\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "metrics = knn_evaluator.eval_ood(fsood=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kAObP3R0CLUm",
        "outputId": "16115487-2d8a-4138-e7ba-c5e2b5bfd3d9"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Performing inference on cifar10 test set...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 45/45 [00:19<00:00,  2.26it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing near ood...\n",
            "Performing inference on cifar100 dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 45/45 [00:18<00:00,  2.49it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on cifar100 dataset...\n",
            "FPR@95: 37.91, AUROC: 89.75 AUPR_IN: 90.11, AUPR_OUT: 88.37\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on tin dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 39/39 [00:17<00:00,  2.28it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on tin dataset...\n",
            "FPR@95: 31.18, AUROC: 91.65 AUPR_IN: 93.35, AUPR_OUT: 89.06\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Computing mean metrics...\n",
            "FPR@95: 34.54, AUROC: 90.70 AUPR_IN: 91.73, AUPR_OUT: 88.71\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Processing far ood...\n",
            "Performing inference on mnist dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 350/350 [02:23<00:00,  2.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on mnist dataset...\n",
            "FPR@95: 20.62, AUROC: 94.41 AUPR_IN: 82.75, AUPR_OUT: 99.01\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on svhn dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 131/131 [00:51<00:00,  2.54it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on svhn dataset...\n",
            "FPR@95: 20.83, AUROC: 92.89 AUPR_IN: 88.98, AUPR_OUT: 96.21\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on texture dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 29/29 [00:34<00:00,  1.19s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on texture dataset...\n",
            "FPR@95: 24.56, AUROC: 93.02 AUPR_IN: 96.04, AUPR_OUT: 87.10\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on places365 dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 176/176 [01:45<00:00,  1.68it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on places365 dataset...\n",
            "FPR@95: 29.50, AUROC: 92.10 AUPR_IN: 81.27, AUPR_OUT: 97.37\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Computing mean metrics...\n",
            "FPR@95: 23.88, AUROC: 93.11 AUPR_IN: 87.26, AUPR_OUT: 94.92\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "ID Acc Eval: 100%|██████████| 45/45 [00:09<00:00,  4.88it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "           FPR@95  AUROC  AUPR_IN  AUPR_OUT   ACC\n",
            "cifar100    37.91  89.75    90.11     88.37 95.22\n",
            "tin         31.18  91.65    93.35     89.06 95.22\n",
            "nearood     34.54  90.70    91.73     88.71 95.22\n",
            "mnist       20.62  94.41    82.75     99.01 95.22\n",
            "svhn        20.83  92.89    88.98     96.21 95.22\n",
            "texture     24.56  93.02    96.04     87.10 95.22\n",
            "places365   29.50  92.10    81.27     97.37 95.22\n",
            "farood      23.88  93.11    87.26     94.92 95.22\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "5. fDBD Implementation and Evaluation"
      ],
      "metadata": {
        "id": "3o5l-Si_j66P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "for i, param in enumerate(net.fc.parameters()):\n",
        "  if i == 0:\n",
        "    w = param.data.cpu().numpy()\n",
        "  else:\n",
        "    b = param.data.cpu().numpy()"
      ],
      "metadata": {
        "id": "hM-IKpxqI-dg"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "denominator_matrix = np.zeros((10,10))\n",
        "for p in range(10):\n",
        "  w_p = w - w[p,:]\n",
        "  denominator = np.linalg.norm(w_p, axis=1)\n",
        "  denominator[p] = 1\n",
        "  denominator_matrix[p, :] = denominator\n",
        "\n",
        "denominator_matrix = torch.tensor(denominator_matrix).cuda()"
      ],
      "metadata": {
        "id": "WWe-_0Lzbfig"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_mean = np.mean(knn_evaluator.postprocessor.activation_log, axis=0)\n",
        "train_mean_tensor = torch.from_numpy(train_mean).cuda()"
      ],
      "metadata": {
        "id": "LFEnEKRkm375"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import Any\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from openood.postprocessors import BasePostprocessor\n",
        "\n",
        "class fDBDPostprocessor(BasePostprocessor):\n",
        "    def __init__(self, config):\n",
        "        super(fDBDPostprocessor, self).__init__(config)\n",
        "        self.APS_mode = False\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def postprocess(self, net: nn.Module, data: Any):\n",
        "        output, feature = net(data, return_feature=True)\n",
        "        values, nn_idx = output.max(1)\n",
        "        logits_sub = torch.abs(output - values.repeat(10, 1).T)\n",
        "        score = torch.sum(logits_sub/denominator_matrix[nn_idx], axis=1)/torch.norm(feature - train_mean_tensor , dim = 1)\n",
        "        return nn_idx, score"
      ],
      "metadata": {
        "id": "uF4L9geeFYjU"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fdbd_evaluator = Evaluator(\n",
        "    net,\n",
        "    id_name='cifar10',                     # the target ID dataset\n",
        "    data_root='./data',                    # change if necessary\n",
        "    config_root=None,                      # see notes above\n",
        "    preprocessor=None,                     # default preprocessing for the target ID dataset\n",
        "    #postprocessor_name=postprocessor_name, # the postprocessor to use\n",
        "    postprocessor= fDBDPostprocessor('./configs'),       # if you want to use your own postprocessor\n",
        "    batch_size=200,                        # for certain methods the results can be slightly affected by batch size\n",
        "    shuffle=False,\n",
        "    num_workers=2)                         # could use more num_workers outside colab"
      ],
      "metadata": {
        "id": "LN0rOtgkHvVM"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "metrics = fdbd_evaluator.eval_ood(fsood=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PlKcE-YPJRJf",
        "outputId": "1dd04c7a-d7f3-4ab4-ba56-cd52feac1c33"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Performing inference on cifar10 test set...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 45/45 [00:12<00:00,  3.69it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing near ood...\n",
            "Performing inference on cifar100 dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 45/45 [00:06<00:00,  6.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on cifar100 dataset...\n",
            "FPR@95: 41.14, AUROC: 89.36 AUPR_IN: 88.80, AUPR_OUT: 88.26\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on tin dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 39/39 [00:09<00:00,  4.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on tin dataset...\n",
            "FPR@95: 31.71, AUROC: 91.48 AUPR_IN: 92.49, AUPR_OUT: 89.26\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Computing mean metrics...\n",
            "FPR@95: 36.43, AUROC: 90.42 AUPR_IN: 90.64, AUPR_OUT: 88.76\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Processing far ood...\n",
            "Performing inference on mnist dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 350/350 [01:00<00:00,  5.74it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on mnist dataset...\n",
            "FPR@95: 20.28, AUROC: 94.59 AUPR_IN: 78.55, AUPR_OUT: 99.10\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on svhn dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 131/131 [00:23<00:00,  5.49it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on svhn dataset...\n",
            "FPR@95: 24.18, AUROC: 91.89 AUPR_IN: 84.48, AUPR_OUT: 95.90\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on texture dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 29/29 [00:28<00:00,  1.03it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on texture dataset...\n",
            "FPR@95: 24.98, AUROC: 92.83 AUPR_IN: 95.65, AUPR_OUT: 87.28\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Performing inference on places365 dataset...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 176/176 [01:07<00:00,  2.61it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing metrics on places365 dataset...\n",
            "FPR@95: 26.40, AUROC: 92.70 AUPR_IN: 82.24, AUPR_OUT: 97.60\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n",
            "Computing mean metrics...\n",
            "FPR@95: 23.96, AUROC: 93.00 AUPR_IN: 85.23, AUPR_OUT: 94.97\n",
            "──────────────────────────────────────────────────────────────────────\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "ID Acc Eval: 100%|██████████| 45/45 [00:06<00:00,  6.66it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "           FPR@95  AUROC  AUPR_IN  AUPR_OUT   ACC\n",
            "cifar100    41.14  89.36    88.80     88.26 95.22\n",
            "tin         31.71  91.48    92.49     89.26 95.22\n",
            "nearood     36.43  90.42    90.64     88.76 95.22\n",
            "mnist       20.28  94.59    78.55     99.10 95.22\n",
            "svhn        24.18  91.89    84.48     95.90 95.22\n",
            "texture     24.98  92.83    95.65     87.28 95.22\n",
            "places365   26.40  92.70    82.24     97.60 95.22\n",
            "farood      23.96  93.00    85.23     94.97 95.22\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    }
  ]
}