MNIST ELM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9b357b66-913e-4015-b4b9-999e62ad9740",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2a6c7d5d-6498-4d1c-93ab-d4f3f7a6c48d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import itertools\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import Dataset, DataLoader, random_split, Subset\n",
    "from torchvision.utils import make_grid, save_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9621dbd2-c6d0-432d-bda3-fdd1074d8297",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfa58052-d47b-49f6-a8ec-97a1a75b874e",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0a9ee402-a3aa-4967-8e73-19455348e231",
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.1307,), (0.3081,))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e488c74c-d163-4f3d-b6f0-25ec9382c0ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = datasets.MNIST(root=\"./mnist\", train=True, download=True, transform=transform)\n",
    "test_dataset = datasets.MNIST(root=\"./mnist\", train=False, download=True, transform=transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "122a84bd-1755-419d-9c3b-42d910423f93",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57ee8402-c2ed-4122-961b-fb783086ab1c",
   "metadata": {},
   "source": [
    "# Extreme Learning Machine (ELM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c6062ea3-b9cf-4129-a9d5-974f504c1d53",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ELM(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(ELM, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.input_weights = nn.Linear(input_size, hidden_size, bias=True)\n",
    "        self.output_weights = None\n",
    "\n",
    "        with torch.no_grad():\n",
    "            nn.init.xavier_uniform_(self.input_weights.weight)\n",
    "            nn.init.zeros_(self.input_weights.bias)\n",
    "\n",
    "        for param in self.input_weights.parameters():\n",
    "            param.requires_grad = False\n",
    "\n",
    "    def forward(self, x):\n",
    "        H = torch.sigmoid(self.input_weights(x))\n",
    "        if self.training:\n",
    "            return H\n",
    "        else:\n",
    "            return H @ self.output_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "11476fe1-9ae6-4b43-853f-1072f7a80712",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ELM(\n",
       "  (input_weights): Linear(in_features=784, out_features=1000, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = ELM(input_size=28 * 28, hidden_size=1000, output_size=10)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e84459e9-e1f7-4da9-9919-0638a556c28f",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1c41b600-357a-4eb6-ac3f-c5d85f70ae1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "H_list = []\n",
    "T_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d01f79c3-7b8b-4d01-bad3-a16edf884bed",
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    for data, target in train_dataloader:\n",
    "        data = data.view(-1, 28 * 28).to(device)\n",
    "        target_onehot = F.one_hot(target, 10).float()\n",
    "\n",
    "        H = model(data)\n",
    "\n",
    "        H_list.append(H)\n",
    "        T_list.append(target_onehot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2ed8cbaf-bd55-45e8-8d26-aec25e3990f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "H = torch.cat(H_list, dim=0)\n",
    "T = torch.cat(T_list, dim=0).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b5d68111-2c14-430e-af71-dcef4368c476",
   "metadata": {},
   "outputs": [],
   "source": [
    "H_pinv = torch.pinverse(H)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b2892941-7398-4851-ad60-3338a23c6034",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.output_weights = H_pinv @ T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07475f21-a0f3-42c4-b7bb-12ff0f222a85",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_output = H @ model.output_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "08fa9622-1f87-4865-a623-f9d97c9df221",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = torch.argmax(train_output, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cd11de58-2420-4cdd-8209-d6f0460c9265",
   "metadata": {},
   "outputs": [],
   "source": [
    "target = torch.argmax(T, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f458a2ba-47c1-4ed8-b333-491673c6c897",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = (pred == target).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "394ff5e4-71b6-4e3d-b9e1-fbcc052761e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 0.9322333335876465\n"
     ]
    }
   ],
   "source": [
    "print(\"Train Accuracy:\", accuracy.cpu().detach().item())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac3cb70a-0ea6-45b1-90ab-1c4eb3cbeb52",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3be0d148-7619-4050-aafb-52e8cb8f431f",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "correct_pred = {class_name: 0 for class_name in classes}\n",
    "total_pred = {class_name: 0 for class_name in classes}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0ffb3a66-29d7-4456-9b4e-a27e4dac4086",
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "\n",
    "    for data, target in test_dataloader:\n",
    "        data = data.view(-1, 28 * 28).to(device)\n",
    "        \n",
    "        outputs = model(data)\n",
    "        _, predictions = torch.max(outputs, 1)\n",
    "        for label, prediction in zip(target, predictions):\n",
    "            if label == prediction:\n",
    "                correct_pred[classes[label]] += 1\n",
    "            total_pred[classes[label]] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f2ea1328-b9db-417c-a086-4b6c2708d30d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for class: 0 is 98.37 %\n",
      "Accuracy for class: 1 is 98.94 %\n",
      "Accuracy for class: 2 is 91.28 %\n",
      "Accuracy for class: 3 is 93.66 %\n",
      "Accuracy for class: 4 is 94.60 %\n",
      "Accuracy for class: 5 is 88.79 %\n",
      "Accuracy for class: 6 is 95.72 %\n",
      "Accuracy for class: 7 is 91.34 %\n",
      "Accuracy for class: 8 is 89.32 %\n",
      "Accuracy for class: 9 is 89.89 %\n"
     ]
    }
   ],
   "source": [
    "for class_name, correct_count in correct_pred.items():\n",
    "    accuracy = 100 * float(correct_count) / total_pred[class_name]\n",
    "    print(f'Accuracy for class: {class_name} is {accuracy:.2f} %')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a1040edc-ff9f-4384-839f-f9395597963d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Accuracy: 93.29 %\n"
     ]
    }
   ],
   "source": [
    "total_accuracy = 100 * float(np.sum(list(correct_pred.values())) / np.sum(list(total_pred.values())))\n",
    "print(f'Total Accuracy: {total_accuracy:.2f} %')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a2904d6-b4ee-4583-b8cd-e31ab2879239",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}