renet_50.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import time\n",
    "import copy\n",
    "import os\n",
    "import random\n",
    "torch.manual_seed(43)\n",
    "random.seed(43)\n",
    "\n",
    "torch.multiprocessing.set_sharing_strategy('file_system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mixup_criterion(criterion, pred, y_a, y_b, lam):\n",
    "    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n",
    "\n",
    "def mixup_data(x, y, alpha=1.0, use_cuda=True):\n",
    "    '''Returns mixed inputs, pairs of targets, and lambda'''\n",
    "    if alpha > 0:\n",
    "        lam = np.random.beta(alpha, alpha)\n",
    "    else:\n",
    "        lam = 1\n",
    "\n",
    "    batch_size = x.size()[0]\n",
    "    if use_cuda:\n",
    "        index = torch.randperm(batch_size).cuda()\n",
    "    else:\n",
    "        index = torch.randperm(batch_size)\n",
    "        \n",
    "    mixed_x = lam * x + (1 - lam) * x[index, :]\n",
    "    y_a, y_b = y, y[index]\n",
    "\n",
    "    return mixed_x, y_a, y_b, lam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class BasicBlock(nn.Module):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_planes, planes, stride=1):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(\n",
    "            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(planes)\n",
    "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,\n",
    "                               stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(planes)\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_planes != self.expansion*planes:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_planes, self.expansion*planes,\n",
    "                          kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(self.expansion*planes)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        out += self.shortcut(x)\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "class Bottleneck(nn.Module):\n",
    "    expansion = 4\n",
    "\n",
    "    def __init__(self, in_planes, planes, stride=1):\n",
    "        super(Bottleneck, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(planes)\n",
    "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,\n",
    "                               stride=stride, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(planes)\n",
    "        self.conv3 = nn.Conv2d(planes, self.expansion *\n",
    "                               planes, kernel_size=1, bias=False)\n",
    "        self.bn3 = nn.BatchNorm2d(self.expansion*planes)\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_planes != self.expansion*planes:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_planes, self.expansion*planes,\n",
    "                          kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(self.expansion*planes)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = F.relu(self.bn2(self.conv2(out)))\n",
    "        out = self.bn3(self.conv3(out))\n",
    "        out += self.shortcut(x)\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, num_blocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_planes = 64\n",
    "\n",
    "        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,\n",
    "                               stride=1, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(64)\n",
    "        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n",
    "        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n",
    "        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n",
    "        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n",
    "        self.linear = nn.Linear(512*block.expansion, num_classes)\n",
    "\n",
    "    def _make_layer(self, block, planes, num_blocks, stride):\n",
    "        strides = [stride] + [1]*(num_blocks-1)\n",
    "        layers = []\n",
    "        for stride in strides:\n",
    "            layers.append(block(self.in_planes, planes, stride))\n",
    "            self.in_planes = planes * block.expansion\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.layer4(out)\n",
    "        out = F.avg_pool2d(out, 4)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.linear(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "def ResNet18():\n",
    "    return ResNet(BasicBlock, [2, 2, 2, 2])\n",
    "\n",
    "\n",
    "def ResNet34():\n",
    "    return ResNet(BasicBlock, [3, 4, 6, 3])\n",
    "\n",
    "\n",
    "def ResNet50():\n",
    "    return ResNet(Bottleneck, [3, 4, 6, 3])\n",
    "\n",
    "\n",
    "def ResNet101():\n",
    "    return ResNet(Bottleneck, [3, 4, 23, 3])\n",
    "\n",
    "\n",
    "def ResNet152():\n",
    "    return ResNet(Bottleneck, [3, 8, 36, 3])\n",
    "\n",
    "\n",
    "def test():\n",
    "    net = ResNet18()\n",
    "    y = net(torch.randn(1, 3, 32, 32))\n",
    "    print(y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "\n",
      "Epoch: 0\n",
      "Loss: 1.838 | Acc: 34.802% (17401/50000)\n",
      "Val Loss: 1.559 | Acc: 45.780% (4578/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 1\n",
      "Loss: 1.310 | Acc: 52.934% (26467/50000)\n",
      "Val Loss: 1.131 | Acc: 61.360% (6136/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 2\n",
      "Loss: 1.058 | Acc: 62.544% (31272/50000)\n",
      "Val Loss: 0.925 | Acc: 67.080% (6708/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 3\n",
      "Loss: 0.868 | Acc: 69.262% (34631/50000)\n",
      "Val Loss: 0.802 | Acc: 73.140% (7314/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 4\n",
      "Loss: 0.727 | Acc: 74.620% (37310/50000)\n",
      "Val Loss: 0.727 | Acc: 76.010% (7601/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 5\n",
      "Loss: 0.623 | Acc: 78.336% (39168/50000)\n",
      "Val Loss: 0.738 | Acc: 76.010% (7601/10000)\n",
      "\n",
      "Epoch: 6\n",
      "Loss: 0.552 | Acc: 80.922% (40461/50000)\n",
      "Val Loss: 0.566 | Acc: 81.170% (8117/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 7\n",
      "Loss: 0.500 | Acc: 82.712% (41356/50000)\n",
      "Val Loss: 0.633 | Acc: 79.200% (7920/10000)\n",
      "\n",
      "Epoch: 8\n",
      "Loss: 0.455 | Acc: 84.372% (42186/50000)\n",
      "Val Loss: 0.500 | Acc: 83.400% (8340/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 9\n",
      "Loss: 0.425 | Acc: 85.300% (42650/50000)\n",
      "Val Loss: 0.480 | Acc: 83.710% (8371/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 10\n",
      "Loss: 0.392 | Acc: 86.430% (43215/50000)\n",
      "Val Loss: 3.232 | Acc: 73.770% (7377/10000)\n",
      "\n",
      "Epoch: 11\n",
      "Loss: 0.370 | Acc: 87.244% (43622/50000)\n",
      "Val Loss: 0.467 | Acc: 85.280% (8528/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 12\n",
      "Loss: 0.341 | Acc: 88.210% (44105/50000)\n",
      "Val Loss: 0.472 | Acc: 85.270% (8527/10000)\n",
      "\n",
      "Epoch: 13\n",
      "Loss: 0.318 | Acc: 89.026% (44513/50000)\n",
      "Val Loss: 0.424 | Acc: 85.800% (8580/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 14\n",
      "Loss: 0.301 | Acc: 89.468% (44734/50000)\n",
      "Val Loss: 0.446 | Acc: 85.030% (8503/10000)\n",
      "\n",
      "Epoch: 15\n",
      "Loss: 0.289 | Acc: 89.970% (44985/50000)\n",
      "Val Loss: 0.372 | Acc: 87.760% (8776/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 16\n",
      "Loss: 0.273 | Acc: 90.470% (45235/50000)\n",
      "Val Loss: 0.423 | Acc: 86.240% (8624/10000)\n",
      "\n",
      "Epoch: 17\n",
      "Loss: 0.262 | Acc: 90.900% (45450/50000)\n",
      "Val Loss: 0.383 | Acc: 87.840% (8784/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 18\n",
      "Loss: 0.248 | Acc: 91.372% (45686/50000)\n",
      "Val Loss: 0.378 | Acc: 88.140% (8814/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 19\n",
      "Loss: 0.234 | Acc: 91.830% (45915/50000)\n",
      "Val Loss: 0.380 | Acc: 88.040% (8804/10000)\n",
      "\n",
      "Epoch: 20\n",
      "Loss: 0.231 | Acc: 91.898% (45949/50000)\n",
      "Val Loss: 0.352 | Acc: 88.440% (8844/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 21\n",
      "Loss: 0.215 | Acc: 92.464% (46232/50000)\n",
      "Val Loss: 0.409 | Acc: 87.230% (8723/10000)\n",
      "\n",
      "Epoch: 22\n",
      "Loss: 0.212 | Acc: 92.474% (46237/50000)\n",
      "Val Loss: 0.343 | Acc: 88.970% (8897/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 23\n",
      "Loss: 0.205 | Acc: 92.838% (46419/50000)\n",
      "Val Loss: 0.334 | Acc: 89.180% (8918/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 24\n",
      "Loss: 0.197 | Acc: 93.058% (46529/50000)\n",
      "Val Loss: 0.329 | Acc: 89.410% (8941/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 25\n",
      "Loss: 0.191 | Acc: 93.434% (46717/50000)\n",
      "Val Loss: 0.347 | Acc: 89.540% (8954/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 26\n",
      "Loss: 0.178 | Acc: 93.714% (46857/50000)\n",
      "Val Loss: 0.375 | Acc: 88.830% (8883/10000)\n",
      "\n",
      "Epoch: 27\n",
      "Loss: 0.178 | Acc: 93.760% (46880/50000)\n",
      "Val Loss: 0.338 | Acc: 89.020% (8902/10000)\n",
      "\n",
      "Epoch: 28\n",
      "Loss: 0.170 | Acc: 93.934% (46967/50000)\n",
      "Val Loss: 0.373 | Acc: 88.720% (8872/10000)\n",
      "\n",
      "Epoch: 29\n",
      "Loss: 0.161 | Acc: 94.462% (47231/50000)\n",
      "Val Loss: 0.374 | Acc: 88.750% (8875/10000)\n",
      "\n",
      "Epoch: 30\n",
      "Loss: 0.159 | Acc: 94.424% (47212/50000)\n",
      "Val Loss: 0.302 | Acc: 90.620% (9062/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 31\n",
      "Loss: 0.155 | Acc: 94.556% (47278/50000)\n",
      "Val Loss: 0.288 | Acc: 91.220% (9122/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 32\n",
      "Loss: 0.144 | Acc: 94.960% (47480/50000)\n",
      "Val Loss: 0.362 | Acc: 89.420% (8942/10000)\n",
      "\n",
      "Epoch: 35\n",
      "Loss: 0.138 | Acc: 95.138% (47569/50000)\n",
      "Val Loss: 0.354 | Acc: 89.890% (8989/10000)\n",
      "\n",
      "Epoch: 36\n",
      "Loss: 0.136 | Acc: 95.200% (47600/50000)\n",
      "Val Loss: 0.320 | Acc: 90.430% (9043/10000)\n",
      "\n",
      "Epoch: 37\n",
      "Loss: 0.132 | Acc: 95.298% (47649/50000)\n",
      "Val Loss: 0.393 | Acc: 88.940% (8894/10000)\n",
      "\n",
      "Epoch: 38\n",
      "Loss: 0.130 | Acc: 95.456% (47728/50000)\n",
      "Val Loss: 0.276 | Acc: 91.480% (9148/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 39\n",
      "Loss: 0.124 | Acc: 95.660% (47830/50000)\n",
      "Val Loss: 0.352 | Acc: 90.110% (9011/10000)\n",
      "\n",
      "Epoch: 40\n",
      "Loss: 0.125 | Acc: 95.558% (47779/50000)\n",
      "Val Loss: 0.345 | Acc: 90.620% (9062/10000)\n",
      "\n",
      "Epoch: 41\n",
      "Loss: 0.119 | Acc: 95.734% (47867/50000)\n",
      "Val Loss: 0.308 | Acc: 90.970% (9097/10000)\n",
      "\n",
      "Epoch: 42\n",
      "Loss: 0.118 | Acc: 95.876% (47938/50000)\n",
      "Val Loss: 0.320 | Acc: 90.530% (9053/10000)\n",
      "\n",
      "Epoch: 43\n",
      "Loss: 0.116 | Acc: 95.954% (47977/50000)\n",
      "Val Loss: 0.331 | Acc: 90.810% (9081/10000)\n",
      "\n",
      "Epoch: 44\n",
      "Loss: 0.114 | Acc: 95.970% (47985/50000)\n",
      "Val Loss: 0.360 | Acc: 89.720% (8972/10000)\n",
      "\n",
      "Epoch: 45\n",
      "Loss: 0.109 | Acc: 96.188% (48094/50000)\n",
      "Val Loss: 0.325 | Acc: 90.950% (9095/10000)\n",
      "\n",
      "Epoch: 46\n",
      "Loss: 0.104 | Acc: 96.340% (48170/50000)\n",
      "Val Loss: 0.402 | Acc: 89.120% (8912/10000)\n",
      "\n",
      "Epoch: 47\n",
      "Loss: 0.104 | Acc: 96.314% (48157/50000)\n",
      "Val Loss: 0.312 | Acc: 91.200% (9120/10000)\n",
      "\n",
      "Epoch: 48\n",
      "Loss: 0.100 | Acc: 96.420% (48210/50000)\n",
      "Val Loss: 0.364 | Acc: 90.250% (9025/10000)\n",
      "\n",
      "Epoch: 49\n",
      "Loss: 0.101 | Acc: 96.484% (48242/50000)\n",
      "Val Loss: 0.361 | Acc: 90.520% (9052/10000)\n",
      "\n",
      "Epoch: 50\n",
      "Loss: 0.095 | Acc: 96.698% (48349/50000)\n",
      "Val Loss: 0.294 | Acc: 91.860% (9186/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 51\n",
      "Loss: 0.098 | Acc: 96.612% (48306/50000)\n",
      "Val Loss: 0.297 | Acc: 91.720% (9172/10000)\n",
      "\n",
      "Epoch: 52\n",
      "Loss: 0.094 | Acc: 96.672% (48336/50000)\n",
      "Val Loss: 0.307 | Acc: 91.140% (9114/10000)\n",
      "\n",
      "Epoch: 53\n",
      "Loss: 0.094 | Acc: 96.706% (48353/50000)\n",
      "Val Loss: 0.366 | Acc: 90.100% (9010/10000)\n",
      "\n",
      "Epoch: 54\n",
      "Loss: 0.094 | Acc: 96.656% (48328/50000)\n",
      "Val Loss: 0.321 | Acc: 91.170% (9117/10000)\n",
      "\n",
      "Epoch: 55\n",
      "Loss: 0.092 | Acc: 96.712% (48356/50000)\n",
      "Val Loss: 0.348 | Acc: 90.480% (9048/10000)\n",
      "\n",
      "Epoch: 56\n",
      "Loss: 0.085 | Acc: 97.044% (48522/50000)\n",
      "Val Loss: 0.303 | Acc: 91.730% (9173/10000)\n",
      "\n",
      "Epoch: 57\n",
      "Loss: 0.092 | Acc: 96.760% (48380/50000)\n",
      "Val Loss: 0.335 | Acc: 91.390% (9139/10000)\n",
      "\n",
      "Epoch: 58\n",
      "Loss: 0.086 | Acc: 97.026% (48513/50000)\n",
      "Val Loss: 0.289 | Acc: 92.110% (9211/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 59\n",
      "Loss: 0.084 | Acc: 96.984% (48492/50000)\n",
      "Val Loss: 0.312 | Acc: 91.270% (9127/10000)\n",
      "\n",
      "Epoch: 60\n",
      "Loss: 0.081 | Acc: 97.204% (48602/50000)\n",
      "Val Loss: 0.321 | Acc: 91.650% (9165/10000)\n",
      "\n",
      "Epoch: 61\n",
      "Loss: 0.076 | Acc: 97.378% (48689/50000)\n",
      "Val Loss: 0.326 | Acc: 91.600% (9160/10000)\n",
      "\n",
      "Epoch: 62\n",
      "Loss: 0.080 | Acc: 97.176% (48588/50000)\n",
      "Val Loss: 0.291 | Acc: 92.380% (9238/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 63\n",
      "Loss: 0.075 | Acc: 97.484% (48742/50000)\n",
      "Val Loss: 0.315 | Acc: 91.790% (9179/10000)\n",
      "\n",
      "Epoch: 64\n",
      "Loss: 0.073 | Acc: 97.482% (48741/50000)\n",
      "Val Loss: 0.320 | Acc: 91.730% (9173/10000)\n",
      "\n",
      "Epoch: 65\n",
      "Loss: 0.076 | Acc: 97.336% (48668/50000)\n",
      "Val Loss: 0.290 | Acc: 92.130% (9213/10000)\n",
      "\n",
      "Epoch: 66\n",
      "Loss: 0.069 | Acc: 97.598% (48799/50000)\n",
      "Val Loss: 0.289 | Acc: 92.510% (9251/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 67\n",
      "Loss: 0.072 | Acc: 97.494% (48747/50000)\n",
      "Val Loss: 0.273 | Acc: 92.200% (9220/10000)\n",
      "\n",
      "Epoch: 68\n",
      "Loss: 0.067 | Acc: 97.686% (48843/50000)\n",
      "Val Loss: 0.283 | Acc: 92.210% (9221/10000)\n",
      "\n",
      "Epoch: 69\n",
      "Loss: 0.072 | Acc: 97.480% (48740/50000)\n",
      "Val Loss: 0.291 | Acc: 92.120% (9212/10000)\n",
      "\n",
      "Epoch: 70\n",
      "Loss: 0.070 | Acc: 97.582% (48791/50000)\n",
      "Val Loss: 0.292 | Acc: 92.160% (9216/10000)\n",
      "\n",
      "Epoch: 71\n",
      "Loss: 0.066 | Acc: 97.732% (48866/50000)\n",
      "Val Loss: 0.291 | Acc: 92.050% (9205/10000)\n",
      "\n",
      "Epoch: 72\n",
      "Loss: 0.065 | Acc: 97.770% (48885/50000)\n",
      "Val Loss: 0.349 | Acc: 90.770% (9077/10000)\n",
      "\n",
      "Epoch: 73\n",
      "Loss: 0.060 | Acc: 97.872% (48936/50000)\n",
      "Val Loss: 0.340 | Acc: 91.280% (9128/10000)\n",
      "\n",
      "Epoch: 74\n",
      "Loss: 0.061 | Acc: 97.946% (48973/50000)\n",
      "Val Loss: 0.281 | Acc: 92.390% (9239/10000)\n",
      "\n",
      "Epoch: 75\n",
      "Loss: 0.059 | Acc: 97.900% (48950/50000)\n",
      "Val Loss: 0.288 | Acc: 92.250% (9225/10000)\n",
      "\n",
      "Epoch: 76\n",
      "Loss: 0.064 | Acc: 97.712% (48856/50000)\n",
      "Val Loss: 0.280 | Acc: 92.370% (9237/10000)\n",
      "\n",
      "Epoch: 77\n",
      "Loss: 0.060 | Acc: 97.936% (48968/50000)\n",
      "Val Loss: 0.298 | Acc: 91.900% (9190/10000)\n",
      "\n",
      "Epoch: 78\n",
      "Loss: 0.059 | Acc: 97.986% (48993/50000)\n",
      "Val Loss: 0.306 | Acc: 92.170% (9217/10000)\n",
      "\n",
      "Epoch: 79\n",
      "Loss: 0.055 | Acc: 98.126% (49063/50000)\n",
      "Val Loss: 0.279 | Acc: 92.600% (9260/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 80\n",
      "Loss: 0.056 | Acc: 98.078% (49039/50000)\n",
      "Val Loss: 0.296 | Acc: 92.360% (9236/10000)\n",
      "\n",
      "Epoch: 81\n",
      "Loss: 0.056 | Acc: 98.120% (49060/50000)\n",
      "Val Loss: 0.283 | Acc: 92.300% (9230/10000)\n",
      "\n",
      "Epoch: 82\n",
      "Loss: 0.049 | Acc: 98.298% (49149/50000)\n",
      "Val Loss: 0.310 | Acc: 91.950% (9195/10000)\n",
      "\n",
      "Epoch: 83\n",
      "Loss: 0.051 | Acc: 98.242% (49121/50000)\n",
      "Val Loss: 0.306 | Acc: 92.490% (9249/10000)\n",
      "\n",
      "Epoch: 84\n",
      "Loss: 0.052 | Acc: 98.336% (49168/50000)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.289 | Acc: 92.610% (9261/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 85\n",
      "Loss: 0.050 | Acc: 98.292% (49146/50000)\n",
      "Val Loss: 0.270 | Acc: 92.850% (9285/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 86\n",
      "Loss: 0.050 | Acc: 98.258% (49129/50000)\n",
      "Val Loss: 0.281 | Acc: 92.760% (9276/10000)\n",
      "\n",
      "Epoch: 87\n",
      "Loss: 0.049 | Acc: 98.344% (49172/50000)\n",
      "Val Loss: 0.248 | Acc: 93.060% (9306/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 88\n",
      "Loss: 0.045 | Acc: 98.534% (49267/50000)\n",
      "Val Loss: 0.283 | Acc: 92.810% (9281/10000)\n",
      "\n",
      "Epoch: 89\n",
      "Loss: 0.046 | Acc: 98.402% (49201/50000)\n",
      "Val Loss: 0.291 | Acc: 92.240% (9224/10000)\n",
      "\n",
      "Epoch: 90\n",
      "Loss: 0.045 | Acc: 98.444% (49222/50000)\n",
      "Val Loss: 0.292 | Acc: 92.690% (9269/10000)\n",
      "\n",
      "Epoch: 91\n",
      "Loss: 0.043 | Acc: 98.540% (49270/50000)\n",
      "Val Loss: 0.259 | Acc: 93.110% (9311/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 92\n",
      "Loss: 0.044 | Acc: 98.508% (49254/50000)\n",
      "Val Loss: 0.281 | Acc: 92.760% (9276/10000)\n",
      "\n",
      "Epoch: 93\n",
      "Loss: 0.042 | Acc: 98.600% (49300/50000)\n",
      "Val Loss: 0.287 | Acc: 92.900% (9290/10000)\n",
      "\n",
      "Epoch: 94\n",
      "Loss: 0.043 | Acc: 98.530% (49265/50000)\n",
      "Val Loss: 0.272 | Acc: 92.890% (9289/10000)\n",
      "\n",
      "Epoch: 95\n",
      "Loss: 0.040 | Acc: 98.704% (49352/50000)\n",
      "Val Loss: 0.285 | Acc: 92.890% (9289/10000)\n",
      "\n",
      "Epoch: 96\n",
      "Loss: 0.039 | Acc: 98.712% (49356/50000)\n",
      "Val Loss: 0.275 | Acc: 92.980% (9298/10000)\n",
      "\n",
      "Epoch: 97\n",
      "Loss: 0.038 | Acc: 98.656% (49328/50000)\n",
      "Val Loss: 0.259 | Acc: 93.450% (9345/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 98\n",
      "Loss: 0.035 | Acc: 98.808% (49404/50000)\n",
      "Val Loss: 0.272 | Acc: 93.120% (9312/10000)\n",
      "\n",
      "Epoch: 99\n",
      "Loss: 0.043 | Acc: 98.528% (49264/50000)\n",
      "Val Loss: 0.267 | Acc: 93.030% (9303/10000)\n",
      "\n",
      "Epoch: 100\n",
      "Loss: 0.034 | Acc: 98.828% (49414/50000)\n",
      "Val Loss: 0.274 | Acc: 93.170% (9317/10000)\n",
      "\n",
      "Epoch: 101\n",
      "Loss: 0.036 | Acc: 98.846% (49423/50000)\n",
      "Val Loss: 0.293 | Acc: 92.670% (9267/10000)\n",
      "\n",
      "Epoch: 102\n",
      "Loss: 0.034 | Acc: 98.850% (49425/50000)\n",
      "Val Loss: 0.276 | Acc: 93.070% (9307/10000)\n",
      "\n",
      "Epoch: 103\n",
      "Loss: 0.031 | Acc: 98.970% (49485/50000)\n",
      "Val Loss: 0.282 | Acc: 93.110% (9311/10000)\n",
      "\n",
      "Epoch: 104\n",
      "Loss: 0.031 | Acc: 98.974% (49487/50000)\n",
      "Val Loss: 0.268 | Acc: 93.430% (9343/10000)\n",
      "\n",
      "Epoch: 105\n",
      "Loss: 0.032 | Acc: 98.942% (49471/50000)\n",
      "Val Loss: 0.269 | Acc: 93.130% (9313/10000)\n",
      "\n",
      "Epoch: 106\n",
      "Loss: 0.032 | Acc: 98.864% (49432/50000)\n",
      "Val Loss: 0.288 | Acc: 92.780% (9278/10000)\n",
      "\n",
      "Epoch: 107\n",
      "Loss: 0.031 | Acc: 98.968% (49484/50000)\n",
      "Val Loss: 0.248 | Acc: 93.850% (9385/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 108\n",
      "Loss: 0.031 | Acc: 98.962% (49481/50000)\n",
      "Val Loss: 0.256 | Acc: 93.730% (9373/10000)\n",
      "\n",
      "Epoch: 109\n",
      "Loss: 0.026 | Acc: 99.164% (49582/50000)\n",
      "Val Loss: 0.245 | Acc: 93.740% (9374/10000)\n",
      "\n",
      "Epoch: 110\n",
      "Loss: 0.027 | Acc: 99.138% (49569/50000)\n",
      "Val Loss: 0.260 | Acc: 93.570% (9357/10000)\n",
      "\n",
      "Epoch: 111\n",
      "Loss: 0.026 | Acc: 99.148% (49574/50000)\n",
      "Val Loss: 0.256 | Acc: 93.520% (9352/10000)\n",
      "\n",
      "Epoch: 112\n",
      "Loss: 0.027 | Acc: 99.120% (49560/50000)\n",
      "Val Loss: 0.267 | Acc: 93.210% (9321/10000)\n",
      "\n",
      "Epoch: 113\n",
      "Loss: 0.026 | Acc: 99.110% (49555/50000)\n",
      "Val Loss: 0.271 | Acc: 93.190% (9319/10000)\n",
      "\n",
      "Epoch: 114\n",
      "Loss: 0.024 | Acc: 99.236% (49618/50000)\n",
      "Val Loss: 0.275 | Acc: 93.550% (9355/10000)\n",
      "\n",
      "Epoch: 115\n",
      "Val Loss: 0.292 | Acc: 93.080% (9308/10000)\n",
      "\n",
      "Epoch: 117\n",
      "Loss: 0.022 | Acc: 99.314% (49657/50000)\n",
      "Val Loss: 0.273 | Acc: 93.590% (9359/10000)\n",
      "\n",
      "Epoch: 118\n",
      "Loss: 0.022 | Acc: 99.282% (49641/50000)\n",
      "Val Loss: 0.271 | Acc: 93.450% (9345/10000)\n",
      "\n",
      "Epoch: 119\n",
      "Loss: 0.025 | Acc: 99.150% (49575/50000)\n",
      "Val Loss: 0.252 | Acc: 93.650% (9365/10000)\n",
      "\n",
      "Epoch: 120\n",
      "Loss: 0.022 | Acc: 99.294% (49647/50000)\n",
      "Val Loss: 0.249 | Acc: 93.730% (9373/10000)\n",
      "\n",
      "Epoch: 121\n",
      "Loss: 0.020 | Acc: 99.364% (49682/50000)\n",
      "Val Loss: 0.236 | Acc: 94.090% (9409/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 122\n",
      "Loss: 0.019 | Acc: 99.394% (49697/50000)\n",
      "Val Loss: 0.257 | Acc: 93.660% (9366/10000)\n",
      "\n",
      "Epoch: 123\n",
      "Loss: 0.019 | Acc: 99.412% (49706/50000)\n",
      "Val Loss: 0.259 | Acc: 93.720% (9372/10000)\n",
      "\n",
      "Epoch: 124\n",
      "Loss: 0.020 | Acc: 99.316% (49658/50000)\n",
      "Val Loss: 0.235 | Acc: 94.060% (9406/10000)\n",
      "\n",
      "Epoch: 125\n",
      "Loss: 0.018 | Acc: 99.442% (49721/50000)\n",
      "Val Loss: 0.239 | Acc: 94.050% (9405/10000)\n",
      "\n",
      "Epoch: 126\n",
      "Loss: 0.017 | Acc: 99.482% (49741/50000)\n",
      "Val Loss: 0.253 | Acc: 93.830% (9383/10000)\n",
      "\n",
      "Epoch: 127\n",
      "Loss: 0.017 | Acc: 99.472% (49736/50000)\n",
      "Val Loss: 0.247 | Acc: 94.020% (9402/10000)\n",
      "\n",
      "Epoch: 128\n",
      "Loss: 0.013 | Acc: 99.582% (49791/50000)\n",
      "Val Loss: 0.251 | Acc: 93.930% (9393/10000)\n",
      "\n",
      "Epoch: 129\n",
      "Loss: 0.014 | Acc: 99.578% (49789/50000)\n",
      "Val Loss: 0.236 | Acc: 94.290% (9429/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 130\n",
      "Loss: 0.015 | Acc: 99.540% (49770/50000)\n",
      "Val Loss: 0.241 | Acc: 94.140% (9414/10000)\n",
      "\n",
      "Epoch: 131\n",
      "Loss: 0.013 | Acc: 99.596% (49798/50000)\n",
      "Val Loss: 0.246 | Acc: 94.110% (9411/10000)\n",
      "\n",
      "Epoch: 132\n",
      "Loss: 0.013 | Acc: 99.554% (49777/50000)\n",
      "Val Loss: 0.242 | Acc: 94.240% (9424/10000)\n",
      "\n",
      "Epoch: 133\n",
      "Loss: 0.012 | Acc: 99.624% (49812/50000)\n",
      "Val Loss: 0.261 | Acc: 93.680% (9368/10000)\n",
      "\n",
      "Epoch: 134\n",
      "Loss: 0.012 | Acc: 99.612% (49806/50000)\n",
      "Val Loss: 0.232 | Acc: 94.320% (9432/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 135\n",
      "Loss: 0.013 | Acc: 99.580% (49790/50000)\n",
      "Val Loss: 0.233 | Acc: 94.220% (9422/10000)\n",
      "\n",
      "Epoch: 136\n",
      "Loss: 0.011 | Acc: 99.648% (49824/50000)\n",
      "Val Loss: 0.229 | Acc: 94.350% (9435/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 137\n",
      "Loss: 0.010 | Acc: 99.690% (49845/50000)\n",
      "Val Loss: 0.230 | Acc: 94.500% (9450/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 138\n",
      "Loss: 0.011 | Acc: 99.648% (49824/50000)\n",
      "Val Loss: 0.252 | Acc: 94.120% (9412/10000)\n",
      "\n",
      "Epoch: 139\n",
      "Loss: 0.009 | Acc: 99.702% (49851/50000)\n",
      "Val Loss: 0.238 | Acc: 94.350% (9435/10000)\n",
      "\n",
      "Epoch: 140\n",
      "Loss: 0.009 | Acc: 99.708% (49854/50000)\n",
      "Val Loss: 0.234 | Acc: 94.560% (9456/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 141\n",
      "Loss: 0.009 | Acc: 99.726% (49863/50000)\n",
      "Val Loss: 0.220 | Acc: 94.880% (9488/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 142\n",
      "Loss: 0.009 | Acc: 99.726% (49863/50000)\n",
      "Val Loss: 0.239 | Acc: 94.530% (9453/10000)\n",
      "\n",
      "Epoch: 143\n",
      "Loss: 0.009 | Acc: 99.742% (49871/50000)\n",
      "Val Loss: 0.228 | Acc: 94.550% (9455/10000)\n",
      "\n",
      "Epoch: 144\n",
      "Loss: 0.009 | Acc: 99.728% (49864/50000)\n",
      "Val Loss: 0.222 | Acc: 94.680% (9468/10000)\n",
      "\n",
      "Epoch: 145\n",
      "Loss: 0.008 | Acc: 99.746% (49873/50000)\n",
      "Val Loss: 0.231 | Acc: 94.580% (9458/10000)\n",
      "\n",
      "Epoch: 146\n",
      "Val Loss: 0.227 | Acc: 94.770% (9477/10000)\n",
      "\n",
      "Epoch: 148\n",
      "Loss: 0.007 | Acc: 99.764% (49882/50000)\n",
      "Val Loss: 0.243 | Acc: 94.460% (9446/10000)\n",
      "\n",
      "Epoch: 149\n",
      "Loss: 0.007 | Acc: 99.778% (49889/50000)\n",
      "Val Loss: 0.228 | Acc: 94.650% (9465/10000)\n",
      "\n",
      "Epoch: 150\n",
      "Loss: 0.006 | Acc: 99.812% (49906/50000)\n",
      "Val Loss: 0.229 | Acc: 94.580% (9458/10000)\n",
      "\n",
      "Epoch: 151\n",
      "Loss: 0.008 | Acc: 99.790% (49895/50000)\n",
      "Val Loss: 0.223 | Acc: 94.530% (9453/10000)\n",
      "\n",
      "Epoch: 152\n",
      "Loss: 0.006 | Acc: 99.822% (49911/50000)\n",
      "Val Loss: 0.221 | Acc: 94.750% (9475/10000)\n",
      "\n",
      "Epoch: 153\n",
      "Loss: 0.006 | Acc: 99.824% (49912/50000)\n",
      "Val Loss: 0.217 | Acc: 94.730% (9473/10000)\n",
      "\n",
      "Epoch: 154\n",
      "Loss: 0.005 | Acc: 99.854% (49927/50000)\n",
      "Val Loss: 0.216 | Acc: 94.900% (9490/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 155\n",
      "Loss: 0.005 | Acc: 99.874% (49937/50000)\n",
      "Val Loss: 0.220 | Acc: 94.760% (9476/10000)\n",
      "\n",
      "Epoch: 156\n",
      "Loss: 0.005 | Acc: 99.820% (49910/50000)\n",
      "Val Loss: 0.226 | Acc: 94.630% (9463/10000)\n",
      "\n",
      "Epoch: 157\n",
      "Loss: 0.006 | Acc: 99.818% (49909/50000)\n",
      "Val Loss: 0.228 | Acc: 94.640% (9464/10000)\n",
      "\n",
      "Epoch: 158\n",
      "Loss: 0.005 | Acc: 99.868% (49934/50000)\n",
      "Val Loss: 0.218 | Acc: 94.820% (9482/10000)\n",
      "\n",
      "Epoch: 159\n",
      "Loss: 0.004 | Acc: 99.864% (49932/50000)\n",
      "Val Loss: 0.218 | Acc: 95.050% (9505/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 160\n",
      "Loss: 0.004 | Acc: 99.898% (49949/50000)\n",
      "Val Loss: 0.216 | Acc: 95.000% (9500/10000)\n",
      "\n",
      "Epoch: 161\n",
      "Loss: 0.004 | Acc: 99.894% (49947/50000)\n",
      "Val Loss: 0.214 | Acc: 94.910% (9491/10000)\n",
      "\n",
      "Epoch: 162\n",
      "Loss: 0.004 | Acc: 99.888% (49944/50000)\n",
      "Val Loss: 0.222 | Acc: 94.840% (9484/10000)\n",
      "\n",
      "Epoch: 163\n",
      "Loss: 0.004 | Acc: 99.896% (49948/50000)\n",
      "Val Loss: 0.220 | Acc: 94.880% (9488/10000)\n",
      "\n",
      "Epoch: 164\n",
      "Loss: 0.004 | Acc: 99.880% (49940/50000)\n",
      "Val Loss: 0.220 | Acc: 94.830% (9483/10000)\n",
      "\n",
      "Epoch: 165\n",
      "Loss: 0.003 | Acc: 99.896% (49948/50000)\n",
      "Val Loss: 0.220 | Acc: 94.910% (9491/10000)\n",
      "\n",
      "Epoch: 166\n",
      "Loss: 0.003 | Acc: 99.920% (49960/50000)\n",
      "Val Loss: 0.215 | Acc: 94.950% (9495/10000)\n",
      "\n",
      "Epoch: 167\n",
      "Loss: 0.003 | Acc: 99.906% (49953/50000)\n",
      "Val Loss: 0.215 | Acc: 95.030% (9503/10000)\n",
      "\n",
      "Epoch: 168\n",
      "Loss: 0.004 | Acc: 99.908% (49954/50000)\n",
      "Val Loss: 0.215 | Acc: 94.970% (9497/10000)\n",
      "\n",
      "Epoch: 169\n",
      "Loss: 0.004 | Acc: 99.896% (49948/50000)\n",
      "Val Loss: 0.213 | Acc: 95.110% (9511/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 170\n",
      "Loss: 0.003 | Acc: 99.924% (49962/50000)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.214 | Acc: 94.960% (9496/10000)\n",
      "\n",
      "Epoch: 171\n",
      "Loss: 0.003 | Acc: 99.916% (49958/50000)\n",
      "Val Loss: 0.213 | Acc: 95.100% (9510/10000)\n",
      "\n",
      "Epoch: 172\n",
      "Loss: 0.002 | Acc: 99.934% (49967/50000)\n",
      "Val Loss: 0.212 | Acc: 95.060% (9506/10000)\n",
      "\n",
      "Epoch: 173\n",
      "Loss: 0.003 | Acc: 99.926% (49963/50000)\n",
      "Val Loss: 0.211 | Acc: 95.100% (9510/10000)\n",
      "\n",
      "Epoch: 174\n",
      "Loss: 0.004 | Acc: 99.906% (49953/50000)\n",
      "Val Loss: 0.212 | Acc: 94.980% (9498/10000)\n",
      "\n",
      "Epoch: 175\n",
      "Loss: 0.003 | Acc: 99.926% (49963/50000)\n",
      "Val Loss: 0.208 | Acc: 95.020% (9502/10000)\n",
      "\n",
      "Epoch: 176\n",
      "Loss: 0.004 | Acc: 99.908% (49954/50000)\n",
      "Val Loss: 0.208 | Acc: 95.050% (9505/10000)\n",
      "\n",
      "Epoch: 177\n",
      "Loss: 0.003 | Acc: 99.930% (49965/50000)\n",
      "Val Loss: 0.207 | Acc: 95.010% (9501/10000)\n",
      "\n",
      "Epoch: 178\n",
      "Loss: 0.003 | Acc: 99.930% (49965/50000)\n",
      "Val Loss: 0.205 | Acc: 95.140% (9514/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 179\n",
      "Loss: 0.003 | Acc: 99.922% (49961/50000)\n",
      "Val Loss: 0.205 | Acc: 95.130% (9513/10000)\n",
      "\n",
      "Epoch: 180\n",
      "Loss: 0.002 | Acc: 99.944% (49972/50000)\n",
      "Val Loss: 0.205 | Acc: 95.210% (9521/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 181\n",
      "Loss: 0.002 | Acc: 99.950% (49975/50000)\n",
      "Val Loss: 0.207 | Acc: 95.190% (9519/10000)\n",
      "\n",
      "Epoch: 182\n",
      "Loss: 0.003 | Acc: 99.930% (49965/50000)\n",
      "Val Loss: 0.204 | Acc: 95.210% (9521/10000)\n",
      "\n",
      "Epoch: 183\n",
      "Loss: 0.003 | Acc: 99.924% (49962/50000)\n",
      "Val Loss: 0.203 | Acc: 95.100% (9510/10000)\n",
      "\n",
      "Epoch: 184\n",
      "Loss: 0.003 | Acc: 99.942% (49971/50000)\n",
      "Val Loss: 0.206 | Acc: 95.280% (9528/10000)\n",
      "Saving..\n",
      "\n",
      "Epoch: 185\n",
      "Loss: 0.003 | Acc: 99.924% (49962/50000)\n",
      "Val Loss: 0.205 | Acc: 95.270% (9527/10000)\n",
      "\n",
      "Epoch: 186\n",
      "Loss: 0.002 | Acc: 99.940% (49970/50000)\n",
      "Val Loss: 0.202 | Acc: 95.190% (9519/10000)\n",
      "\n",
      "Epoch: 187\n",
      "Loss: 0.002 | Acc: 99.954% (49977/50000)\n",
      "Val Loss: 0.204 | Acc: 95.190% (9519/10000)\n",
      "\n",
      "Epoch: 188\n",
      "Loss: 0.003 | Acc: 99.922% (49961/50000)\n",
      "Val Loss: 0.202 | Acc: 95.160% (9516/10000)\n",
      "\n",
      "Epoch: 189\n",
      "Loss: 0.002 | Acc: 99.938% (49969/50000)\n",
      "Val Loss: 0.205 | Acc: 95.220% (9522/10000)\n",
      "\n",
      "Epoch: 190\n",
      "Loss: 0.002 | Acc: 99.954% (49977/50000)\n",
      "Val Loss: 0.203 | Acc: 95.150% (9515/10000)\n",
      "\n",
      "Epoch: 191\n",
      "Loss: 0.002 | Acc: 99.948% (49974/50000)\n",
      "Val Loss: 0.201 | Acc: 95.190% (9519/10000)\n",
      "\n",
      "Epoch: 192\n",
      "Loss: 0.002 | Acc: 99.930% (49965/50000)\n",
      "Val Loss: 0.201 | Acc: 95.180% (9518/10000)\n",
      "\n",
      "Epoch: 193\n",
      "Loss: 0.002 | Acc: 99.944% (49972/50000)\n",
      "Val Loss: 0.201 | Acc: 95.180% (9518/10000)\n",
      "\n",
      "Epoch: 194\n",
      "Loss: 0.003 | Acc: 99.938% (49969/50000)\n",
      "Val Loss: 0.203 | Acc: 95.230% (9523/10000)\n",
      "\n",
      "Epoch: 195\n",
      "Loss: 0.003 | Acc: 99.916% (49958/50000)\n",
      "Val Loss: 0.202 | Acc: 95.200% (9520/10000)\n",
      "\n",
      "Epoch: 196\n",
      "Loss: 0.003 | Acc: 99.914% (49957/50000)\n",
      "Val Loss: 0.204 | Acc: 95.140% (9514/10000)\n",
      "\n",
      "Epoch: 197\n",
      "Loss: 0.002 | Acc: 99.950% (49975/50000)\n",
      "Val Loss: 0.202 | Acc: 95.120% (9512/10000)\n",
      "\n",
      "Epoch: 198\n",
      "Loss: 0.003 | Acc: 99.906% (49953/50000)\n",
      "Val Loss: 0.202 | Acc: 95.170% (9517/10000)\n",
      "\n",
      "Epoch: 199\n",
      "Loss: 0.002 | Acc: 99.952% (49976/50000)\n",
      "Val Loss: 0.205 | Acc: 95.200% (9520/10000)\n",
      "Best val Acc: 95.280000\n"
     ]
    }
   ],
   "source": [
    "best_acc = 0  # best test accuracy\n",
    "start_epoch = 0  # start from epoch 0 or last checkpoint epoch\n",
    "transform_train = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "    transforms.RandomErasing(p=0.5, scale=(0.02, 0.1)),\n",
    "])\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=True, download=True, transform=transform_train)\n",
    "trainloader = torch.utils.data.DataLoader(\n",
    "    trainset, batch_size=128, shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=False, download=True, transform=transform_test)\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset, batch_size=128, shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer',\n",
    "           'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "net = ResNet50()\n",
    "net = net.cuda()\n",
    "\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01,\n",
    "                      momentum=0.9, weight_decay=5e-4)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)\n",
    "\n",
    "# Training\n",
    "def train(epoch):\n",
    "    print('\\nEpoch: %d' % epoch)\n",
    "    net.train()\n",
    "    train_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(trainloader):\n",
    "        inputs, targets = inputs.cuda(), targets.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        \n",
    "        use_mixup = False\n",
    "        if random.random() < 0.85:\n",
    "            use_mixup = True\n",
    "\n",
    "        if use_mixup:\n",
    "            data, y_a, y_b, lam = mixup_data(inputs, targets)\n",
    "            outputs = net(inputs)\n",
    "        else:\n",
    "            outputs = net(inputs)\n",
    "\n",
    "        _, predicted = outputs.max(1)\n",
    "\n",
    "        if use_mixup:\n",
    "            loss = mixup_criterion(criterion, outputs, y_a, y_b, lam)\n",
    "        else:\n",
    "            loss = criterion(outputs, targets)\n",
    "        \n",
    "        \n",
    "\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "\n",
    "        total += targets.size(0)\n",
    "        correct += predicted.eq(targets).sum().item()\n",
    "\n",
    "    print( 'Loss: %.3f | Acc: %.3f%% (%d/%d)'\n",
    "                     % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    global best_acc\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "            inputs, targets = inputs.cuda(), targets.cuda()\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "\n",
    "        print( 'Val Loss: %.3f | Acc: %.3f%% (%d/%d)'\n",
    "                         % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))\n",
    "\n",
    "    # Save checkpoint.\n",
    "    acc = 100.*correct/total\n",
    "    if acc > best_acc:\n",
    "        print('Saving..')\n",
    "        state = {\n",
    "            'net': net.state_dict(),\n",
    "            'acc': acc,\n",
    "            'epoch': epoch,\n",
    "        }\n",
    "        if not os.path.isdir('checkpoint'):\n",
    "            os.mkdir('checkpoint')\n",
    "        torch.save(state, './checkpoint/ckpt.pth')\n",
    "        best_acc = acc\n",
    "\n",
    "\n",
    "for epoch in range(200):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "    scheduler.step()\n",
    "    \n",
    "print('Best val Acc: {:4f}'.format(best_acc))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py37",
   "language": "python",
   "name": "py37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}