examples/12_Optimizing_AUROC_Loss_on_Tabular_Data.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2QKJzNEd6bbn"
      },
      "source": [
        "# **Tutorial for Optimizing AUROC Loss on Tabular Data**\n",
        "\n",
        "**Author**: Zhuoning Yuan \\\\\n",
        "\n",
        "**Introduction**\n",
        "\n",
        "In this tutorial, you will learn how to quickly train a MLP model by optimizing **AUROC** score using our novel optimization methods on **Tabular Data** ([Credit Fraud](https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud)). For image classification tutorial, please refer to [AUCM](https://github.com/Optimization-AI/LibAUC/blob/main/examples/02_Optimizing_AUROC_with_ResNet20_on_Imbalanced_CIFAR10.ipynb). After completion of this tutorial, you should be able to use LibAUC to train your own models on your own datasets.\n",
        "\n",
        "**Useful Resources**\n",
        "\n",
        "* Website: https://libauc.org\n",
        "* Github: https://github.com/Optimization-AI/LibAUC\n",
        "\n",
        "**References**\n",
        "\n",
        "If you find this tutorial helpful,  please acknowledge our library and cite the following papers:\n",
        "<pre>\n",
        "@inproceedings{yuan2021large,\n",
        "  title={Large-scale robust deep auc maximization: A new surrogate loss and empirical studies on medical image classification},\n",
        "  author={Yuan, Zhuoning and Yan, Yan and Sonka, Milan and Yang, Tianbao},\n",
        "  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},\n",
        "  pages={3040--3049},\n",
        "  year={2021}\n",
        "}\n",
        "</pre>\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SJlahR4u9uSA"
      },
      "outputs": [],
      "source": [
        "!pip install libauc==1.2.0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h44OjwNncoUp"
      },
      "source": [
        "# **Import LibAUC**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "jVXbEQ5E9xyj"
      },
      "outputs": [],
      "source": [
        "from libauc.losses import AUCMLoss\n",
        "from libauc.optimizers import PESG\n",
        "from libauc.sampler import DualSampler\n",
        "from libauc.metrics import auroc\n",
        "from libauc.models import MLP\n",
        "\n",
        "import torch \n",
        "import numpy as np\n",
        "from torch.utils.data import Dataset\n",
        "from sklearn.metrics import roc_auc_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e4CMX6LPcuSc"
      },
      "source": [
        "# **Reproducibility**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "JTnrfY6Gcsoo"
      },
      "outputs": [],
      "source": [
        "def set_all_seeds(SEED):\n",
        "    # REPRODUCIBILITY\n",
        "    torch.manual_seed(SEED)\n",
        "    np.random.seed(SEED)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C-ivD4T7cxCc"
      },
      "source": [
        "# **Loading and Preprocessing Credit Fraud Dataset**\n",
        "Reference: https://www.tensorflow.org/tutorials/structured_data/imbalanced_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cDY6x54agBCW",
        "outputId": "7f9c4aeb-477b-4d94-d2ca-02fc607a4096"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Examples:\n",
            "    Total: 284807\n",
            "    Positive: 492 (0.17% of total)\n",
            "\n",
            "Training labels shape: (182276,)\n",
            "Validation labels shape: (45569,)\n",
            "Test labels shape: (56962,)\n",
            "Training features shape: (182276, 29)\n",
            "Validation features shape: (45569, 29)\n",
            "Test features shape: (56962, 29)\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import sklearn\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "\n",
        "raw_df = pd.read_csv('https://storage.googleapis.com/download.tensorflow.org/data/creditcard.csv')\n",
        "neg, pos = np.bincount(raw_df['Class'])\n",
        "total = neg + pos\n",
        "print('Examples:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(total, pos, 100 * pos / total))\n",
        "\n",
        "cleaned_df = raw_df.copy()\n",
        "\n",
        "# You don't want the `Time` column.\n",
        "cleaned_df.pop('Time')\n",
        "\n",
        "# The `Amount` column covers a huge range. Convert to log-space.\n",
        "eps = 0.001 # 0 => 0.1¢\n",
        "cleaned_df['Log Ammount'] = np.log(cleaned_df.pop('Amount')+eps)\n",
        "\n",
        "# Use a utility from sklearn to split and shuffle your dataset.\n",
        "train_df, test_df = train_test_split(cleaned_df, test_size=0.2, random_state=42)\n",
        "train_df, val_df = train_test_split(train_df, test_size=0.2, random_state=42)\n",
        "\n",
        "# Form np arrays of labels and features.\n",
        "train_labels = np.array(train_df.pop('Class'))\n",
        "bool_train_labels = train_labels != 0\n",
        "val_labels = np.array(val_df.pop('Class'))\n",
        "test_labels = np.array(test_df.pop('Class'))\n",
        "\n",
        "train_features = np.array(train_df)\n",
        "val_features = np.array(val_df)\n",
        "test_features = np.array(test_df)\n",
        "\n",
        "# Normalize the input features using the sklearn StandardScaler. This will set the mean to 0 and standard deviation to 1.\n",
        "# Note: The StandardScaler is only fit using the train_features to be sure the model is not peeking at the validation or test sets.\n",
        "scaler = StandardScaler()\n",
        "train_features = scaler.fit_transform(train_features)\n",
        "\n",
        "val_features = scaler.transform(val_features)\n",
        "test_features = scaler.transform(test_features)\n",
        "\n",
        "train_features = np.clip(train_features, -5, 5)\n",
        "val_features = np.clip(val_features, -5, 5)\n",
        "test_features = np.clip(test_features, -5, 5)\n",
        "\n",
        "\n",
        "print('Training labels shape:', train_labels.shape)\n",
        "print('Validation labels shape:', val_labels.shape)\n",
        "print('Test labels shape:', test_labels.shape)\n",
        "\n",
        "print('Training features shape:', train_features.shape)\n",
        "print('Validation features shape:', val_features.shape)\n",
        "print('Test features shape:', test_features.shape)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1l1cu_g4hxj4"
      },
      "source": [
        "# **Paramaters**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "BSGmLmVxjZCX"
      },
      "outputs": [],
      "source": [
        "SEED = 123\n",
        "\n",
        "# tunable parameters\n",
        "BATCH_SIZE = 2048\n",
        "lr = 0.1\n",
        "epoch_decay = 0.002\n",
        "weight_decay = 0\n",
        "margin = 1.0\n",
        "\n",
        "# sampling parameters\n",
        "sampling_rate = 0.1 # e.g., this ensures 0.1*1024 = 102 positive samples in each mini-batch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lGX8q9WIlgwI"
      },
      "source": [
        "# **Loading Dataset**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "lA6Klcb6k3xA"
      },
      "outputs": [],
      "source": [
        "class CreditFraudDataset(Dataset):\n",
        "    def __init__(self, data, target, shuffle=False):\n",
        "        list_id = np.arange(len(data))\n",
        "        if shuffle:\n",
        "           np.random.seed(123)\n",
        "           np.random.shuffle(list_id)\n",
        "        self.data = data.astype(np.float32)[list_id] # numpy array\n",
        "        self.targets = target.astype(np.float32)[list_id] # numpy array\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        data = self.data[index]\n",
        "        target = self.targets[index]\n",
        "        return data, target\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.data.shape[0]\n",
        "\n",
        "trainDataset = CreditFraudDataset(train_features, train_labels, shuffle=True)\n",
        "valDataset = CreditFraudDataset(val_features, val_labels)\n",
        "testDataset = CreditFraudDataset(test_features, test_labels)\n",
        "\n",
        "sampler = DualSampler(trainDataset, BATCH_SIZE, sampling_rate=sampling_rate)\n",
        "trainloader = torch.utils.data.DataLoader(trainDataset, batch_size=BATCH_SIZE, sampler=sampler, shuffle=False, num_workers=1, pin_memory=True)\n",
        "valloader = torch.utils.data.DataLoader(valDataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=1,  pin_memory=False)\n",
        "testloader = torch.utils.data.DataLoader(testDataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=1,  pin_memory=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VM8M8aGUj7dw"
      },
      "source": [
        "# **Creating models & AUC Optimizer**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "loivAnZHEPmv"
      },
      "outputs": [],
      "source": [
        "import torch \n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "# Multilayer Perceptron\n",
        "class MLP(torch.nn.Module):\n",
        "    def __init__(self, input_dim=29, hidden_sizes=16, num_classes=1):\n",
        "        super().__init__()\n",
        "        self.hidden_sizes = hidden_sizes\n",
        "        self.layers = torch.nn.Linear(input_dim, hidden_sizes)\n",
        "        self.dropout = nn.Dropout(p=0.5)\n",
        "        self.classifer = torch.nn.Linear(hidden_sizes, num_classes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"forward pass\"\"\"\n",
        "        x = self.layers(x)\n",
        "        x = F.relu(x)\n",
        "        x = self.dropout(x)\n",
        "        return self.classifer(x) \n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mBwayo8Yj7E-",
        "outputId": "15556671-3946-4a29-be1c-09874756520a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MLP(\n",
            "  (layers): Linear(in_features=29, out_features=16, bias=True)\n",
            "  (dropout): Dropout(p=0.5, inplace=False)\n",
            "  (classifer): Linear(in_features=16, out_features=1, bias=True)\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "set_all_seeds(SEED)\n",
        "model = MLP(input_dim=29, hidden_sizes=16, num_classes=1) \n",
        "model = model.cuda()\n",
        "print (model)\n",
        "\n",
        "loss_fn = AUCMLoss()\n",
        "optimizer = PESG(model, \n",
        "                 loss_fn=loss_fn,\n",
        "                 lr=lr, \n",
        "                 margin=margin, \n",
        "                 epoch_decay=epoch_decay, \n",
        "                 weight_decay=weight_decay)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75K73TmSk_xQ"
      },
      "source": [
        "# **Training**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "XBeqpIKet0EY"
      },
      "outputs": [],
      "source": [
        "def evaluate(data, model):\n",
        "     model.eval()\n",
        "     pred = []\n",
        "     true = [] \n",
        "     for j, data in enumerate(data):\n",
        "         data, targets = data\n",
        "         data = data.cuda()\n",
        "         y_pred = model(data)\n",
        "         pred.append(y_pred.cpu().detach().numpy())\n",
        "         true.append(targets.numpy())\n",
        "     true = np.concatenate(true)\n",
        "     pred = np.concatenate(pred)\n",
        "     val_auc =  roc_auc_score(true, pred) \n",
        "     return val_auc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PqNSMQZ-lAud",
        "outputId": "a4b7c769-b2b9-4607-c76b-34f1d9a26c08"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start Training\n",
            "------------------------------\n",
            "epoch: 0, train_loss: 0.027446, train_auc:0.871964, val_auc:0.946429, test_auc:0.940484,  lr:0.100000\n",
            "epoch: 1, train_loss: 0.027886, train_auc:0.943964, val_auc:0.957875, test_auc:0.948275,  lr:0.100000\n",
            "epoch: 2, train_loss: 0.022244, train_auc:0.954968, val_auc:0.964605, test_auc:0.955106,  lr:0.100000\n",
            "epoch: 3, train_loss: 0.023176, train_auc:0.960363, val_auc:0.968141, test_auc:0.961279,  lr:0.100000\n",
            "epoch: 4, train_loss: 0.021818, train_auc:0.963592, val_auc:0.970230, test_auc:0.965589,  lr:0.100000\n",
            "epoch: 5, train_loss: 0.022460, train_auc:0.967376, val_auc:0.971485, test_auc:0.969102,  lr:0.100000\n",
            "epoch: 6, train_loss: 0.020066, train_auc:0.966668, val_auc:0.972433, test_auc:0.972133,  lr:0.100000\n",
            "epoch: 7, train_loss: 0.019405, train_auc:0.968375, val_auc:0.973234, test_auc:0.974988,  lr:0.100000\n",
            "epoch: 8, train_loss: 0.018477, train_auc:0.970607, val_auc:0.973970, test_auc:0.977620,  lr:0.100000\n",
            "epoch: 9, train_loss: 0.019867, train_auc:0.971803, val_auc:0.974663, test_auc:0.979312,  lr:0.100000\n",
            "epoch: 10, train_loss: 0.019537, train_auc:0.974263, val_auc:0.975394, test_auc:0.980748,  lr:0.100000\n",
            "epoch: 11, train_loss: 0.017595, train_auc:0.974706, val_auc:0.975647, test_auc:0.982226,  lr:0.100000\n",
            "epoch: 12, train_loss: 0.019956, train_auc:0.974567, val_auc:0.975969, test_auc:0.982229,  lr:0.100000\n",
            "epoch: 13, train_loss: 0.018181, train_auc:0.976363, val_auc:0.975823, test_auc:0.982562,  lr:0.100000\n",
            "epoch: 14, train_loss: 0.016673, train_auc:0.978031, val_auc:0.975909, test_auc:0.982925,  lr:0.100000\n",
            "epoch: 15, train_loss: 0.017862, train_auc:0.976566, val_auc:0.976207, test_auc:0.983994,  lr:0.100000\n",
            "epoch: 16, train_loss: 0.016973, train_auc:0.977384, val_auc:0.976510, test_auc:0.984945,  lr:0.100000\n",
            "epoch: 17, train_loss: 0.017325, train_auc:0.976605, val_auc:0.976817, test_auc:0.985877,  lr:0.100000\n",
            "epoch: 18, train_loss: 0.017442, train_auc:0.976408, val_auc:0.976592, test_auc:0.986332,  lr:0.100000\n",
            "epoch: 19, train_loss: 0.017819, train_auc:0.976545, val_auc:0.976978, test_auc:0.986922,  lr:0.100000\n",
            "epoch: 20, train_loss: 0.016431, train_auc:0.979157, val_auc:0.976898, test_auc:0.986981,  lr:0.100000\n",
            "epoch: 21, train_loss: 0.016261, train_auc:0.978934, val_auc:0.977228, test_auc:0.987642,  lr:0.100000\n",
            "epoch: 22, train_loss: 0.015126, train_auc:0.978828, val_auc:0.977519, test_auc:0.987981,  lr:0.100000\n",
            "epoch: 23, train_loss: 0.016764, train_auc:0.979871, val_auc:0.977901, test_auc:0.988458,  lr:0.100000\n",
            "epoch: 24, train_loss: 0.015818, train_auc:0.979637, val_auc:0.978189, test_auc:0.988448,  lr:0.100000\n",
            "epoch: 25, train_loss: 0.016359, train_auc:0.977902, val_auc:0.978768, test_auc:0.988815,  lr:0.100000\n",
            "epoch: 26, train_loss: 0.015286, train_auc:0.978669, val_auc:0.978790, test_auc:0.988873,  lr:0.100000\n",
            "epoch: 27, train_loss: 0.016839, train_auc:0.980936, val_auc:0.979169, test_auc:0.989429,  lr:0.100000\n",
            "epoch: 28, train_loss: 0.014201, train_auc:0.980063, val_auc:0.979471, test_auc:0.989356,  lr:0.100000\n",
            "epoch: 29, train_loss: 0.014114, train_auc:0.980011, val_auc:0.979386, test_auc:0.989394,  lr:0.100000\n",
            "epoch: 30, train_loss: 0.015655, train_auc:0.982252, val_auc:0.979398, test_auc:0.989609,  lr:0.100000\n",
            "epoch: 31, train_loss: 0.015468, train_auc:0.982920, val_auc:0.979283, test_auc:0.989937,  lr:0.100000\n",
            "epoch: 32, train_loss: 0.015333, train_auc:0.981633, val_auc:0.979073, test_auc:0.989641,  lr:0.100000\n",
            "epoch: 33, train_loss: 0.015350, train_auc:0.980184, val_auc:0.979645, test_auc:0.989733,  lr:0.100000\n",
            "epoch: 34, train_loss: 0.014068, train_auc:0.980279, val_auc:0.979873, test_auc:0.989660,  lr:0.100000\n",
            "epoch: 35, train_loss: 0.014225, train_auc:0.981510, val_auc:0.979692, test_auc:0.989580,  lr:0.100000\n",
            "epoch: 36, train_loss: 0.014050, train_auc:0.981221, val_auc:0.980119, test_auc:0.989805,  lr:0.100000\n",
            "epoch: 37, train_loss: 0.015571, train_auc:0.981776, val_auc:0.980403, test_auc:0.989955,  lr:0.100000\n",
            "epoch: 38, train_loss: 0.014480, train_auc:0.981087, val_auc:0.980529, test_auc:0.989669,  lr:0.100000\n",
            "epoch: 39, train_loss: 0.013828, train_auc:0.981337, val_auc:0.981070, test_auc:0.989784,  lr:0.100000\n",
            "epoch: 40, train_loss: 0.015271, train_auc:0.981078, val_auc:0.980627, test_auc:0.989597,  lr:0.100000\n",
            "epoch: 41, train_loss: 0.014988, train_auc:0.983303, val_auc:0.981133, test_auc:0.989461,  lr:0.100000\n",
            "epoch: 42, train_loss: 0.013815, train_auc:0.981388, val_auc:0.981312, test_auc:0.989468,  lr:0.100000\n",
            "epoch: 43, train_loss: 0.014948, train_auc:0.980936, val_auc:0.981049, test_auc:0.989507,  lr:0.100000\n",
            "epoch: 44, train_loss: 0.013182, train_auc:0.981473, val_auc:0.981382, test_auc:0.989341,  lr:0.100000\n",
            "epoch: 45, train_loss: 0.015420, train_auc:0.981819, val_auc:0.981295, test_auc:0.989428,  lr:0.100000\n",
            "epoch: 46, train_loss: 0.015875, train_auc:0.982443, val_auc:0.982049, test_auc:0.989478,  lr:0.100000\n",
            "epoch: 47, train_loss: 0.015180, train_auc:0.982477, val_auc:0.981485, test_auc:0.989441,  lr:0.100000\n",
            "epoch: 48, train_loss: 0.014907, train_auc:0.981057, val_auc:0.980825, test_auc:0.989200,  lr:0.100000\n",
            "epoch: 49, train_loss: 0.013515, train_auc:0.982032, val_auc:0.981194, test_auc:0.989358,  lr:0.100000\n",
            "Reducing learning rate to 0.01000 @ T=4900!\n",
            "Updating regularizer @ T=4900!\n",
            "epoch: 50, train_loss: 0.014968, train_auc:0.982081, val_auc:0.981151, test_auc:0.989295,  lr:0.010000\n",
            "epoch: 51, train_loss: 0.014121, train_auc:0.981446, val_auc:0.981112, test_auc:0.989262,  lr:0.010000\n",
            "epoch: 52, train_loss: 0.014792, train_auc:0.981354, val_auc:0.981069, test_auc:0.989235,  lr:0.010000\n",
            "epoch: 53, train_loss: 0.014898, train_auc:0.981338, val_auc:0.981075, test_auc:0.989222,  lr:0.010000\n",
            "epoch: 54, train_loss: 0.014320, train_auc:0.981772, val_auc:0.981025, test_auc:0.989198,  lr:0.010000\n",
            "epoch: 55, train_loss: 0.015007, train_auc:0.982366, val_auc:0.980860, test_auc:0.989130,  lr:0.010000\n",
            "epoch: 56, train_loss: 0.013957, train_auc:0.980805, val_auc:0.980872, test_auc:0.989117,  lr:0.010000\n",
            "epoch: 57, train_loss: 0.013544, train_auc:0.981925, val_auc:0.980805, test_auc:0.989099,  lr:0.010000\n",
            "epoch: 58, train_loss: 0.016241, train_auc:0.983173, val_auc:0.980798, test_auc:0.989086,  lr:0.010000\n",
            "epoch: 59, train_loss: 0.014381, train_auc:0.983023, val_auc:0.980848, test_auc:0.989089,  lr:0.010000\n",
            "epoch: 60, train_loss: 0.014814, train_auc:0.982583, val_auc:0.980803, test_auc:0.989030,  lr:0.010000\n",
            "epoch: 61, train_loss: 0.012768, train_auc:0.982484, val_auc:0.980889, test_auc:0.989046,  lr:0.010000\n",
            "epoch: 62, train_loss: 0.012468, train_auc:0.981556, val_auc:0.980930, test_auc:0.989023,  lr:0.010000\n",
            "epoch: 63, train_loss: 0.013238, train_auc:0.979999, val_auc:0.980871, test_auc:0.989016,  lr:0.010000\n",
            "epoch: 64, train_loss: 0.014495, train_auc:0.982727, val_auc:0.980854, test_auc:0.989080,  lr:0.010000\n",
            "epoch: 65, train_loss: 0.014698, train_auc:0.982646, val_auc:0.980736, test_auc:0.989057,  lr:0.010000\n",
            "epoch: 66, train_loss: 0.013530, train_auc:0.982122, val_auc:0.980673, test_auc:0.989054,  lr:0.010000\n",
            "epoch: 67, train_loss: 0.014284, train_auc:0.982349, val_auc:0.980648, test_auc:0.989027,  lr:0.010000\n",
            "epoch: 68, train_loss: 0.014970, train_auc:0.982247, val_auc:0.980711, test_auc:0.989032,  lr:0.010000\n",
            "epoch: 69, train_loss: 0.016085, train_auc:0.980398, val_auc:0.980677, test_auc:0.989019,  lr:0.010000\n",
            "epoch: 70, train_loss: 0.013710, train_auc:0.981449, val_auc:0.980724, test_auc:0.989046,  lr:0.010000\n",
            "epoch: 71, train_loss: 0.014687, train_auc:0.982278, val_auc:0.980781, test_auc:0.989062,  lr:0.010000\n",
            "epoch: 72, train_loss: 0.014002, train_auc:0.982489, val_auc:0.980802, test_auc:0.989098,  lr:0.010000\n",
            "epoch: 73, train_loss: 0.014220, train_auc:0.981046, val_auc:0.980868, test_auc:0.989091,  lr:0.010000\n",
            "epoch: 74, train_loss: 0.013168, train_auc:0.982453, val_auc:0.980812, test_auc:0.989094,  lr:0.010000\n",
            "Reducing learning rate to 0.00100 @ T=7350!\n",
            "Updating regularizer @ T=7350!\n",
            "epoch: 75, train_loss: 0.016023, train_auc:0.984788, val_auc:0.980815, test_auc:0.989098,  lr:0.001000\n",
            "epoch: 76, train_loss: 0.012563, train_auc:0.983828, val_auc:0.980817, test_auc:0.989102,  lr:0.001000\n",
            "epoch: 77, train_loss: 0.014654, train_auc:0.981510, val_auc:0.980820, test_auc:0.989099,  lr:0.001000\n",
            "epoch: 78, train_loss: 0.014325, train_auc:0.982513, val_auc:0.980831, test_auc:0.989098,  lr:0.001000\n",
            "epoch: 79, train_loss: 0.014242, train_auc:0.983171, val_auc:0.980831, test_auc:0.989101,  lr:0.001000\n",
            "epoch: 80, train_loss: 0.014147, train_auc:0.980081, val_auc:0.980842, test_auc:0.989099,  lr:0.001000\n",
            "epoch: 81, train_loss: 0.013795, train_auc:0.982870, val_auc:0.980823, test_auc:0.989096,  lr:0.001000\n",
            "epoch: 82, train_loss: 0.014678, train_auc:0.982575, val_auc:0.980824, test_auc:0.989105,  lr:0.001000\n",
            "epoch: 83, train_loss: 0.013198, train_auc:0.982020, val_auc:0.980814, test_auc:0.989099,  lr:0.001000\n",
            "epoch: 84, train_loss: 0.015024, train_auc:0.981582, val_auc:0.980816, test_auc:0.989101,  lr:0.001000\n",
            "epoch: 85, train_loss: 0.013046, train_auc:0.982119, val_auc:0.980826, test_auc:0.989106,  lr:0.001000\n",
            "epoch: 86, train_loss: 0.014473, train_auc:0.982803, val_auc:0.980819, test_auc:0.989108,  lr:0.001000\n",
            "epoch: 87, train_loss: 0.014433, train_auc:0.981597, val_auc:0.980821, test_auc:0.989108,  lr:0.001000\n",
            "epoch: 88, train_loss: 0.014153, train_auc:0.981679, val_auc:0.980823, test_auc:0.989106,  lr:0.001000\n",
            "epoch: 89, train_loss: 0.013462, train_auc:0.980667, val_auc:0.980822, test_auc:0.989108,  lr:0.001000\n",
            "epoch: 90, train_loss: 0.016342, train_auc:0.981775, val_auc:0.980834, test_auc:0.989111,  lr:0.001000\n",
            "epoch: 91, train_loss: 0.013731, train_auc:0.981800, val_auc:0.980829, test_auc:0.989109,  lr:0.001000\n",
            "epoch: 92, train_loss: 0.013416, train_auc:0.981840, val_auc:0.980825, test_auc:0.989104,  lr:0.001000\n",
            "epoch: 93, train_loss: 0.015391, train_auc:0.980564, val_auc:0.980825, test_auc:0.989104,  lr:0.001000\n",
            "epoch: 94, train_loss: 0.013617, train_auc:0.980810, val_auc:0.980832, test_auc:0.989102,  lr:0.001000\n",
            "epoch: 95, train_loss: 0.014683, train_auc:0.982182, val_auc:0.980839, test_auc:0.989101,  lr:0.001000\n",
            "epoch: 96, train_loss: 0.013961, train_auc:0.981472, val_auc:0.980830, test_auc:0.989099,  lr:0.001000\n",
            "epoch: 97, train_loss: 0.014281, train_auc:0.982440, val_auc:0.980826, test_auc:0.989098,  lr:0.001000\n",
            "epoch: 98, train_loss: 0.014275, train_auc:0.983521, val_auc:0.980822, test_auc:0.989097,  lr:0.001000\n",
            "epoch: 99, train_loss: 0.012714, train_auc:0.981064, val_auc:0.980830, test_auc:0.989102,  lr:0.001000\n"
          ]
        }
      ],
      "source": [
        "print ('Start Training')\n",
        "print ('-'*30)\n",
        "for epoch in range(100):\n",
        "     if epoch == 50 or epoch==75:\n",
        "         # decrease learning rate by 10x & update regularizer\n",
        "         optimizer.update_regularizer(decay_factor=10)\n",
        "  \n",
        "     train_pred = []\n",
        "     train_true = []\n",
        "     model.train()  \n",
        "     for data, targets in trainloader:\n",
        "         data, targets  = data.cuda(), targets.cuda()\n",
        "         y_pred = model(data)\n",
        "         #y_prob = torch.sigmoid(y_pred) # options: sigmoid, l2, none\n",
        "         loss = loss_fn(y_pred, targets)\n",
        "         optimizer.zero_grad()\n",
        "         loss.backward()\n",
        "         optimizer.step()\n",
        "         train_pred.append(y_pred.cpu().detach().numpy())\n",
        "         train_true.append(targets.cpu().detach().numpy())\n",
        "     \n",
        "     train_true = np.concatenate(train_true)\n",
        "     train_pred = np.concatenate(train_pred)\n",
        "     train_auc = roc_auc_score(train_true, train_pred) \n",
        "\n",
        "     val_auc = evaluate(valloader, model)\n",
        "     test_auc = evaluate(testloader, model)\n",
        "\n",
        "     # print results\n",
        "     print(\"epoch: {}, train_loss: {:4f}, train_auc:{:4f}, val_auc:{:4f}, test_auc:{:4f},  lr:{:4f}\".format(epoch, loss.item(), train_auc, val_auc, test_auc, optimizer.lr ))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}