GAN - CIFAR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GAN Image Example: CIFAR\n",
    "\n",
    "* _Author_: Sebastian Nowozin (Sebastian.Nowozin@microsoft.com)\n",
    "* _Date_: 16th July 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torch\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.utils as vutils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "CIFAR10 = torchvision.datasets.CIFAR10('data-cifar10', train=True,\n",
    "    download=True, transform=transform)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GAN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConjugateDualFunction:\n",
    "    def __init__(self, divergence_name):\n",
    "        self.divergence_name = divergence_name\n",
    "\n",
    "    def T(self, v):\n",
    "        \"\"\"Compute T(v) repersentation\n",
    "        \n",
    "        Arguments\n",
    "        v -- The scalar output (full real number range) of the discriminator\n",
    "        \"\"\"\n",
    "        if self.divergence_name == \"kl\":\n",
    "            return v\n",
    "        elif self.divergence_name == \"klrev\":\n",
    "            return -F.exp(v)\n",
    "        elif self.divergence_name == \"pearson\":\n",
    "            return v\n",
    "        elif self.divergence_name == \"neyman\":\n",
    "            return 1.0 - F.exp(v)\n",
    "        elif self.divergence_name == \"hellinger\":\n",
    "            return 1.0 - F.exp(v)\n",
    "        elif self.divergence_name == \"jensen\":\n",
    "            return math.log(2.0) - F.softplus(-v)\n",
    "        elif self.divergence_name == \"gan\":\n",
    "            return -F.softplus(-v)\n",
    "        else:\n",
    "            raise ValueError(\"Unknown divergence name in t function.\")\n",
    "\n",
    "    def fstarT(self, v):\n",
    "        \"\"\"Compute the f^*(T(v)) representation\n",
    "        \n",
    "        Arguments\n",
    "        v -- The scalar output of the variational function neural network.\n",
    "        \"\"\"\n",
    "        if self.divergence_name == \"kl\":\n",
    "            return torch.exp(v - 1.0)\n",
    "        elif self.divergence_name == \"klrev\":\n",
    "            return -1.0 - v\n",
    "        elif self.divergence_name == \"pearson\":\n",
    "            return 0.25*v*v + v\n",
    "        elif self.divergence_name == \"neyman\":\n",
    "            return 2.0 - 2.0*F.exp(0.5*v)\n",
    "        elif self.divergence_name == \"hellinger\":\n",
    "            return F.exp(-v) - 1.0\n",
    "        elif self.divergence_name == \"jensen\":\n",
    "            return F.softplus(v) - math.log(2.0)\n",
    "        elif self.divergence_name == \"gan\":\n",
    "            return F.softplus(v)\n",
    "        else:\n",
    "            raise ValueError(\"Unknown divergence name in fstar_t function.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DCGAN architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DCGANGenerator(nn.Module):\n",
    "    def __init__(self, nrand):\n",
    "        super(DCGANGenerator, self).__init__()\n",
    "        self.lin1 = nn.Linear(nrand, 4*4*512)\n",
    "        init.xavier_uniform_(self.lin1.weight, gain=0.1)\n",
    "        self.lin1bn = nn.BatchNorm1d(4*4*512)\n",
    "        self.dc1 = nn.ConvTranspose2d(512, 256, 4, stride=2, padding=1)\n",
    "        self.dc1bn = nn.BatchNorm2d(256)\n",
    "        self.dc2 = nn.ConvTranspose2d(256, 128, 4, stride=2, padding=1)\n",
    "        self.dc2bn = nn.BatchNorm2d(128)\n",
    "        self.dc3a = nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1)\n",
    "        self.dc3abn = nn.BatchNorm2d(64)\n",
    "        self.dc3b = nn.Conv2d(64, 3, 3, stride=1, padding=1)\n",
    "\n",
    "    def forward(self, z):\n",
    "        h = F.relu(self.lin1bn(self.lin1(z)))\n",
    "        h = torch.reshape(h, (-1, 512, 4, 4))\n",
    "\n",
    "        # deconv stack\n",
    "        h = F.relu(self.dc1bn(self.dc1(h)))\n",
    "        h = F.relu(self.dc2bn(self.dc2(h)))\n",
    "        h = F.relu(self.dc3abn(self.dc3a(h)))\n",
    "        x = self.dc3b(h)\n",
    "\n",
    "        return x\n",
    "\n",
    "class DCGANDiscriminator(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DCGANDiscriminator, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 64, 4, stride=2, padding=1)\n",
    "        self.conv1bn = nn.BatchNorm2d(64)\n",
    "        self.conv2 = nn.Conv2d(64, 128, 4, stride=2, padding=1)\n",
    "        self.conv2bn = nn.BatchNorm2d(128)\n",
    "        self.conv3 = nn.Conv2d(128, 256, 4, stride=2, padding=1)\n",
    "        self.conv3bn = nn.BatchNorm2d(256)\n",
    "        self.lin1 = nn.Linear(4*4*256, 512)\n",
    "        self.lin1bn = nn.BatchNorm1d(512)\n",
    "        self.lin2 = nn.Linear(512, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h = F.elu(self.conv1bn(self.conv1(x)))\n",
    "        h = F.elu(self.conv2bn(self.conv2(h)))\n",
    "        h = F.elu(self.conv3bn(self.conv3(h)))\n",
    "        h = torch.reshape(h, (-1, 4*4*256))\n",
    "\n",
    "        h = F.elu(self.lin1bn(self.lin1(h)))\n",
    "        v = self.lin2(h)\n",
    "\n",
    "        return v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FGANLearningObjective(nn.Module):\n",
    "    def __init__(self, gen, disc, divergence_name=\"gan\", gamma=0.01):\n",
    "        super(FGANLearningObjective, self).__init__()\n",
    "        self.gen = gen\n",
    "        self.disc = disc\n",
    "        self.conj = ConjugateDualFunction(divergence_name)\n",
    "        self.gammahalf = 0.5*gamma\n",
    "\n",
    "    def forward(self, xreal, zmodel):\n",
    "        # Real data\n",
    "        vreal = self.disc(xreal)    # Real data discriminator output\n",
    "        Treal = self.conj.T(vreal)  # Mapped to T-space\n",
    "\n",
    "        # Model data\n",
    "        xmodel = self.gen(zmodel)   # Map noise to data\n",
    "        vmodel = self.disc(xmodel)  # Model data discriminator output\n",
    "        fstar_Tmodel = self.conj.fstarT(vmodel)   # Mapped to f^*(T)\n",
    "\n",
    "        # Compute generator loss\n",
    "        loss_gen = -fstar_Tmodel.mean()\n",
    "\n",
    "        # Compute discriminator loss (negation because we minimize)\n",
    "        loss_disc = fstar_Tmodel.mean() - Treal.mean()\n",
    "\n",
    "        # Compute gradient penalty as per (Mescheder et al., ICML 2018)\n",
    "        if self.gammahalf > 0.0:\n",
    "            #grad_pd = torch.autograd.grad(Treal.mean(), xreal,\n",
    "            #    create_graph=True)[0]\n",
    "            #grad_pd_norm2 = grad_pd.pow(2).sum()\n",
    "            #gradient_penalty = self.gammahalf * grad_pd_norm2\n",
    "            #loss_disc += gradient_penalty\n",
    "            batchsize = xreal.size(0)\n",
    "            grad_pd = torch.autograd.grad(Treal.sum(), xreal,\n",
    "                create_graph=True, only_inputs=True)[0]\n",
    "            grad_pd_norm2 = grad_pd.pow(2)\n",
    "            grad_pd_norm2 = grad_pd_norm2.view(batchsize, -1).sum(1)\n",
    "            gradient_penalty = self.gammahalf * grad_pd_norm2.mean()\n",
    "            loss_disc += gradient_penalty\n",
    "\n",
    "        return loss_gen, loss_disc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=2)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:2\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "nrand = 100\n",
    "gen = DCGANGenerator(nrand)\n",
    "disc = DCGANDiscriminator()\n",
    "fgan = FGANLearningObjective(gen, disc, \"gan\", gamma=10.0)\n",
    "fgan = fgan.to(device)\n",
    "#fgan = torch.nn.DataParallel(fgan)\n",
    "#fgan.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "batchsize = 64\n",
    "optimizer_gen = optim.Adam(fgan.gen.parameters(), lr=1.0e-3)\n",
    "optimizer_disc = optim.Adam(fgan.disc.parameters(), lr=1.0e-3)\n",
    "\n",
    "trainloader = torch.utils.data.DataLoader(CIFAR10,\n",
    "    batch_size=batchsize, shuffle=True, num_workers=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  iter 1  obj(D) 1.4613  obj(G) -0.6857\n",
      "epoch 1  iter 783  obj(D) 1.3303  obj(G) -0.6956\n",
      "epoch 2  iter 1565  obj(D) 1.3545  obj(G) -0.6759\n",
      "epoch 3  iter 2347  obj(D) 1.2330  obj(G) -0.6286\n",
      "epoch 4  iter 3129  obj(D) 1.3652  obj(G) -0.6625\n",
      "epoch 5  iter 3911  obj(D) 1.2568  obj(G) -0.6419\n",
      "epoch 6  iter 4693  obj(D) 0.9561  obj(G) -0.5000\n",
      "epoch 7  iter 5475  obj(D) 0.9355  obj(G) -0.5433\n",
      "epoch 8  iter 6257  obj(D) 1.1334  obj(G) -0.6481\n",
      "epoch 9  iter 7039  obj(D) 1.1495  obj(G) -0.5758\n",
      "epoch 10  iter 7821  obj(D) 1.3493  obj(G) -0.6366\n",
      "epoch 11  iter 8603  obj(D) 1.0560  obj(G) -0.6393\n",
      "epoch 12  iter 9385  obj(D) 0.8939  obj(G) -0.4566\n",
      "epoch 13  iter 10167  obj(D) 1.3560  obj(G) -0.7240\n",
      "epoch 14  iter 10949  obj(D) 0.8296  obj(G) -0.4685\n",
      "epoch 15  iter 11731  obj(D) 1.1155  obj(G) -0.4456\n",
      "epoch 16  iter 12513  obj(D) 1.1282  obj(G) -0.5707\n",
      "epoch 17  iter 13295  obj(D) 1.2698  obj(G) -0.8322\n",
      "epoch 18  iter 14077  obj(D) 1.2235  obj(G) -0.6891\n",
      "epoch 19  iter 14859  obj(D) 1.5130  obj(G) -0.9399\n",
      "epoch 20  iter 15641  obj(D) 1.2440  obj(G) -0.6899\n",
      "epoch 21  iter 16423  obj(D) 1.2843  obj(G) -0.8105\n",
      "epoch 22  iter 17205  obj(D) 1.2173  obj(G) -0.6466\n",
      "epoch 23  iter 17987  obj(D) 1.3020  obj(G) -0.5936\n",
      "epoch 24  iter 18769  obj(D) 1.1897  obj(G) -0.6488\n",
      "epoch 25  iter 19551  obj(D) 1.3135  obj(G) -0.6417\n",
      "epoch 26  iter 20333  obj(D) 1.3119  obj(G) -0.6719\n",
      "epoch 27  iter 21115  obj(D) 1.3580  obj(G) -0.7327\n",
      "epoch 28  iter 21897  obj(D) 1.2920  obj(G) -0.6654\n",
      "epoch 29  iter 22679  obj(D) 1.3014  obj(G) -0.6817\n",
      "epoch 30  iter 23461  obj(D) 1.4796  obj(G) -0.8137\n",
      "epoch 31  iter 24243  obj(D) 1.4609  obj(G) -0.8480\n",
      "epoch 32  iter 25025  obj(D) 1.4167  obj(G) -0.7350\n",
      "epoch 33  iter 25807  obj(D) 1.3481  obj(G) -0.6519\n",
      "epoch 34  iter 26589  obj(D) 1.3573  obj(G) -0.6611\n",
      "epoch 35  iter 27371  obj(D) 1.3107  obj(G) -0.6754\n",
      "epoch 36  iter 28153  obj(D) 1.3339  obj(G) -0.6849\n",
      "epoch 37  iter 28935  obj(D) 1.4197  obj(G) -0.7532\n",
      "epoch 38  iter 29717  obj(D) 1.3531  obj(G) -0.6998\n",
      "epoch 39  iter 30499  obj(D) 1.3304  obj(G) -0.7246\n",
      "epoch 40  iter 31281  obj(D) 1.3555  obj(G) -0.7050\n",
      "epoch 41  iter 32063  obj(D) 1.3339  obj(G) -0.6800\n",
      "epoch 42  iter 32845  obj(D) 1.4041  obj(G) -0.7432\n",
      "epoch 43  iter 33627  obj(D) 1.2265  obj(G) -0.6663\n",
      "epoch 44  iter 34409  obj(D) 1.4393  obj(G) -0.7077\n",
      "epoch 45  iter 35191  obj(D) 1.3414  obj(G) -0.7001\n",
      "epoch 46  iter 35973  obj(D) 1.3726  obj(G) -0.7060\n",
      "epoch 47  iter 36755  obj(D) 1.3272  obj(G) -0.7121\n",
      "epoch 48  iter 37537  obj(D) 1.2920  obj(G) -0.6754\n",
      "epoch 49  iter 38319  obj(D) 1.3140  obj(G) -0.7081\n",
      "epoch 50  iter 39101  obj(D) 1.3040  obj(G) -0.7099\n",
      "epoch 51  iter 39883  obj(D) 1.3444  obj(G) -0.7217\n",
      "epoch 52  iter 40665  obj(D) 1.2516  obj(G) -0.6741\n",
      "epoch 53  iter 41447  obj(D) 1.3297  obj(G) -0.6991\n",
      "epoch 54  iter 42229  obj(D) 1.3409  obj(G) -0.7318\n",
      "epoch 55  iter 43011  obj(D) 1.3940  obj(G) -0.7094\n",
      "epoch 56  iter 43793  obj(D) 1.3001  obj(G) -0.6672\n",
      "epoch 57  iter 44575  obj(D) 1.3350  obj(G) -0.7161\n",
      "epoch 58  iter 45357  obj(D) 1.3403  obj(G) -0.6625\n",
      "epoch 59  iter 46139  obj(D) 1.2408  obj(G) -0.6621\n",
      "epoch 60  iter 46921  obj(D) 1.2689  obj(G) -0.6945\n",
      "epoch 61  iter 47703  obj(D) 1.3097  obj(G) -0.7048\n",
      "epoch 62  iter 48485  obj(D) 1.3091  obj(G) -0.7070\n",
      "epoch 63  iter 49267  obj(D) 1.3381  obj(G) -0.7270\n",
      "epoch 64  iter 50049  obj(D) 1.3347  obj(G) -0.6978\n",
      "epoch 65  iter 50831  obj(D) 1.2286  obj(G) -0.6516\n",
      "epoch 66  iter 51613  obj(D) 1.3196  obj(G) -0.7076\n",
      "epoch 67  iter 52395  obj(D) 1.2892  obj(G) -0.6534\n",
      "epoch 68  iter 53177  obj(D) 1.2597  obj(G) -0.7271\n",
      "epoch 69  iter 53959  obj(D) 1.2555  obj(G) -0.6664\n",
      "epoch 70  iter 54741  obj(D) 1.2196  obj(G) -0.6590\n",
      "epoch 71  iter 55523  obj(D) 1.2831  obj(G) -0.6517\n",
      "epoch 72  iter 56305  obj(D) 1.2826  obj(G) -0.6975\n",
      "epoch 73  iter 57087  obj(D) 1.1444  obj(G) -0.6966\n",
      "epoch 74  iter 57869  obj(D) 1.2400  obj(G) -0.6369\n",
      "epoch 75  iter 58651  obj(D) 1.3243  obj(G) -0.7310\n",
      "epoch 76  iter 59433  obj(D) 1.2972  obj(G) -0.7199\n",
      "epoch 77  iter 60215  obj(D) 1.3556  obj(G) -0.6843\n",
      "epoch 78  iter 60997  obj(D) 1.2931  obj(G) -0.6657\n",
      "epoch 79  iter 61779  obj(D) 1.1565  obj(G) -0.7074\n",
      "epoch 80  iter 62561  obj(D) 1.2175  obj(G) -0.6891\n",
      "epoch 81  iter 63343  obj(D) 1.2461  obj(G) -0.6634\n",
      "epoch 82  iter 64125  obj(D) 1.3115  obj(G) -0.7360\n",
      "epoch 83  iter 64907  obj(D) 1.2111  obj(G) -0.6869\n",
      "epoch 84  iter 65689  obj(D) 1.2213  obj(G) -0.6099\n",
      "epoch 85  iter 66471  obj(D) 1.2403  obj(G) -0.7710\n",
      "epoch 86  iter 67253  obj(D) 1.1767  obj(G) -0.6564\n",
      "epoch 87  iter 68035  obj(D) 1.1866  obj(G) -0.6178\n",
      "epoch 88  iter 68817  obj(D) 1.2301  obj(G) -0.6644\n",
      "epoch 89  iter 69599  obj(D) 1.2515  obj(G) -0.6639\n",
      "epoch 90  iter 70381  obj(D) 1.1696  obj(G) -0.5967\n",
      "epoch 91  iter 71163  obj(D) 1.2766  obj(G) -0.7226\n",
      "epoch 92  iter 71945  obj(D) 1.1972  obj(G) -0.6401\n",
      "epoch 93  iter 72727  obj(D) 1.2457  obj(G) -0.6806\n",
      "epoch 94  iter 73509  obj(D) 1.2310  obj(G) -0.6617\n",
      "epoch 95  iter 74291  obj(D) 1.1194  obj(G) -0.6504\n",
      "epoch 96  iter 75073  obj(D) 1.2047  obj(G) -0.6378\n",
      "epoch 97  iter 75855  obj(D) 1.2342  obj(G) -0.7075\n",
      "epoch 98  iter 76637  obj(D) 1.0240  obj(G) -0.5824\n",
      "epoch 99  iter 77419  obj(D) 1.1462  obj(G) -0.6063\n",
      "epoch 100  iter 78201  obj(D) 1.0980  obj(G) -0.6239\n",
      "epoch 101  iter 78983  obj(D) 1.1120  obj(G) -0.5690\n",
      "epoch 102  iter 79765  obj(D) 1.1008  obj(G) -0.7144\n",
      "epoch 103  iter 80547  obj(D) 1.1663  obj(G) -0.6351\n",
      "epoch 104  iter 81329  obj(D) 1.1990  obj(G) -0.7003\n",
      "epoch 105  iter 82111  obj(D) 1.1022  obj(G) -0.5853\n",
      "epoch 106  iter 82893  obj(D) 1.0390  obj(G) -0.6481\n",
      "epoch 107  iter 83675  obj(D) 1.0907  obj(G) -0.6503\n",
      "epoch 108  iter 84457  obj(D) 1.1810  obj(G) -0.7302\n",
      "epoch 109  iter 85239  obj(D) 1.1133  obj(G) -0.7124\n",
      "epoch 110  iter 86021  obj(D) 1.1162  obj(G) -0.6355\n",
      "epoch 111  iter 86803  obj(D) 1.0328  obj(G) -0.6608\n",
      "epoch 112  iter 87585  obj(D) 1.1098  obj(G) -0.6717\n",
      "epoch 113  iter 88367  obj(D) 1.3715  obj(G) -0.7497\n",
      "epoch 114  iter 89149  obj(D) 1.3127  obj(G) -0.8281\n",
      "epoch 115  iter 89931  obj(D) 1.1222  obj(G) -0.7319\n",
      "epoch 116  iter 90713  obj(D) 1.1799  obj(G) -0.6970\n",
      "epoch 117  iter 91495  obj(D) 1.0788  obj(G) -0.6550\n",
      "epoch 118  iter 92277  obj(D) 1.2229  obj(G) -0.7099\n",
      "epoch 119  iter 93059  obj(D) 1.0087  obj(G) -0.6937\n",
      "epoch 120  iter 93841  obj(D) 1.0624  obj(G) -0.7634\n",
      "epoch 121  iter 94623  obj(D) 0.9397  obj(G) -0.6649\n",
      "epoch 122  iter 95405  obj(D) 1.0288  obj(G) -0.6414\n",
      "epoch 123  iter 96187  obj(D) 1.1386  obj(G) -0.7676\n",
      "epoch 124  iter 96969  obj(D) 1.0985  obj(G) -0.7526\n",
      "epoch 125  iter 97751  obj(D) 1.0241  obj(G) -0.6327\n",
      "epoch 126  iter 98533  obj(D) 1.0914  obj(G) -0.5715\n",
      "epoch 127  iter 99315  obj(D) 0.9895  obj(G) -0.6094\n",
      "epoch 128  iter 100097  obj(D) 1.2006  obj(G) -0.7447\n",
      "epoch 129  iter 100879  obj(D) 0.9843  obj(G) -0.6332\n",
      "epoch 130  iter 101661  obj(D) 1.0114  obj(G) -0.7610\n",
      "epoch 131  iter 102443  obj(D) 1.0456  obj(G) -0.7959\n",
      "epoch 132  iter 103225  obj(D) 1.0997  obj(G) -0.7368\n",
      "epoch 133  iter 104007  obj(D) 0.9402  obj(G) -0.7406\n",
      "epoch 134  iter 104789  obj(D) 1.0471  obj(G) -0.6053\n",
      "epoch 135  iter 105571  obj(D) 1.0760  obj(G) -0.7709\n",
      "epoch 136  iter 106353  obj(D) 1.3201  obj(G) -0.6950\n",
      "epoch 137  iter 107135  obj(D) 1.0548  obj(G) -0.7884\n",
      "epoch 138  iter 107917  obj(D) 1.1893  obj(G) -0.8424\n",
      "epoch 139  iter 108699  obj(D) 0.9538  obj(G) -0.6645\n",
      "epoch 140  iter 109481  obj(D) 1.1304  obj(G) -0.7008\n",
      "epoch 141  iter 110263  obj(D) 1.2934  obj(G) -0.7499\n",
      "epoch 142  iter 111045  obj(D) 1.0841  obj(G) -0.5789\n",
      "epoch 143  iter 111827  obj(D) 1.0659  obj(G) -0.7225\n",
      "epoch 144  iter 112609  obj(D) 1.4787  obj(G) -0.7620\n",
      "epoch 145  iter 113391  obj(D) 1.1410  obj(G) -0.7676\n",
      "epoch 146  iter 114173  obj(D) 1.0283  obj(G) -0.6707\n",
      "epoch 147  iter 114955  obj(D) 0.9853  obj(G) -0.6611\n",
      "epoch 148  iter 115737  obj(D) 0.8771  obj(G) -0.6514\n",
      "epoch 149  iter 116519  obj(D) 0.8189  obj(G) -0.5385\n",
      "epoch 150  iter 117301  obj(D) 1.2372  obj(G) -0.6099\n",
      "epoch 151  iter 118083  obj(D) 0.9193  obj(G) -0.6716\n",
      "epoch 152  iter 118865  obj(D) 0.7544  obj(G) -0.6148\n",
      "epoch 153  iter 119647  obj(D) 1.2693  obj(G) -0.7845\n",
      "epoch 154  iter 120429  obj(D) 1.0612  obj(G) -0.9310\n",
      "epoch 155  iter 121211  obj(D) 0.9384  obj(G) -0.6039\n",
      "epoch 156  iter 121993  obj(D) 1.0198  obj(G) -0.7176\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 157  iter 122775  obj(D) 0.7876  obj(G) -0.6376\n",
      "epoch 158  iter 123557  obj(D) 0.8966  obj(G) -0.6107\n",
      "epoch 159  iter 124339  obj(D) 1.4445  obj(G) -0.8904\n",
      "epoch 160  iter 125121  obj(D) 0.9392  obj(G) -0.6347\n",
      "epoch 161  iter 125903  obj(D) 0.7870  obj(G) -0.6551\n",
      "epoch 162  iter 126685  obj(D) 0.8489  obj(G) -0.5642\n",
      "epoch 163  iter 127467  obj(D) 0.8951  obj(G) -0.6522\n",
      "epoch 164  iter 128249  obj(D) 1.0173  obj(G) -0.8929\n",
      "epoch 165  iter 129031  obj(D) 0.7900  obj(G) -0.6457\n",
      "epoch 166  iter 129813  obj(D) 0.9947  obj(G) -0.7554\n",
      "epoch 167  iter 130595  obj(D) 1.0502  obj(G) -0.3953\n",
      "epoch 168  iter 131377  obj(D) 1.3208  obj(G) -0.5406\n",
      "epoch 169  iter 132159  obj(D) 0.6329  obj(G) -0.5329\n",
      "epoch 170  iter 132941  obj(D) 0.8676  obj(G) -0.5637\n",
      "epoch 171  iter 133723  obj(D) 0.9474  obj(G) -0.4868\n",
      "epoch 172  iter 134505  obj(D) 1.0741  obj(G) -0.7831\n",
      "epoch 173  iter 135287  obj(D) 1.0607  obj(G) -0.8684\n",
      "epoch 174  iter 136069  obj(D) 0.8705  obj(G) -0.7109\n",
      "epoch 175  iter 136851  obj(D) 1.3481  obj(G) -0.7050\n",
      "epoch 176  iter 137633  obj(D) 0.8153  obj(G) -0.7060\n",
      "epoch 177  iter 138415  obj(D) 0.8322  obj(G) -0.7568\n",
      "epoch 178  iter 139197  obj(D) 1.3549  obj(G) -0.9603\n",
      "epoch 179  iter 139979  obj(D) 1.0016  obj(G) -0.7817\n",
      "epoch 180  iter 140761  obj(D) 0.8021  obj(G) -0.7204\n",
      "epoch 181  iter 141543  obj(D) 0.2757  obj(G) -0.2013\n",
      "epoch 182  iter 142325  obj(D) 0.6969  obj(G) -0.5378\n",
      "epoch 183  iter 143107  obj(D) 0.8194  obj(G) -0.7700\n",
      "epoch 184  iter 143889  obj(D) 0.8426  obj(G) -0.8248\n",
      "epoch 185  iter 144671  obj(D) 1.0329  obj(G) -0.9465\n",
      "epoch 186  iter 145453  obj(D) 0.8009  obj(G) -0.6131\n",
      "epoch 187  iter 146235  obj(D) 0.8593  obj(G) -0.8307\n",
      "epoch 188  iter 147017  obj(D) 0.8860  obj(G) -0.6270\n",
      "epoch 189  iter 147799  obj(D) 1.0952  obj(G) -0.6882\n",
      "epoch 190  iter 148581  obj(D) 0.6403  obj(G) -0.5000\n",
      "epoch 191  iter 149363  obj(D) 0.3834  obj(G) -0.3096\n",
      "epoch 192  iter 150145  obj(D) 0.7173  obj(G) -0.6343\n",
      "epoch 193  iter 150927  obj(D) 0.8349  obj(G) -0.6333\n",
      "epoch 194  iter 151709  obj(D) 0.5718  obj(G) -0.5362\n",
      "epoch 195  iter 152491  obj(D) 0.6627  obj(G) -0.5108\n",
      "epoch 196  iter 153273  obj(D) 1.1848  obj(G) -0.7991\n",
      "epoch 197  iter 154055  obj(D) 1.0382  obj(G) -1.0229\n",
      "epoch 198  iter 154837  obj(D) 0.7408  obj(G) -0.7099\n",
      "epoch 199  iter 155619  obj(D) 1.1519  obj(G) -1.0593\n",
      "epoch 200  iter 156401  obj(D) 2.0590  obj(G) -0.5937\n",
      "epoch 201  iter 157183  obj(D) 0.1717  obj(G) -0.1481\n",
      "epoch 202  iter 157965  obj(D) 0.0985  obj(G) -0.0674\n",
      "epoch 203  iter 158747  obj(D) 0.0562  obj(G) -0.0471\n",
      "epoch 204  iter 159529  obj(D) 0.1205  obj(G) -0.1078\n",
      "epoch 205  iter 160311  obj(D) 1.0135  obj(G) -0.8883\n",
      "epoch 206  iter 161093  obj(D) 1.0901  obj(G) -0.7536\n",
      "epoch 207  iter 161875  obj(D) 1.2646  obj(G) -1.1298\n",
      "epoch 208  iter 162657  obj(D) 0.7121  obj(G) -0.6916\n",
      "epoch 209  iter 163439  obj(D) 0.7393  obj(G) -0.5271\n",
      "epoch 210  iter 164221  obj(D) 0.7342  obj(G) -0.7118\n",
      "epoch 211  iter 165003  obj(D) 1.0363  obj(G) -0.7492\n",
      "epoch 212  iter 165785  obj(D) 1.4302  obj(G) -1.0432\n",
      "epoch 213  iter 166567  obj(D) 0.8582  obj(G) -0.8245\n",
      "epoch 214  iter 167349  obj(D) 0.6359  obj(G) -0.6005\n",
      "epoch 215  iter 168131  obj(D) 0.8264  obj(G) -0.5201\n",
      "epoch 216  iter 168913  obj(D) 0.9296  obj(G) -0.8743\n",
      "epoch 217  iter 169695  obj(D) 0.7057  obj(G) -0.6878\n",
      "epoch 218  iter 170477  obj(D) 0.7974  obj(G) -0.7684\n",
      "epoch 219  iter 171259  obj(D) 0.6402  obj(G) -0.4284\n",
      "epoch 220  iter 172041  obj(D) 1.2433  obj(G) -1.1569\n",
      "epoch 221  iter 172823  obj(D) 0.6458  obj(G) -0.6119\n",
      "epoch 222  iter 173605  obj(D) 1.0352  obj(G) -1.0039\n",
      "epoch 223  iter 174387  obj(D) 0.8136  obj(G) -0.6828\n",
      "epoch 224  iter 175169  obj(D) 0.6480  obj(G) -0.5289\n",
      "epoch 225  iter 175951  obj(D) 0.9457  obj(G) -0.4837\n",
      "epoch 226  iter 176733  obj(D) 0.7104  obj(G) -0.6795\n",
      "epoch 227  iter 177515  obj(D) 1.1564  obj(G) -0.3961\n",
      "epoch 228  iter 178297  obj(D) 1.0476  obj(G) -1.0247\n",
      "epoch 229  iter 179079  obj(D) 1.2355  obj(G) -1.1921\n",
      "epoch 230  iter 179861  obj(D) 1.0575  obj(G) -0.8418\n",
      "epoch 231  iter 180643  obj(D) 0.6917  obj(G) -0.3712\n",
      "epoch 232  iter 181425  obj(D) 0.5032  obj(G) -0.4162\n",
      "epoch 233  iter 182207  obj(D) 0.6803  obj(G) -0.6527\n",
      "epoch 234  iter 182989  obj(D) 1.2879  obj(G) -1.1778\n",
      "epoch 235  iter 183771  obj(D) 0.6354  obj(G) -0.5930\n",
      "epoch 236  iter 184553  obj(D) 0.6580  obj(G) -0.6327\n",
      "epoch 237  iter 185335  obj(D) 0.5861  obj(G) -0.5694\n",
      "epoch 238  iter 186117  obj(D) 0.6267  obj(G) -0.5863\n",
      "epoch 239  iter 186899  obj(D) 0.6377  obj(G) -0.5804\n",
      "epoch 240  iter 187681  obj(D) 0.8000  obj(G) -0.7766\n",
      "epoch 241  iter 188463  obj(D) 0.4912  obj(G) -0.4748\n",
      "epoch 242  iter 189245  obj(D) 0.5361  obj(G) -0.4772\n",
      "epoch 243  iter 190027  obj(D) 0.6569  obj(G) -0.2829\n",
      "epoch 244  iter 190809  obj(D) 0.6018  obj(G) -0.4940\n",
      "epoch 245  iter 191591  obj(D) 1.7785  obj(G) -1.7075\n",
      "epoch 246  iter 192373  obj(D) 0.7901  obj(G) -0.3295\n",
      "epoch 247  iter 193155  obj(D) 1.5961  obj(G) -1.5854\n",
      "epoch 248  iter 193937  obj(D) 0.7695  obj(G) -0.6687\n",
      "epoch 249  iter 194719  obj(D) 0.5544  obj(G) -0.5353\n",
      "epoch 250  iter 195501  obj(D) 0.2153  obj(G) -0.1881\n",
      "epoch 251  iter 196283  obj(D) 0.5433  obj(G) -0.4271\n",
      "epoch 252  iter 197065  obj(D) 0.3851  obj(G) -0.3513\n",
      "epoch 253  iter 197847  obj(D) 1.0595  obj(G) -0.9825\n",
      "epoch 254  iter 198629  obj(D) 0.6498  obj(G) -0.6374\n",
      "epoch 255  iter 199411  obj(D) 0.9756  obj(G) -0.8066\n",
      "epoch 256  iter 200193  obj(D) 0.7475  obj(G) -0.7199\n",
      "epoch 257  iter 200975  obj(D) 0.4986  obj(G) -0.4577\n",
      "epoch 258  iter 201757  obj(D) 0.3983  obj(G) -0.3894\n",
      "epoch 259  iter 202539  obj(D) 1.1781  obj(G) -0.0219\n",
      "epoch 260  iter 203321  obj(D) 0.0824  obj(G) -0.0457\n",
      "epoch 261  iter 204103  obj(D) 0.0265  obj(G) -0.0186\n",
      "epoch 262  iter 204885  obj(D) 0.0222  obj(G) -0.0062\n",
      "epoch 263  iter 205667  obj(D) 1.5722  obj(G) -1.4400\n",
      "epoch 264  iter 206449  obj(D) 0.5065  obj(G) -0.4378\n",
      "epoch 265  iter 207231  obj(D) 2.1570  obj(G) -2.1498\n",
      "epoch 266  iter 208013  obj(D) 3.5325  obj(G) -0.5171\n",
      "epoch 267  iter 208795  obj(D) 0.8627  obj(G) -0.6294\n",
      "epoch 268  iter 209577  obj(D) 1.0808  obj(G) -1.0784\n",
      "epoch 269  iter 210359  obj(D) 1.1803  obj(G) -1.1346\n",
      "epoch 270  iter 211141  obj(D) 0.9475  obj(G) -0.9181\n",
      "epoch 271  iter 211923  obj(D) 0.7035  obj(G) -0.6906\n",
      "epoch 272  iter 212705  obj(D) 2.2730  obj(G) -1.0481\n",
      "epoch 273  iter 213487  obj(D) 0.6232  obj(G) -0.5893\n",
      "epoch 274  iter 214269  obj(D) 1.3201  obj(G) -1.3161\n",
      "epoch 275  iter 215051  obj(D) 0.6319  obj(G) -0.5836\n",
      "epoch 276  iter 215833  obj(D) 0.7901  obj(G) -0.5686\n",
      "epoch 277  iter 216615  obj(D) 0.5076  obj(G) -0.5005\n",
      "epoch 278  iter 217397  obj(D) 0.6129  obj(G) -0.4880\n",
      "epoch 279  iter 218179  obj(D) 1.1929  obj(G) -1.1325\n",
      "epoch 280  iter 218961  obj(D) 0.8047  obj(G) -0.2304\n",
      "epoch 281  iter 219743  obj(D) 0.4375  obj(G) -0.4036\n",
      "epoch 282  iter 220525  obj(D) 0.7909  obj(G) -0.7330\n",
      "epoch 283  iter 221307  obj(D) 1.3451  obj(G) -1.3363\n",
      "epoch 284  iter 222089  obj(D) 0.7018  obj(G) -0.6644\n",
      "epoch 285  iter 222871  obj(D) 0.8706  obj(G) -0.8381\n",
      "epoch 286  iter 223653  obj(D) 0.4516  obj(G) -0.4440\n",
      "epoch 287  iter 224435  obj(D) 0.7264  obj(G) -0.7000\n",
      "epoch 288  iter 225217  obj(D) 0.7583  obj(G) -0.7381\n",
      "epoch 289  iter 225999  obj(D) 0.9838  obj(G) -0.8363\n",
      "epoch 290  iter 226781  obj(D) 0.5318  obj(G) -0.4740\n",
      "epoch 291  iter 227563  obj(D) 0.4889  obj(G) -0.4400\n",
      "epoch 292  iter 228345  obj(D) 2.4625  obj(G) -2.4576\n",
      "epoch 293  iter 229127  obj(D) 0.5712  obj(G) -0.5629\n",
      "epoch 294  iter 229909  obj(D) 1.1819  obj(G) -1.0808\n",
      "epoch 295  iter 230691  obj(D) 0.7914  obj(G) -0.7810\n",
      "epoch 296  iter 231473  obj(D) 3.6664  obj(G) -0.2910\n",
      "epoch 297  iter 232255  obj(D) 4.6295  obj(G) -4.6289\n",
      "epoch 298  iter 233037  obj(D) 1.0333  obj(G) -0.8884\n",
      "epoch 299  iter 233819  obj(D) 0.6497  obj(G) -0.3788\n",
      "epoch 300  iter 234601  obj(D) 0.9440  obj(G) -0.7011\n",
      "epoch 301  iter 235383  obj(D) 0.4271  obj(G) -0.4209\n",
      "epoch 302  iter 236165  obj(D) 0.6402  obj(G) -0.6274\n",
      "epoch 303  iter 236947  obj(D) 0.5727  obj(G) -0.5676\n",
      "epoch 304  iter 237729  obj(D) 0.4609  obj(G) -0.4196\n",
      "epoch 305  iter 238511  obj(D) 0.5393  obj(G) -0.5142\n",
      "epoch 306  iter 239293  obj(D) 0.4246  obj(G) -0.4015\n",
      "epoch 307  iter 240075  obj(D) 0.2973  obj(G) -0.2588\n",
      "epoch 308  iter 240857  obj(D) 0.8910  obj(G) -0.8785\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 309  iter 241639  obj(D) 0.8234  obj(G) -0.4799\n",
      "epoch 310  iter 242421  obj(D) 1.7026  obj(G) -1.6779\n",
      "epoch 311  iter 243203  obj(D) 0.7477  obj(G) -0.3120\n",
      "epoch 312  iter 243985  obj(D) 0.8660  obj(G) -0.6774\n",
      "epoch 313  iter 244767  obj(D) 1.4863  obj(G) -1.4843\n",
      "epoch 314  iter 245549  obj(D) 1.6165  obj(G) -1.5803\n",
      "epoch 315  iter 246331  obj(D) 0.8799  obj(G) -0.7179\n",
      "epoch 316  iter 247113  obj(D) 0.5746  obj(G) -0.5677\n",
      "epoch 317  iter 247895  obj(D) 0.4309  obj(G) -0.4255\n",
      "epoch 318  iter 248677  obj(D) 2.1282  obj(G) -1.1178\n",
      "epoch 319  iter 249459  obj(D) 1.1464  obj(G) -1.0733\n",
      "epoch 320  iter 250241  obj(D) 0.7623  obj(G) -0.7562\n",
      "epoch 321  iter 251023  obj(D) 0.4022  obj(G) -0.3480\n",
      "epoch 322  iter 251805  obj(D) 0.7237  obj(G) -0.4746\n",
      "epoch 323  iter 252587  obj(D) 0.3996  obj(G) -0.1901\n",
      "epoch 324  iter 253369  obj(D) 0.4154  obj(G) -0.4075\n",
      "epoch 325  iter 254151  obj(D) 1.3167  obj(G) -0.7032\n",
      "epoch 326  iter 254933  obj(D) 0.8807  obj(G) -0.8801\n",
      "epoch 327  iter 255715  obj(D) 0.4922  obj(G) -0.4226\n",
      "epoch 328  iter 256497  obj(D) 0.6525  obj(G) -0.4383\n",
      "epoch 329  iter 257279  obj(D) 2.6216  obj(G) -0.7626\n",
      "epoch 330  iter 258061  obj(D) 0.3825  obj(G) -0.3680\n",
      "epoch 331  iter 258843  obj(D) 0.4428  obj(G) -0.3111\n",
      "epoch 332  iter 259625  obj(D) 0.4498  obj(G) -0.4146\n",
      "epoch 333  iter 260407  obj(D) 0.5222  obj(G) -0.4709\n",
      "epoch 334  iter 261189  obj(D) 0.4494  obj(G) -0.2247\n",
      "epoch 335  iter 261971  obj(D) 1.9758  obj(G) -0.8110\n",
      "epoch 336  iter 262753  obj(D) 0.5223  obj(G) -0.4472\n",
      "epoch 337  iter 263535  obj(D) 0.2353  obj(G) -0.2109\n",
      "epoch 338  iter 264317  obj(D) 0.1552  obj(G) -0.1483\n",
      "epoch 339  iter 265099  obj(D) 0.0255  obj(G) -0.0202\n",
      "epoch 340  iter 265881  obj(D) 0.0255  obj(G) -0.0072\n",
      "epoch 341  iter 266663  obj(D) 0.0297  obj(G) -0.0197\n",
      "epoch 342  iter 267445  obj(D) 0.0056  obj(G) -0.0037\n",
      "epoch 343  iter 268227  obj(D) 0.0076  obj(G) -0.0039\n",
      "epoch 344  iter 269009  obj(D) 0.0934  obj(G) -0.0863\n",
      "epoch 345  iter 269791  obj(D) 0.3873  obj(G) -0.3853\n",
      "epoch 346  iter 270573  obj(D) 0.0842  obj(G) -0.0700\n",
      "epoch 347  iter 271355  obj(D) 0.0901  obj(G) -0.0868\n",
      "epoch 348  iter 272137  obj(D) 0.0962  obj(G) -0.0880\n",
      "epoch 349  iter 272919  obj(D) 0.0163  obj(G) -0.0122\n",
      "epoch 350  iter 273701  obj(D) 0.0057  obj(G) -0.0042\n",
      "epoch 351  iter 274483  obj(D) 0.0040  obj(G) -0.0012\n",
      "epoch 352  iter 275265  obj(D) 0.0136  obj(G) -0.0018\n",
      "epoch 353  iter 276047  obj(D) 0.0069  obj(G) -0.0038\n",
      "epoch 354  iter 276829  obj(D) 0.0155  obj(G) -0.0127\n",
      "epoch 355  iter 277611  obj(D) 0.0251  obj(G) -0.0016\n",
      "epoch 356  iter 278393  obj(D) 0.0283  obj(G) -0.0254\n",
      "epoch 357  iter 279175  obj(D) 0.0048  obj(G) -0.0028\n",
      "epoch 358  iter 279957  obj(D) 0.0010  obj(G) -0.0001\n",
      "epoch 359  iter 280739  obj(D) 0.0005  obj(G) -0.0002\n",
      "epoch 360  iter 281521  obj(D) 0.0004  obj(G) -0.0003\n",
      "epoch 361  iter 282303  obj(D) 0.0044  obj(G) -0.0041\n",
      "epoch 362  iter 283085  obj(D) 0.0006  obj(G) -0.0003\n",
      "epoch 363  iter 283867  obj(D) 0.0011  obj(G) -0.0001\n",
      "epoch 364  iter 284649  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 365  iter 285431  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 366  iter 286213  obj(D) 0.0003  obj(G) -0.0001\n",
      "epoch 367  iter 286995  obj(D) 0.0005  obj(G) -0.0004\n",
      "epoch 368  iter 287777  obj(D) 0.0006  obj(G) -0.0002\n",
      "epoch 369  iter 288559  obj(D) 0.1523  obj(G) -0.1301\n",
      "epoch 370  iter 289341  obj(D) 0.5577  obj(G) -0.5466\n",
      "epoch 371  iter 290123  obj(D) 1.1581  obj(G) -1.0221\n",
      "epoch 372  iter 290905  obj(D) 0.6772  obj(G) -0.6536\n",
      "epoch 373  iter 291687  obj(D) 0.7946  obj(G) -0.7070\n",
      "epoch 374  iter 292469  obj(D) 0.4687  obj(G) -0.4160\n",
      "epoch 375  iter 293251  obj(D) 0.4220  obj(G) -0.3586\n",
      "epoch 376  iter 294033  obj(D) 0.3121  obj(G) -0.2704\n",
      "epoch 377  iter 294815  obj(D) 0.8628  obj(G) -0.8577\n",
      "epoch 378  iter 295597  obj(D) 0.2784  obj(G) -0.2779\n",
      "epoch 379  iter 296379  obj(D) 2.4359  obj(G) -2.4326\n",
      "epoch 380  iter 297161  obj(D) 1.2722  obj(G) -1.2704\n",
      "epoch 381  iter 297943  obj(D) 0.3108  obj(G) -0.3084\n",
      "epoch 382  iter 298725  obj(D) 1.8238  obj(G) -1.8230\n",
      "epoch 383  iter 299507  obj(D) 0.5617  obj(G) -0.5535\n",
      "epoch 384  iter 300289  obj(D) 1.5920  obj(G) -0.4516\n",
      "epoch 385  iter 301071  obj(D) 0.1483  obj(G) -0.1373\n",
      "epoch 386  iter 301853  obj(D) 0.5414  obj(G) -0.5254\n",
      "epoch 387  iter 302635  obj(D) 0.3231  obj(G) -0.3218\n",
      "epoch 388  iter 303417  obj(D) 0.6857  obj(G) -0.1856\n",
      "epoch 389  iter 304199  obj(D) 0.3037  obj(G) -0.2935\n",
      "epoch 390  iter 304981  obj(D) 0.0663  obj(G) -0.0542\n",
      "epoch 391  iter 305763  obj(D) 0.8963  obj(G) -0.8948\n",
      "epoch 392  iter 306545  obj(D) 0.2148  obj(G) -0.1732\n",
      "epoch 393  iter 307327  obj(D) 0.0673  obj(G) -0.0232\n",
      "epoch 394  iter 308109  obj(D) 0.0459  obj(G) -0.0326\n",
      "epoch 395  iter 308891  obj(D) 0.0024  obj(G) -0.0013\n",
      "epoch 396  iter 309673  obj(D) 0.0170  obj(G) -0.0164\n",
      "epoch 397  iter 310455  obj(D) 0.0299  obj(G) -0.0293\n",
      "epoch 398  iter 311237  obj(D) 0.0016  obj(G) -0.0002\n",
      "epoch 399  iter 312019  obj(D) 0.0105  obj(G) -0.0101\n",
      "epoch 400  iter 312801  obj(D) 0.0044  obj(G) -0.0039\n",
      "epoch 401  iter 313583  obj(D) 0.0078  obj(G) -0.0055\n",
      "epoch 402  iter 314365  obj(D) 1.3552  obj(G) -1.3536\n",
      "epoch 403  iter 315147  obj(D) 0.5804  obj(G) -0.5501\n",
      "epoch 404  iter 315929  obj(D) 1.7408  obj(G) -1.7374\n",
      "epoch 405  iter 316711  obj(D) 0.5338  obj(G) -0.5260\n",
      "epoch 406  iter 317493  obj(D) 0.5977  obj(G) -0.5596\n",
      "epoch 407  iter 318275  obj(D) 0.3819  obj(G) -0.3769\n",
      "epoch 408  iter 319057  obj(D) 0.5638  obj(G) -0.5604\n",
      "epoch 409  iter 319839  obj(D) 0.5859  obj(G) -0.5842\n",
      "epoch 410  iter 320621  obj(D) 0.0828  obj(G) -0.0388\n",
      "epoch 411  iter 321403  obj(D) 0.2116  obj(G) -0.1773\n",
      "epoch 412  iter 322185  obj(D) 1.3486  obj(G) -0.7414\n",
      "epoch 413  iter 322967  obj(D) 0.4668  obj(G) -0.4281\n",
      "epoch 414  iter 323749  obj(D) 0.4157  obj(G) -0.2775\n",
      "epoch 415  iter 324531  obj(D) 0.8022  obj(G) -0.2134\n",
      "epoch 416  iter 325313  obj(D) 0.3677  obj(G) -0.3657\n",
      "epoch 417  iter 326095  obj(D) 0.4125  obj(G) -0.3923\n",
      "epoch 418  iter 326877  obj(D) 0.0741  obj(G) -0.0689\n",
      "epoch 419  iter 327659  obj(D) 0.1393  obj(G) -0.1217\n",
      "epoch 420  iter 328441  obj(D) 0.0446  obj(G) -0.0439\n",
      "epoch 421  iter 329223  obj(D) 0.0077  obj(G) -0.0064\n",
      "epoch 422  iter 330005  obj(D) 0.7016  obj(G) -0.7005\n",
      "epoch 423  iter 330787  obj(D) 1.0628  obj(G) -1.0609\n",
      "epoch 424  iter 331569  obj(D) 0.1859  obj(G) -0.1831\n",
      "epoch 425  iter 332351  obj(D) 0.6068  obj(G) -0.6000\n",
      "epoch 426  iter 333133  obj(D) 0.1748  obj(G) -0.1727\n",
      "epoch 427  iter 333915  obj(D) 1.1577  obj(G) -1.1260\n",
      "epoch 428  iter 334697  obj(D) 0.4915  obj(G) -0.4821\n",
      "epoch 429  iter 335479  obj(D) 0.2306  obj(G) -0.2113\n",
      "epoch 430  iter 336261  obj(D) 0.3192  obj(G) -0.3036\n",
      "epoch 431  iter 337043  obj(D) 0.2635  obj(G) -0.2598\n",
      "epoch 432  iter 337825  obj(D) 0.8905  obj(G) -0.7470\n",
      "epoch 433  iter 338607  obj(D) 0.5188  obj(G) -0.5165\n",
      "epoch 434  iter 339389  obj(D) 0.0669  obj(G) -0.0646\n",
      "epoch 435  iter 340171  obj(D) 0.4529  obj(G) -0.4328\n",
      "epoch 436  iter 340953  obj(D) 0.1230  obj(G) -0.1214\n",
      "epoch 437  iter 341735  obj(D) 0.8132  obj(G) -0.8118\n",
      "epoch 438  iter 342517  obj(D) 0.7496  obj(G) -0.7457\n",
      "epoch 439  iter 343299  obj(D) 0.6632  obj(G) -0.5370\n",
      "epoch 440  iter 344081  obj(D) 0.3466  obj(G) -0.2590\n",
      "epoch 441  iter 344863  obj(D) 0.0918  obj(G) -0.0882\n",
      "epoch 442  iter 345645  obj(D) 0.4534  obj(G) -0.3619\n",
      "epoch 443  iter 346427  obj(D) 0.1302  obj(G) -0.1201\n",
      "epoch 444  iter 347209  obj(D) 0.0372  obj(G) -0.0329\n",
      "epoch 445  iter 347991  obj(D) 0.0097  obj(G) -0.0043\n",
      "epoch 446  iter 348773  obj(D) 0.0090  obj(G) -0.0066\n",
      "epoch 447  iter 349555  obj(D) 0.0075  obj(G) -0.0045\n",
      "epoch 448  iter 350337  obj(D) 0.0043  obj(G) -0.0036\n",
      "epoch 449  iter 351119  obj(D) 0.0034  obj(G) -0.0029\n",
      "epoch 450  iter 351901  obj(D) 2.1262  obj(G) -0.4248\n",
      "epoch 451  iter 352683  obj(D) 0.4031  obj(G) -0.3487\n",
      "epoch 452  iter 353465  obj(D) 0.8765  obj(G) -0.7953\n",
      "epoch 453  iter 354247  obj(D) 0.6045  obj(G) -0.6032\n",
      "epoch 454  iter 355029  obj(D) 1.1479  obj(G) -0.7218\n",
      "epoch 455  iter 355811  obj(D) 0.2407  obj(G) -0.1789\n",
      "epoch 456  iter 356593  obj(D) 0.0419  obj(G) -0.0361\n",
      "epoch 457  iter 357375  obj(D) 0.7166  obj(G) -0.6822\n",
      "epoch 458  iter 358157  obj(D) 0.6122  obj(G) -0.5576\n",
      "epoch 459  iter 358939  obj(D) 0.5510  obj(G) -0.5491\n",
      "epoch 460  iter 359721  obj(D) 1.0416  obj(G) -0.8282\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 461  iter 360503  obj(D) 0.5504  obj(G) -0.4480\n",
      "epoch 462  iter 361285  obj(D) 0.5996  obj(G) -0.5489\n",
      "epoch 463  iter 362067  obj(D) 2.1543  obj(G) -2.1239\n",
      "epoch 464  iter 362849  obj(D) 0.4378  obj(G) -0.1347\n",
      "epoch 465  iter 363631  obj(D) 0.4703  obj(G) -0.4684\n",
      "epoch 466  iter 364413  obj(D) 0.5344  obj(G) -0.0708\n",
      "epoch 467  iter 365195  obj(D) 0.6727  obj(G) -0.3965\n",
      "epoch 468  iter 365977  obj(D) 0.5988  obj(G) -0.5318\n",
      "epoch 469  iter 366759  obj(D) 0.6866  obj(G) -0.3885\n",
      "epoch 470  iter 367541  obj(D) 0.6367  obj(G) -0.6345\n",
      "epoch 471  iter 368323  obj(D) 0.1515  obj(G) -0.1446\n",
      "epoch 472  iter 369105  obj(D) 0.7337  obj(G) -0.7254\n",
      "epoch 473  iter 369887  obj(D) 0.0533  obj(G) -0.0480\n",
      "epoch 474  iter 370669  obj(D) 0.1121  obj(G) -0.1074\n",
      "epoch 475  iter 371451  obj(D) 0.3157  obj(G) -0.0502\n",
      "epoch 476  iter 372233  obj(D) 0.1043  obj(G) -0.1010\n",
      "epoch 477  iter 373015  obj(D) 0.0571  obj(G) -0.0391\n",
      "epoch 478  iter 373797  obj(D) 0.0036  obj(G) -0.0016\n",
      "epoch 479  iter 374579  obj(D) 0.0116  obj(G) -0.0098\n",
      "epoch 480  iter 375361  obj(D) 0.0097  obj(G) -0.0085\n",
      "epoch 481  iter 376143  obj(D) 0.0080  obj(G) -0.0037\n",
      "epoch 482  iter 376925  obj(D) 5.0114  obj(G) -5.0068\n",
      "epoch 483  iter 377707  obj(D) 0.2147  obj(G) -0.2058\n",
      "epoch 484  iter 378489  obj(D) 1.6310  obj(G) -1.6169\n",
      "epoch 485  iter 379271  obj(D) 0.7205  obj(G) -0.6306\n",
      "epoch 486  iter 380053  obj(D) 1.1044  obj(G) -1.0887\n",
      "epoch 487  iter 380835  obj(D) 1.0507  obj(G) -0.7601\n",
      "epoch 488  iter 381617  obj(D) 0.8641  obj(G) -0.8466\n",
      "epoch 489  iter 382399  obj(D) 0.1954  obj(G) -0.1890\n",
      "epoch 490  iter 383181  obj(D) 1.2998  obj(G) -1.2893\n",
      "epoch 491  iter 383963  obj(D) 0.4794  obj(G) -0.4185\n",
      "epoch 492  iter 384745  obj(D) 0.8010  obj(G) -0.7903\n",
      "epoch 493  iter 385527  obj(D) 0.3817  obj(G) -0.2916\n",
      "epoch 494  iter 386309  obj(D) 1.0074  obj(G) -1.0067\n",
      "epoch 495  iter 387091  obj(D) 0.0789  obj(G) -0.0774\n",
      "epoch 496  iter 387873  obj(D) 0.0031  obj(G) -0.0016\n",
      "epoch 497  iter 388655  obj(D) 0.0243  obj(G) -0.0216\n",
      "epoch 498  iter 389437  obj(D) 0.0329  obj(G) -0.0280\n",
      "epoch 499  iter 390219  obj(D) 0.0423  obj(G) -0.0415\n",
      "epoch 500  iter 391001  obj(D) 0.1880  obj(G) -0.1834\n",
      "epoch 501  iter 391783  obj(D) 0.2372  obj(G) -0.2303\n",
      "epoch 502  iter 392565  obj(D) 0.4100  obj(G) -0.3977\n",
      "epoch 503  iter 393347  obj(D) 0.6644  obj(G) -0.6435\n",
      "epoch 504  iter 394129  obj(D) 0.2724  obj(G) -0.2301\n",
      "epoch 505  iter 394911  obj(D) 0.5765  obj(G) -0.5531\n",
      "epoch 506  iter 395693  obj(D) 1.4955  obj(G) -1.4788\n",
      "epoch 507  iter 396475  obj(D) 3.2723  obj(G) -3.2691\n",
      "epoch 508  iter 397257  obj(D) 0.3241  obj(G) -0.3162\n",
      "epoch 509  iter 398039  obj(D) 0.4218  obj(G) -0.4173\n",
      "epoch 510  iter 398821  obj(D) 1.1972  obj(G) -1.1963\n",
      "epoch 511  iter 399603  obj(D) 1.9396  obj(G) -0.8819\n",
      "epoch 512  iter 400385  obj(D) 0.3105  obj(G) -0.2795\n",
      "epoch 513  iter 401167  obj(D) 1.0322  obj(G) -0.7680\n",
      "epoch 514  iter 401949  obj(D) 1.9291  obj(G) -1.9267\n",
      "epoch 515  iter 402731  obj(D) 0.5102  obj(G) -0.4995\n",
      "epoch 516  iter 403513  obj(D) 0.9842  obj(G) -0.9071\n",
      "epoch 517  iter 404295  obj(D) 0.4398  obj(G) -0.4027\n",
      "epoch 518  iter 405077  obj(D) 0.2137  obj(G) -0.2074\n",
      "epoch 519  iter 405859  obj(D) 0.0547  obj(G) -0.0273\n",
      "epoch 520  iter 406641  obj(D) 0.0455  obj(G) -0.0413\n",
      "epoch 521  iter 407423  obj(D) 0.1507  obj(G) -0.1430\n",
      "epoch 522  iter 408205  obj(D) 0.0092  obj(G) -0.0075\n",
      "epoch 523  iter 408987  obj(D) 0.0044  obj(G) -0.0010\n",
      "epoch 524  iter 409769  obj(D) 0.0076  obj(G) -0.0069\n",
      "epoch 525  iter 410551  obj(D) 0.1897  obj(G) -0.1856\n",
      "epoch 526  iter 411333  obj(D) 1.3047  obj(G) -1.2153\n",
      "epoch 527  iter 412115  obj(D) 4.2647  obj(G) -4.2644\n",
      "epoch 528  iter 412897  obj(D) 0.2219  obj(G) -0.0983\n",
      "epoch 529  iter 413679  obj(D) 0.0897  obj(G) -0.0425\n",
      "epoch 530  iter 414461  obj(D) 2.5232  obj(G) -1.2259\n",
      "epoch 531  iter 415243  obj(D) 0.3282  obj(G) -0.2792\n",
      "epoch 532  iter 416025  obj(D) 0.9950  obj(G) -0.9932\n",
      "epoch 533  iter 416807  obj(D) 0.4262  obj(G) -0.3311\n",
      "epoch 534  iter 417589  obj(D) 0.2323  obj(G) -0.1972\n",
      "epoch 535  iter 418371  obj(D) 0.5843  obj(G) -0.2656\n",
      "epoch 536  iter 419153  obj(D) 1.1654  obj(G) -1.1537\n",
      "epoch 537  iter 419935  obj(D) 0.1773  obj(G) -0.1508\n",
      "epoch 538  iter 420717  obj(D) 0.5555  obj(G) -0.5531\n",
      "epoch 539  iter 421499  obj(D) 0.0751  obj(G) -0.0445\n",
      "epoch 540  iter 422281  obj(D) 0.0623  obj(G) -0.0171\n",
      "epoch 541  iter 423063  obj(D) 0.0382  obj(G) -0.0362\n",
      "epoch 542  iter 423845  obj(D) 0.0704  obj(G) -0.0690\n",
      "epoch 543  iter 424627  obj(D) 0.0077  obj(G) -0.0064\n",
      "epoch 544  iter 425409  obj(D) 0.0028  obj(G) -0.0015\n",
      "epoch 545  iter 426191  obj(D) 0.0035  obj(G) -0.0030\n",
      "epoch 546  iter 426973  obj(D) 0.0039  obj(G) -0.0024\n",
      "epoch 547  iter 427755  obj(D) 0.0051  obj(G) -0.0016\n",
      "epoch 548  iter 428537  obj(D) 2.7540  obj(G) -2.7228\n",
      "epoch 549  iter 429319  obj(D) 0.5411  obj(G) -0.5185\n",
      "epoch 550  iter 430101  obj(D) 0.5398  obj(G) -0.5375\n",
      "epoch 551  iter 430883  obj(D) 3.0648  obj(G) -3.0636\n",
      "epoch 552  iter 431665  obj(D) 1.0479  obj(G) -0.9530\n",
      "epoch 553  iter 432447  obj(D) 0.2486  obj(G) -0.2266\n",
      "epoch 554  iter 433229  obj(D) 0.9235  obj(G) -0.9160\n",
      "epoch 555  iter 434011  obj(D) 0.0727  obj(G) -0.0706\n",
      "epoch 556  iter 434793  obj(D) 3.8977  obj(G) -3.8887\n",
      "epoch 557  iter 435575  obj(D) 0.4848  obj(G) -0.4326\n",
      "epoch 558  iter 436357  obj(D) 0.4723  obj(G) -0.4669\n",
      "epoch 559  iter 437139  obj(D) 0.2811  obj(G) -0.2576\n",
      "epoch 560  iter 437921  obj(D) 0.3570  obj(G) -0.3540\n",
      "epoch 561  iter 438703  obj(D) 0.3802  obj(G) -0.3508\n",
      "epoch 562  iter 439485  obj(D) 1.1428  obj(G) -0.8342\n",
      "epoch 563  iter 440267  obj(D) 0.1458  obj(G) -0.1401\n",
      "epoch 564  iter 441049  obj(D) 0.0404  obj(G) -0.0393\n",
      "epoch 565  iter 441831  obj(D) 1.4108  obj(G) -1.4086\n",
      "epoch 566  iter 442613  obj(D) 2.8676  obj(G) -2.8568\n",
      "epoch 567  iter 443395  obj(D) 1.1155  obj(G) -1.1017\n",
      "epoch 568  iter 444177  obj(D) 1.2116  obj(G) -1.1953\n",
      "epoch 569  iter 444959  obj(D) 0.4938  obj(G) -0.4864\n",
      "epoch 570  iter 445741  obj(D) 0.0276  obj(G) -0.0242\n",
      "epoch 571  iter 446523  obj(D) 1.9389  obj(G) -1.0639\n",
      "epoch 572  iter 447305  obj(D) 3.0163  obj(G) -3.0157\n",
      "epoch 573  iter 448087  obj(D) 2.8232  obj(G) -2.8126\n",
      "epoch 574  iter 448869  obj(D) 1.4153  obj(G) -1.4100\n",
      "epoch 575  iter 449651  obj(D) 0.5075  obj(G) -0.5035\n",
      "epoch 576  iter 450433  obj(D) 0.2235  obj(G) -0.2153\n",
      "epoch 577  iter 451215  obj(D) 0.5281  obj(G) -0.4740\n",
      "epoch 578  iter 451997  obj(D) 0.8499  obj(G) -0.6691\n",
      "epoch 579  iter 452779  obj(D) 0.2327  obj(G) -0.2143\n",
      "epoch 580  iter 453561  obj(D) 0.4856  obj(G) -0.4697\n",
      "epoch 581  iter 454343  obj(D) 0.3262  obj(G) -0.3184\n",
      "epoch 582  iter 455125  obj(D) 1.6747  obj(G) -1.6682\n",
      "epoch 583  iter 455907  obj(D) 0.6344  obj(G) -0.5985\n",
      "epoch 584  iter 456689  obj(D) 0.7732  obj(G) -0.7686\n",
      "epoch 585  iter 457471  obj(D) 0.2449  obj(G) -0.2237\n",
      "epoch 586  iter 458253  obj(D) 0.3733  obj(G) -0.3662\n",
      "epoch 587  iter 459035  obj(D) 0.5399  obj(G) -0.5326\n",
      "epoch 588  iter 459817  obj(D) 0.5306  obj(G) -0.5165\n",
      "epoch 589  iter 460599  obj(D) 0.7621  obj(G) -0.7592\n",
      "epoch 590  iter 461381  obj(D) 0.2062  obj(G) -0.1987\n",
      "epoch 591  iter 462163  obj(D) 0.6247  obj(G) -0.3367\n",
      "epoch 592  iter 462945  obj(D) 0.3525  obj(G) -0.3508\n",
      "epoch 593  iter 463727  obj(D) 0.1172  obj(G) -0.1085\n",
      "epoch 594  iter 464509  obj(D) 0.0552  obj(G) -0.0488\n",
      "epoch 595  iter 465291  obj(D) 0.0482  obj(G) -0.0432\n",
      "epoch 596  iter 466073  obj(D) 0.2569  obj(G) -0.0040\n",
      "epoch 597  iter 466855  obj(D) 0.0071  obj(G) -0.0068\n",
      "epoch 598  iter 467637  obj(D) 0.0072  obj(G) -0.0037\n",
      "epoch 599  iter 468419  obj(D) 0.0007  obj(G) -0.0001\n",
      "epoch 600  iter 469201  obj(D) 0.0453  obj(G) -0.0093\n",
      "epoch 601  iter 469983  obj(D) 0.0127  obj(G) -0.0116\n",
      "epoch 602  iter 470765  obj(D) 0.0093  obj(G) -0.0069\n",
      "epoch 603  iter 471547  obj(D) 0.0071  obj(G) -0.0065\n",
      "epoch 604  iter 472329  obj(D) 0.0109  obj(G) -0.0098\n",
      "epoch 605  iter 473111  obj(D) 0.0006  obj(G) -0.0003\n",
      "epoch 606  iter 473893  obj(D) 0.0015  obj(G) -0.0013\n",
      "epoch 607  iter 474675  obj(D) 0.0099  obj(G) -0.0068\n",
      "epoch 608  iter 475457  obj(D) 0.0031  obj(G) -0.0029\n",
      "epoch 609  iter 476239  obj(D) 0.0024  obj(G) -0.0023\n",
      "epoch 610  iter 477021  obj(D) 0.0024  obj(G) -0.0017\n",
      "epoch 611  iter 477803  obj(D) 0.0030  obj(G) -0.0029\n",
      "epoch 612  iter 478585  obj(D) 0.0001  obj(G) -0.0000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 613  iter 479367  obj(D) 0.5515  obj(G) -0.5506\n",
      "epoch 614  iter 480149  obj(D) 0.0984  obj(G) -0.0981\n",
      "epoch 615  iter 480931  obj(D) 0.0560  obj(G) -0.0557\n",
      "epoch 616  iter 481713  obj(D) 0.0520  obj(G) -0.0497\n",
      "epoch 617  iter 482495  obj(D) 0.0073  obj(G) -0.0065\n",
      "epoch 618  iter 483277  obj(D) 0.0078  obj(G) -0.0068\n",
      "epoch 619  iter 484059  obj(D) 0.0013  obj(G) -0.0001\n",
      "epoch 620  iter 484841  obj(D) 0.0039  obj(G) -0.0034\n",
      "epoch 621  iter 485623  obj(D) 0.0005  obj(G) -0.0002\n",
      "epoch 622  iter 486405  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 623  iter 487187  obj(D) 0.0019  obj(G) -0.0017\n",
      "epoch 624  iter 487969  obj(D) 0.0010  obj(G) -0.0009\n",
      "epoch 625  iter 488751  obj(D) 0.0017  obj(G) -0.0007\n",
      "epoch 626  iter 489533  obj(D) 0.0263  obj(G) -0.0247\n",
      "epoch 627  iter 490315  obj(D) 0.0031  obj(G) -0.0007\n",
      "epoch 628  iter 491097  obj(D) 0.0013  obj(G) -0.0012\n",
      "epoch 629  iter 491879  obj(D) 0.0352  obj(G) -0.0116\n",
      "epoch 630  iter 492661  obj(D) 0.0801  obj(G) -0.0764\n",
      "epoch 631  iter 493443  obj(D) 0.0038  obj(G) -0.0035\n",
      "epoch 632  iter 494225  obj(D) 0.0024  obj(G) -0.0004\n",
      "epoch 633  iter 495007  obj(D) 0.0005  obj(G) -0.0004\n",
      "epoch 634  iter 495789  obj(D) 0.0068  obj(G) -0.0056\n",
      "epoch 635  iter 496571  obj(D) 0.0037  obj(G) -0.0021\n",
      "epoch 636  iter 497353  obj(D) 0.2349  obj(G) -0.2341\n",
      "epoch 637  iter 498135  obj(D) 0.0141  obj(G) -0.0092\n",
      "epoch 638  iter 498917  obj(D) 0.2361  obj(G) -0.2271\n",
      "epoch 639  iter 499699  obj(D) 0.0124  obj(G) -0.0115\n",
      "epoch 640  iter 500481  obj(D) 0.0026  obj(G) -0.0015\n",
      "epoch 641  iter 501263  obj(D) 0.0056  obj(G) -0.0051\n",
      "epoch 642  iter 502045  obj(D) 0.0026  obj(G) -0.0024\n",
      "epoch 643  iter 502827  obj(D) 0.0178  obj(G) -0.0171\n",
      "epoch 644  iter 503609  obj(D) 0.0012  obj(G) -0.0007\n",
      "epoch 645  iter 504391  obj(D) 0.0030  obj(G) -0.0003\n",
      "epoch 646  iter 505173  obj(D) 0.0011  obj(G) -0.0007\n",
      "epoch 647  iter 505955  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 648  iter 506737  obj(D) 0.0010  obj(G) -0.0008\n",
      "epoch 649  iter 507519  obj(D) 0.0004  obj(G) -0.0001\n",
      "epoch 650  iter 508301  obj(D) 0.0003  obj(G) -0.0001\n",
      "epoch 651  iter 509083  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 652  iter 509865  obj(D) 0.0006  obj(G) -0.0005\n",
      "epoch 653  iter 510647  obj(D) 0.0058  obj(G) -0.0051\n",
      "epoch 654  iter 511429  obj(D) 0.0004  obj(G) -0.0001\n",
      "epoch 655  iter 512211  obj(D) 0.0029  obj(G) -0.0009\n",
      "epoch 656  iter 512993  obj(D) 0.0013  obj(G) -0.0007\n",
      "epoch 657  iter 513775  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 658  iter 514557  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 659  iter 515339  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 660  iter 516121  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 661  iter 516903  obj(D) 0.0003  obj(G) -0.0003\n",
      "epoch 662  iter 517685  obj(D) 0.0453  obj(G) -0.0426\n",
      "epoch 663  iter 518467  obj(D) 0.0517  obj(G) -0.0511\n",
      "epoch 664  iter 519249  obj(D) 0.0074  obj(G) -0.0058\n",
      "epoch 665  iter 520031  obj(D) 0.0164  obj(G) -0.0159\n",
      "epoch 666  iter 520813  obj(D) 0.0019  obj(G) -0.0016\n",
      "epoch 667  iter 521595  obj(D) 0.0008  obj(G) -0.0006\n",
      "epoch 668  iter 522377  obj(D) 0.0009  obj(G) -0.0006\n",
      "epoch 669  iter 523159  obj(D) 0.0005  obj(G) -0.0003\n",
      "epoch 670  iter 523941  obj(D) 0.0005  obj(G) -0.0004\n",
      "epoch 671  iter 524723  obj(D) 0.0007  obj(G) -0.0006\n",
      "epoch 672  iter 525505  obj(D) 0.0041  obj(G) -0.0040\n",
      "epoch 673  iter 526287  obj(D) 0.0099  obj(G) -0.0028\n",
      "epoch 674  iter 527069  obj(D) 0.0009  obj(G) -0.0008\n",
      "epoch 675  iter 527851  obj(D) 0.0007  obj(G) -0.0002\n",
      "epoch 676  iter 528633  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 677  iter 529415  obj(D) 0.0006  obj(G) -0.0001\n",
      "epoch 678  iter 530197  obj(D) 0.0002  obj(G) -0.0000\n",
      "epoch 679  iter 530979  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 680  iter 531761  obj(D) 0.0009  obj(G) -0.0007\n",
      "epoch 681  iter 532543  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 682  iter 533325  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 683  iter 534107  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 684  iter 534889  obj(D) 0.0013  obj(G) -0.0013\n",
      "epoch 685  iter 535671  obj(D) 0.0580  obj(G) -0.0554\n",
      "epoch 686  iter 536453  obj(D) 0.0944  obj(G) -0.0915\n",
      "epoch 687  iter 537235  obj(D) 0.0038  obj(G) -0.0004\n",
      "epoch 688  iter 538017  obj(D) 0.0017  obj(G) -0.0007\n",
      "epoch 689  iter 538799  obj(D) 0.0007  obj(G) -0.0002\n",
      "epoch 690  iter 539581  obj(D) 0.0019  obj(G) -0.0013\n",
      "epoch 691  iter 540363  obj(D) 0.0032  obj(G) -0.0022\n",
      "epoch 692  iter 541145  obj(D) 0.0023  obj(G) -0.0016\n",
      "epoch 693  iter 541927  obj(D) 0.0013  obj(G) -0.0012\n",
      "epoch 694  iter 542709  obj(D) 1.0885  obj(G) -0.6234\n",
      "epoch 695  iter 543491  obj(D) 1.0233  obj(G) -0.8282\n",
      "epoch 696  iter 544273  obj(D) 0.4445  obj(G) -0.4327\n",
      "epoch 697  iter 545055  obj(D) 3.6476  obj(G) -1.8221\n",
      "epoch 698  iter 545837  obj(D) 0.2565  obj(G) -0.2496\n",
      "epoch 699  iter 546619  obj(D) 1.3222  obj(G) -1.3160\n",
      "epoch 700  iter 547401  obj(D) 0.1082  obj(G) -0.0834\n",
      "epoch 701  iter 548183  obj(D) 0.0368  obj(G) -0.0336\n",
      "epoch 702  iter 548965  obj(D) 0.0208  obj(G) -0.0187\n",
      "epoch 703  iter 549747  obj(D) 0.0015  obj(G) -0.0006\n",
      "epoch 704  iter 550529  obj(D) 0.0014  obj(G) -0.0003\n",
      "epoch 705  iter 551311  obj(D) 0.0091  obj(G) -0.0005\n",
      "epoch 706  iter 552093  obj(D) 0.0009  obj(G) -0.0004\n",
      "epoch 707  iter 552875  obj(D) 0.0002  obj(G) -0.0000\n",
      "epoch 708  iter 553657  obj(D) 0.0004  obj(G) -0.0003\n",
      "epoch 709  iter 554439  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 710  iter 555221  obj(D) 0.0011  obj(G) -0.0009\n",
      "epoch 711  iter 556003  obj(D) 0.0008  obj(G) -0.0007\n",
      "epoch 712  iter 556785  obj(D) 0.2393  obj(G) -0.2142\n",
      "epoch 713  iter 557567  obj(D) 0.0395  obj(G) -0.0361\n",
      "epoch 714  iter 558349  obj(D) 0.2310  obj(G) -0.2309\n",
      "epoch 715  iter 559131  obj(D) 0.0258  obj(G) -0.0180\n",
      "epoch 716  iter 559913  obj(D) 0.0189  obj(G) -0.0151\n",
      "epoch 717  iter 560695  obj(D) 0.0600  obj(G) -0.0580\n",
      "epoch 718  iter 561477  obj(D) 0.8989  obj(G) -0.8488\n",
      "epoch 719  iter 562259  obj(D) 5.2561  obj(G) -5.2560\n",
      "epoch 720  iter 563041  obj(D) 0.1773  obj(G) -0.1617\n",
      "epoch 721  iter 563823  obj(D) 0.1535  obj(G) -0.1465\n",
      "epoch 722  iter 564605  obj(D) 0.1404  obj(G) -0.1347\n",
      "epoch 723  iter 565387  obj(D) 1.9461  obj(G) -1.9457\n",
      "epoch 724  iter 566169  obj(D) 0.0383  obj(G) -0.0363\n",
      "epoch 725  iter 566951  obj(D) 0.6248  obj(G) -0.6243\n",
      "epoch 726  iter 567733  obj(D) 0.5930  obj(G) -0.5927\n",
      "epoch 727  iter 568515  obj(D) 0.0394  obj(G) -0.0344\n",
      "epoch 728  iter 569297  obj(D) 0.9758  obj(G) -0.9745\n",
      "epoch 729  iter 570079  obj(D) 0.2531  obj(G) -0.2136\n",
      "epoch 730  iter 570861  obj(D) 0.3095  obj(G) -0.2385\n",
      "epoch 731  iter 571643  obj(D) 0.0674  obj(G) -0.0547\n",
      "epoch 732  iter 572425  obj(D) 0.0220  obj(G) -0.0135\n",
      "epoch 733  iter 573207  obj(D) 0.0054  obj(G) -0.0048\n",
      "epoch 734  iter 573989  obj(D) 0.0024  obj(G) -0.0006\n",
      "epoch 735  iter 574771  obj(D) 0.0019  obj(G) -0.0010\n",
      "epoch 736  iter 575553  obj(D) 0.0013  obj(G) -0.0008\n",
      "epoch 737  iter 576335  obj(D) 0.0047  obj(G) -0.0043\n",
      "epoch 738  iter 577117  obj(D) 0.0002  obj(G) -0.0000\n",
      "epoch 739  iter 577899  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 740  iter 578681  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 741  iter 579463  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 742  iter 580245  obj(D) 0.0005  obj(G) -0.0003\n",
      "epoch 743  iter 581027  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 744  iter 581809  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 745  iter 582591  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 746  iter 583373  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 747  iter 584155  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 748  iter 584937  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 749  iter 585719  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 750  iter 586501  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 751  iter 587283  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 752  iter 588065  obj(D) 0.0105  obj(G) -0.0083\n",
      "epoch 753  iter 588847  obj(D) 0.1927  obj(G) -0.1919\n",
      "epoch 754  iter 589629  obj(D) 0.0826  obj(G) -0.0820\n",
      "epoch 755  iter 590411  obj(D) 0.5108  obj(G) -0.4986\n",
      "epoch 756  iter 591193  obj(D) 0.0217  obj(G) -0.0192\n",
      "epoch 757  iter 591975  obj(D) 0.1028  obj(G) -0.1005\n",
      "epoch 758  iter 592757  obj(D) 0.0083  obj(G) -0.0068\n",
      "epoch 759  iter 593539  obj(D) 0.0092  obj(G) -0.0070\n",
      "epoch 760  iter 594321  obj(D) 0.0013  obj(G) -0.0008\n",
      "epoch 761  iter 595103  obj(D) 0.0048  obj(G) -0.0041\n",
      "epoch 762  iter 595885  obj(D) 0.0007  obj(G) -0.0002\n",
      "epoch 763  iter 596667  obj(D) 0.0009  obj(G) -0.0003\n",
      "epoch 764  iter 597449  obj(D) 0.0036  obj(G) -0.0034\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 765  iter 598231  obj(D) 0.0008  obj(G) -0.0007\n",
      "epoch 766  iter 599013  obj(D) 0.0009  obj(G) -0.0007\n",
      "epoch 767  iter 599795  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 768  iter 600577  obj(D) 0.0004  obj(G) -0.0000\n",
      "epoch 769  iter 601359  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 770  iter 602141  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 771  iter 602923  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 772  iter 603705  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 773  iter 604487  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 774  iter 605269  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 775  iter 606051  obj(D) 0.0005  obj(G) -0.0005\n",
      "epoch 776  iter 606833  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 777  iter 607615  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 778  iter 608397  obj(D) 0.0006  obj(G) -0.0006\n",
      "epoch 779  iter 609179  obj(D) 0.0003  obj(G) -0.0003\n",
      "epoch 780  iter 609961  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 781  iter 610743  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 782  iter 611525  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 783  iter 612307  obj(D) 0.0381  obj(G) -0.0381\n",
      "epoch 784  iter 613089  obj(D) 0.0047  obj(G) -0.0046\n",
      "epoch 785  iter 613871  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 786  iter 614653  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 787  iter 615435  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 788  iter 616217  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 789  iter 616999  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 790  iter 617781  obj(D) 0.0036  obj(G) -0.0036\n",
      "epoch 791  iter 618563  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 792  iter 619345  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 793  iter 620127  obj(D) 0.0002  obj(G) -0.0002\n",
      "epoch 794  iter 620909  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 795  iter 621691  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 796  iter 622473  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 797  iter 623255  obj(D) 0.0002  obj(G) -0.0002\n",
      "epoch 798  iter 624037  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 799  iter 624819  obj(D) 0.0010  obj(G) -0.0010\n",
      "epoch 800  iter 625601  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 801  iter 626383  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 802  iter 627165  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 803  iter 627947  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 804  iter 628729  obj(D) 0.0003  obj(G) -0.0001\n",
      "epoch 805  iter 629511  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 806  iter 630293  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 807  iter 631075  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 808  iter 631857  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 809  iter 632639  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 810  iter 633421  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 811  iter 634203  obj(D) 0.0020  obj(G) -0.0020\n",
      "epoch 812  iter 634985  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 813  iter 635767  obj(D) 0.0003  obj(G) -0.0003\n",
      "epoch 814  iter 636549  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 815  iter 637331  obj(D) 0.0021  obj(G) -0.0021\n",
      "epoch 816  iter 638113  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 817  iter 638895  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 818  iter 639677  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 819  iter 640459  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 820  iter 641241  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 821  iter 642023  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 822  iter 642805  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 823  iter 643587  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 824  iter 644369  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 825  iter 645151  obj(D) 0.0016  obj(G) -0.0008\n",
      "epoch 826  iter 645933  obj(D) 0.0075  obj(G) -0.0064\n",
      "epoch 827  iter 646715  obj(D) 0.0008  obj(G) -0.0007\n",
      "epoch 828  iter 647497  obj(D) 0.0005  obj(G) -0.0004\n",
      "epoch 829  iter 648279  obj(D) 0.0007  obj(G) -0.0001\n",
      "epoch 830  iter 649061  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 831  iter 649843  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 832  iter 650625  obj(D) 0.0002  obj(G) -0.0002\n",
      "epoch 833  iter 651407  obj(D) 0.0013  obj(G) -0.0010\n",
      "epoch 834  iter 652189  obj(D) 0.0003  obj(G) -0.0003\n",
      "epoch 835  iter 652971  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 836  iter 653753  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 837  iter 654535  obj(D) 0.0005  obj(G) -0.0001\n",
      "epoch 838  iter 655317  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 839  iter 656099  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 840  iter 656881  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 841  iter 657663  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 842  iter 658445  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 843  iter 659227  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 844  iter 660009  obj(D) 0.0007  obj(G) -0.0000\n",
      "epoch 845  iter 660791  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 846  iter 661573  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 847  iter 662355  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 848  iter 663137  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 849  iter 663919  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 850  iter 664701  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 851  iter 665483  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 852  iter 666265  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 853  iter 667047  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 854  iter 667829  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 855  iter 668611  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 856  iter 669393  obj(D) 0.0002  obj(G) -0.0002\n",
      "epoch 857  iter 670175  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 858  iter 670957  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 859  iter 671739  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 860  iter 672521  obj(D) 0.0003  obj(G) -0.0003\n",
      "epoch 861  iter 673303  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 862  iter 674085  obj(D) 0.0012  obj(G) -0.0011\n",
      "epoch 863  iter 674867  obj(D) 0.3176  obj(G) -0.2851\n",
      "epoch 864  iter 675649  obj(D) 0.1301  obj(G) -0.1160\n",
      "epoch 865  iter 676431  obj(D) 0.0322  obj(G) -0.0302\n",
      "epoch 866  iter 677213  obj(D) 1.5391  obj(G) -1.3472\n",
      "epoch 867  iter 677995  obj(D) 0.5714  obj(G) -0.5696\n",
      "epoch 868  iter 678777  obj(D) 0.1470  obj(G) -0.0654\n",
      "epoch 869  iter 679559  obj(D) 0.0346  obj(G) -0.0340\n",
      "epoch 870  iter 680341  obj(D) 0.0292  obj(G) -0.0231\n",
      "epoch 871  iter 681123  obj(D) 0.0298  obj(G) -0.0275\n",
      "epoch 872  iter 681905  obj(D) 0.0018  obj(G) -0.0001\n",
      "epoch 873  iter 682687  obj(D) 0.0002  obj(G) -0.0000\n",
      "epoch 874  iter 683469  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 875  iter 684251  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 876  iter 685033  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 877  iter 685815  obj(D) 0.0002  obj(G) -0.0002\n",
      "epoch 878  iter 686597  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 879  iter 687379  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 880  iter 688161  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 881  iter 688943  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 882  iter 689725  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 883  iter 690507  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 884  iter 691289  obj(D) 0.0027  obj(G) -0.0026\n",
      "epoch 885  iter 692071  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 886  iter 692853  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 887  iter 693635  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 888  iter 694417  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 889  iter 695199  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 890  iter 695981  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 891  iter 696763  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 892  iter 697545  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 893  iter 698327  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 894  iter 699109  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 895  iter 699891  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 896  iter 700673  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 897  iter 701455  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 898  iter 702237  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 899  iter 703019  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 900  iter 703801  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 901  iter 704583  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 902  iter 705365  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 903  iter 706147  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 904  iter 706929  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 905  iter 707711  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 906  iter 708493  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 907  iter 709275  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 908  iter 710057  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 909  iter 710839  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 910  iter 711621  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 911  iter 712403  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 912  iter 713185  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 913  iter 713967  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 914  iter 714749  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 915  iter 715531  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 916  iter 716313  obj(D) 0.0000  obj(G) -0.0000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 917  iter 717095  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 918  iter 717877  obj(D) 0.0051  obj(G) -0.0051\n",
      "epoch 919  iter 718659  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 920  iter 719441  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 921  iter 720223  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 922  iter 721005  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 923  iter 721787  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 924  iter 722569  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 925  iter 723351  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 926  iter 724133  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 927  iter 724915  obj(D) 0.0246  obj(G) -0.0243\n",
      "epoch 928  iter 725697  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 929  iter 726479  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 930  iter 727261  obj(D) 0.0009  obj(G) -0.0009\n",
      "epoch 931  iter 728043  obj(D) 0.0033  obj(G) -0.0033\n",
      "epoch 932  iter 728825  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 933  iter 729607  obj(D) 0.0007  obj(G) -0.0007\n",
      "epoch 934  iter 730389  obj(D) 0.0003  obj(G) -0.0003\n",
      "epoch 935  iter 731171  obj(D) 0.0074  obj(G) -0.0072\n",
      "epoch 936  iter 731953  obj(D) 0.1303  obj(G) -0.1302\n",
      "epoch 937  iter 732735  obj(D) 3.7899  obj(G) -3.7242\n",
      "epoch 938  iter 733517  obj(D) 2.0749  obj(G) -2.0621\n",
      "epoch 939  iter 734299  obj(D) 0.5015  obj(G) -0.4965\n",
      "epoch 940  iter 735081  obj(D) 0.4162  obj(G) -0.3094\n",
      "epoch 941  iter 735863  obj(D) 0.5024  obj(G) -0.5020\n",
      "epoch 942  iter 736645  obj(D) 0.6179  obj(G) -0.6085\n",
      "epoch 943  iter 737427  obj(D) 0.1534  obj(G) -0.1525\n",
      "epoch 944  iter 738209  obj(D) 0.6284  obj(G) -0.5948\n",
      "epoch 945  iter 738991  obj(D) 0.1888  obj(G) -0.1864\n",
      "epoch 946  iter 739773  obj(D) 0.7710  obj(G) -0.7542\n",
      "epoch 947  iter 740555  obj(D) 0.0837  obj(G) -0.0817\n",
      "epoch 948  iter 741337  obj(D) 2.3603  obj(G) -1.4928\n",
      "epoch 949  iter 742119  obj(D) 0.1612  obj(G) -0.1501\n",
      "epoch 950  iter 742901  obj(D) 0.0312  obj(G) -0.0294\n",
      "epoch 951  iter 743683  obj(D) 0.0027  obj(G) -0.0020\n",
      "epoch 952  iter 744465  obj(D) 0.0017  obj(G) -0.0001\n",
      "epoch 953  iter 745247  obj(D) 0.0024  obj(G) -0.0003\n",
      "epoch 954  iter 746029  obj(D) 0.0010  obj(G) -0.0008\n",
      "epoch 955  iter 746811  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 956  iter 747593  obj(D) 0.0003  obj(G) -0.0002\n",
      "epoch 957  iter 748375  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 958  iter 749157  obj(D) 0.0005  obj(G) -0.0004\n",
      "epoch 959  iter 749939  obj(D) 0.0002  obj(G) -0.0002\n",
      "epoch 960  iter 750721  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 961  iter 751503  obj(D) 0.0006  obj(G) -0.0006\n",
      "epoch 962  iter 752285  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 963  iter 753067  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 964  iter 753849  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 965  iter 754631  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 966  iter 755413  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 967  iter 756195  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 968  iter 756977  obj(D) 0.0144  obj(G) -0.0143\n",
      "epoch 969  iter 757759  obj(D) 0.0207  obj(G) -0.0206\n",
      "epoch 970  iter 758541  obj(D) 0.0650  obj(G) -0.0620\n",
      "epoch 971  iter 759323  obj(D) 0.0427  obj(G) -0.0415\n",
      "epoch 972  iter 760105  obj(D) 1.0166  obj(G) -0.9914\n",
      "epoch 973  iter 760887  obj(D) 1.9102  obj(G) -1.8556\n",
      "epoch 974  iter 761669  obj(D) 3.3048  obj(G) -1.3746\n",
      "epoch 975  iter 762451  obj(D) 0.5145  obj(G) -0.5059\n",
      "epoch 976  iter 763233  obj(D) 2.2285  obj(G) -2.1607\n",
      "epoch 977  iter 764015  obj(D) 0.1748  obj(G) -0.1620\n",
      "epoch 978  iter 764797  obj(D) 0.1652  obj(G) -0.1604\n",
      "epoch 979  iter 765579  obj(D) 0.2679  obj(G) -0.2391\n",
      "epoch 980  iter 766361  obj(D) 0.3668  obj(G) -0.3554\n",
      "epoch 981  iter 767143  obj(D) 0.3226  obj(G) -0.3082\n",
      "epoch 982  iter 767925  obj(D) 0.2689  obj(G) -0.2365\n",
      "epoch 983  iter 768707  obj(D) 0.0211  obj(G) -0.0176\n",
      "epoch 984  iter 769489  obj(D) 0.0005  obj(G) -0.0001\n",
      "epoch 985  iter 770271  obj(D) 0.0048  obj(G) -0.0043\n",
      "epoch 986  iter 771053  obj(D) 0.0058  obj(G) -0.0054\n",
      "epoch 987  iter 771835  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 988  iter 772617  obj(D) 0.0002  obj(G) -0.0000\n",
      "epoch 989  iter 773399  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 990  iter 774181  obj(D) 0.0001  obj(G) -0.0001\n",
      "epoch 991  iter 774963  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 992  iter 775745  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 993  iter 776527  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 994  iter 777309  obj(D) 0.0016  obj(G) -0.0016\n",
      "epoch 995  iter 778091  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 996  iter 778873  obj(D) 0.0000  obj(G) -0.0000\n",
      "epoch 997  iter 779655  obj(D) 0.0001  obj(G) -0.0000\n",
      "epoch 998  iter 780437  obj(D) 0.0002  obj(G) -0.0001\n",
      "epoch 999  iter 781219  obj(D) 0.0090  obj(G) -0.0017\n"
     ]
    }
   ],
   "source": [
    "writer = SummaryWriter(log_dir=\"runs/CIFAR10\", comment=\"f-GAN-JS\")\n",
    "\n",
    "nepochs = 1000\n",
    "niter = 0\n",
    "for epoch in range(nepochs):\n",
    "    zmodel = Variable(torch.rand((batchsize,nrand), device=device))\n",
    "    xmodel = fgan.gen(zmodel)\n",
    "    xmodelimg = vutils.make_grid(xmodel,\n",
    "        normalize=True, scale_each=True)\n",
    "    writer.add_image('Generated', xmodelimg, global_step=niter)\n",
    "    #writer.file_writer.flush()\n",
    "\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        niter += 1\n",
    "        imgs, labels = data\n",
    "\n",
    "        fgan.zero_grad()\n",
    "\n",
    "        # Generate real data (from known distribution) and noise\n",
    "        xreal = Variable(imgs.to(device), requires_grad=True)\n",
    "        zmodel = Variable(torch.rand((batchsize,nrand), device=device))\n",
    "\n",
    "        loss_gen, loss_disc = fgan(xreal, zmodel)\n",
    "        writer.add_scalar('obj/disc', loss_disc, niter)\n",
    "        writer.add_scalar('obj/gen', loss_gen, niter)\n",
    "        if i == 0:\n",
    "            print(\"epoch %d  iter %d  obj(D) %.4f  obj(G) %.4f\" % (epoch, niter, loss_disc, loss_gen))\n",
    "\n",
    "        fgan.gen.zero_grad()\n",
    "        loss_gen.backward(retain_graph=True)\n",
    "        optimizer_gen.step()\n",
    "\n",
    "        fgan.disc.zero_grad()\n",
    "        loss_disc.backward()\n",
    "        optimizer_disc.step()\n",
    "\n",
    "        #if epoch == 0 and i == 0:\n",
    "        #    writer.add_graph(fgan, input_to_model=(xreal,zmodel))\n",
    "\n",
    "writer.export_scalars_to_json(\"./all_scalars.json\")\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}