Experiments/tf_GAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#该函数将给出权重初始化的方法\n",
    "def variable_init(size):\n",
    "    in_dim = size[0]\n",
    "\n",
    "    #计算随机生成变量所服从的正态分布标准差\n",
    "    w_stddev = 1. / tf.sqrt(in_dim / 2.)\n",
    "    return tf.random_normal(shape=size, stddev=w_stddev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#定义输入矩阵的占位符，输入层单元为784，None代表批量大小的占位，X代表输入的真实图片。占位符的数值类型为32位浮点型\n",
    "X = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "\n",
    "#定义判别器的权重矩阵和偏置项向量，由此可知判别网络为三层全连接网络\n",
    "D_W1 = tf.Variable(variable_init([784, 128]))\n",
    "D_b1 = tf.Variable(tf.zeros(shape=[128]))\n",
    "\n",
    "D_W2 = tf.Variable(variable_init([128, 1]))\n",
    "D_b2 = tf.Variable(tf.zeros(shape=[1]))\n",
    "\n",
    "theta_D = [D_W1, D_W2, D_b1, D_b2]\n",
    "\n",
    "#定义生成器的输入噪声为100维度的向量组，None根据批量大小确定\n",
    "Z = tf.placeholder(tf.float32, shape=[None, 100])\n",
    "\n",
    "#定义生成器的权重与偏置项。输入层为100个神经元且接受随机噪声，\n",
    "#输出层为784个神经元，并输出手写字体图片。生成网络根据原论文为三层全连接网络\n",
    "G_W1 = tf.Variable(variable_init([100, 128]))\n",
    "G_b1 = tf.Variable(tf.zeros(shape=[128]))\n",
    "\n",
    "G_W2 = tf.Variable(variable_init([128, 784]))\n",
    "G_b2 = tf.Variable(tf.zeros(shape=[784]))\n",
    "\n",
    "theta_G = [G_W1, G_W2, G_b1, G_b2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#定义一个可以生成m*n阶随机矩阵的函数，该矩阵的元素服从均匀分布，随机生成的z就为生成器的输入\n",
    "def sample_Z(m, n):\n",
    "    return np.random.uniform(-1., 1., size=[m, n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#定义生成器\n",
    "def generator(z):\n",
    "    \n",
    "    #第一层先计算 y=z*G_W1+G-b1,然后投入激活函数计算G_h1=ReLU（y）,G_h1 为第二次层神经网络的输出激活值\n",
    "    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n",
    "    \n",
    "    #以下两个语句计算第二层传播到第三层的激活结果，第三层的激活结果是含有784个元素的向量，该向量转化28×28就可以表示图像\n",
    "    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n",
    "    G_prob = tf.nn.sigmoid(G_log_prob)\n",
    "    return G_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#定义判别器\n",
    "def discriminator(x):\n",
    "    \n",
    "    #计算D_h1=ReLU（x*D_W1+D_b1）,该层的输入为含784个元素的向量\n",
    "    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n",
    "    \n",
    "    #计算第三层的输出结果。因为使用的是Sigmoid函数，则该输出结果是一个取值为[0,1]间的标量（见上述权重定义）\n",
    "    #即判别输入的图像到底是真（=1）还是假（=0）\n",
    "    D_logit = tf.matmul(D_h1, D_W2) + D_b2\n",
    "    D_prob = tf.nn.sigmoid(D_logit)\n",
    "    \n",
    "    #返回判别为真的概率和第三层的输入值，输出D_logit是为了将其输入tf.nn.sigmoid_cross_entropy_with_logits()以构建损失函数\n",
    "    return D_prob, D_logit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#该函数用于输出生成图片\n",
    "def plot(samples):\n",
    "    fig = plt.figure(figsize=(4, 4))\n",
    "    gs = gridspec.GridSpec(4, 4)\n",
    "    gs.update(wspace=0.05, hspace=0.05)\n",
    "\n",
    "    for i, sample in enumerate(samples):\n",
    "        ax = plt.subplot(gs[i])\n",
    "        plt.axis('off')\n",
    "        ax.set_xticklabels([])\n",
    "        ax.set_yticklabels([])\n",
    "        ax.set_aspect('equal')\n",
    "        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n",
    "\n",
    "    return fig\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 交叉熵损失函数\n",
    "sigmoid_cross_entropy_with_logits函数的输入是logits和targets，logits就是神经网络模型中的 W * X矩阵，且不需要经过Sigmoid激活函数。而targets的shape和logits相同，即正确的标注值。若令x = logits、 z = labels，那么该函数的表达式为z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#输入随机噪声z而输出生成样本\n",
    "G_sample = generator(Z)\n",
    "\n",
    "#分别输入真实图片和生成的图片，并投入判别器以判断真伪\n",
    "D_real, D_logit_real = discriminator(X)\n",
    "D_fake, D_logit_fake = discriminator(G_sample)\n",
    "\n",
    "#以下为原论文的判别器损失和生成器损失，但本实现并没有使用该损失函数\n",
    "# D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake))\n",
    "# G_loss = -tf.reduce_mean(tf.log(D_fake))\n",
    "\n",
    "# 我们使用交叉熵作为判别器和生成器的损失函数，因为sigmoid_cross_entropy_with_logits内部会对预测输入执行Sigmoid函数，\n",
    "#所以我们取判别器最后一层未投入激活函数的值，即D_h1*D_W2+D_b2。\n",
    "#tf.ones_like(D_logit_real)创建维度和D_logit_real相等的全是1的标注，真实图片。\n",
    "D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))\n",
    "D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))\n",
    "\n",
    "#损失函数为两部分，即E[log(D(x))]+E[log(1-D(G(z)))]，将真的判别为假和将假的判别为真\n",
    "D_loss = D_loss_real + D_loss_fake\n",
    "\n",
    "#同样使用交叉熵构建生成器损失函数\n",
    "G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))\n",
    "\n",
    "#定义判别器和生成器的优化方法为Adam算法，关键字var_list表明最小化损失函数所更新的权重矩阵\n",
    "D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)\n",
    "G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "#选择训练的批量大小和随机生成噪声的维度\n",
    "mb_size = 128\n",
    "Z_dim = 100\n",
    "\n",
    "#读取数据集MNIST，并放在当前目录data文件夹下MNIST文件夹中，如果该地址没有数据，则下载数据至该文件夹\n",
    "mnist = input_data.read_data_sets(\"./data/MNIST/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter: 0\n",
      "D loss: 1.671\n",
      "G_loss: 1.718\n",
      "\n",
      "Iter: 2000\n",
      "D loss: 0.05008\n",
      "G_loss: 4.74\n",
      "\n",
      "Iter: 4000\n",
      "D loss: 0.3667\n",
      "G_loss: 4.85\n",
      "\n",
      "Iter: 6000\n",
      "D loss: 0.3974\n",
      "G_loss: 4.059\n",
      "\n",
      "Iter: 8000\n",
      "D loss: 0.7007\n",
      "G_loss: 2.628\n",
      "\n",
      "Iter: 10000\n",
      "D loss: 0.4421\n",
      "G_loss: 3.05\n",
      "\n",
      "Iter: 12000\n",
      "D loss: 0.7872\n",
      "G_loss: 2.562\n",
      "\n",
      "Iter: 14000\n",
      "D loss: 0.7155\n",
      "G_loss: 2.877\n",
      "\n",
      "Iter: 16000\n",
      "D loss: 0.9827\n",
      "G_loss: 2.042\n",
      "\n",
      "Iter: 18000\n",
      "D loss: 0.7171\n",
      "G_loss: 1.966\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#打开一个会话运行计算图\n",
    "sess = tf.Session()\n",
    "\n",
    "#初始化所有定义的变量\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "#如果当前目录下不存在out文件夹，则创建该文件夹\n",
    "if not os.path.exists('out/'):\n",
    "    os.makedirs('out/')\n",
    "\n",
    "#初始化，并开始迭代训练，100W次\n",
    "i = 0\n",
    "for it in range(20000):\n",
    "    \n",
    "    #每2000次输出一张生成器生成的图片\n",
    "    if it % 2000 == 0:\n",
    "        samples = sess.run(G_sample, feed_dict={Z: sample_Z(16, Z_dim)})\n",
    "\n",
    "        fig = plot(samples)\n",
    "        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n",
    "        i += 1\n",
    "        plt.close(fig)\n",
    "    \n",
    "    #next_batch抽取下一个批量的图片，该方法返回一个矩阵，即shape=[mb_size，784]，每一行是一张图片，共批量大小行\n",
    "    X_mb, _ = mnist.train.next_batch(mb_size)\n",
    "    \n",
    "    #投入数据并根据优化方法迭代一次，计算损失后返回损失值\n",
    "    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(mb_size, Z_dim)})\n",
    "    _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(mb_size, Z_dim)})\n",
    "\n",
    "\n",
    "    #每迭代2000次输出迭代数、生成器损失和判别器损失\n",
    "    if it % 2000 == 0:\n",
    "        print('Iter: {}'.format(it))\n",
    "        print('D loss: {:.4}'. format(D_loss_curr))\n",
    "        print('G_loss: {:.4}'.format(G_loss_curr))\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}