Tensroflow- Conv 56x56-30epochs-0.0015lr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from src.data import Data\n",
    "from src.plothelp import *\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and pre-process the dataset:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = Data(\"img\")\n",
    "data.load_pickle(\"100x100.pickle \")\n",
    "data.resize_img((56,56)) \n",
    "X_train, X_test, y_train, y_test = data.train_test_split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4698, 56, 56, 3) (522, 56, 56, 3) (4698,) (522,)\n"
     ]
    }
   ],
   "source": [
    "#Convert python lists to np arrays\n",
    "X_trains = np.asarray(X_train)\n",
    "X_tests = np.asarray(X_test)\n",
    "y_trains = np.asarray(y_train)\n",
    "y_tests = np.asarray(y_test)\n",
    "print(X_trains.shape, X_tests.shape, y_trains.shape, y_tests.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_oh, y_test_oh= data.one_hot(y_trains, y_tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54, 87, 4)\n"
     ]
    }
   ],
   "source": [
    "# Split y_train into 54 batches containing 87 img\n",
    "y_batch = np.array_split(y_oh, 54 )\n",
    "y_batch = np.asarray(y_batch)\n",
    "print(y_batch.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54, 87, 56, 56, 3)\n"
     ]
    }
   ],
   "source": [
    "# Split X_test into 54 batches of 87 img\n",
    "X_train_batch = np.array_split(X_trains, 54 )\n",
    "X_train_batch = np.asarray(X_train_batch)\n",
    "print(X_train_batch.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.0015\n",
    "batch_size = 87\n",
    "n_epochs = 40\n",
    "keep_prob  = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# input X: 28x28 grayscale images\n",
    "X = tf.placeholder(tf.float32, [None, 56, 56, 3])\n",
    "Y = tf.placeholder(tf.float32, [None, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l1 = 4  # first convolutional layer output depth\n",
    "l2 = 8  # second convolutional layer output depth\n",
    "l3 = 12  # third convolutional layer\n",
    "l4 = 12  # forth conv\n",
    "l5 = 200  # fully connected layer\n",
    "num_classes = 4 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w1 = tf.Variable(tf.truncated_normal([5, 5, 3, l1], stddev=0.1))  # 5x5 patch, 3 input channel, K output channels\n",
    "b1 = tf.Variable(tf.random_normal([l1]))\n",
    "\n",
    "w2 = tf.Variable(tf.truncated_normal([5, 5, l1, l2], stddev=0.1))\n",
    "b2 = tf.Variable(tf.random_normal([l2]))\n",
    "\n",
    "w3 = tf.Variable(tf.truncated_normal([4, 4, l2, l3], stddev=0.1))\n",
    "b3 = tf.Variable(tf.random_normal([l3]))\n",
    "\n",
    "w4 = tf.Variable(tf.truncated_normal([4, 4, l3,l4], stddev=0.1))\n",
    "b4 = tf.Variable(tf.random_normal([l4]))\n",
    "\n",
    "w5 = tf.Variable(tf.truncated_normal([7 * 7 * l4, l5], stddev=0.1))\n",
    "b5 = tf.Variable(tf.random_normal([l5]))\n",
    "\n",
    "w = tf.Variable(tf.truncated_normal([l5, num_classes], stddev=0.1))\n",
    "b = tf.Variable(tf.random_normal([num_classes]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The model\n",
    "Y1 = tf.nn.relu(tf.nn.conv2d(X, w1, strides=[1, 1, 1, 1], padding='SAME') + b1)\n",
    "Y1 = tf.nn.dropout(Y1,keep_prob=keep_prob) #for adding dropout, comment to disable\n",
    "\n",
    "Y2 = tf.nn.conv2d(Y1, w2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y2 = tf.nn.max_pool(Y2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 28x28\n",
    "Y2 = tf.nn.relu( Y2+ b2)\n",
    "Y2 = tf.nn.dropout(Y2,keep_prob=keep_prob) #for adding dropout, comment to disable\n",
    "\n",
    "Y3 = tf.nn.conv2d(Y2, w3, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y3 = tf.nn.max_pool(Y3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 14x14\n",
    "Y3 = tf.nn.relu(Y3+ b3)\n",
    "Y3 = tf.nn.dropout(Y3,keep_prob=keep_prob) #for adding dropout, comment to disable\n",
    "\n",
    "Y4 = tf.nn.conv2d(Y3, w4, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y4 = tf.nn.max_pool(Y3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 7x7\n",
    "Y4 = tf.nn.relu(Y4+ b4)\n",
    "Y4 = tf.nn.dropout(Y4,keep_prob=keep_prob)\n",
    "\n",
    "# fully connected layer\n",
    "fc = tf.reshape(Y4, shape=[-1, 7 * 7 * l4])\n",
    "\n",
    "Y5 = tf.nn.relu(tf.matmul(fc, w5) + b5)\n",
    "Ylogits = tf.matmul(Y5, w) + b\n",
    "logits = tf.nn.softmax(Ylogits)\n",
    "\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y)\n",
    "cross_entropy = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "# Calculate the accuracy of the model\n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 accuracy= 0.261813537\n",
      "Epoch: 0002 accuracy= 0.294167732\n",
      "Epoch: 0003 accuracy= 0.291400598\n",
      "Epoch: 0004 accuracy= 0.294380586\n",
      "Epoch: 0005 accuracy= 0.310557684\n",
      "Epoch: 0006 accuracy= 0.309067688\n",
      "Epoch: 0007 accuracy= 0.325457643\n",
      "Epoch: 0008 accuracy= 0.337164751\n",
      "Epoch: 0009 accuracy= 0.370157516\n",
      "Epoch: 0010 accuracy= 0.369306087\n",
      "Epoch: 0011 accuracy= 0.398467433\n",
      "Epoch: 0012 accuracy= 0.406343126\n",
      "Epoch: 0013 accuracy= 0.418688805\n",
      "Epoch: 0014 accuracy= 0.430821626\n",
      "Epoch: 0015 accuracy= 0.433588761\n",
      "Epoch: 0016 accuracy= 0.445934439\n",
      "Epoch: 0017 accuracy= 0.448063004\n",
      "Epoch: 0018 accuracy= 0.464878671\n",
      "Epoch: 0019 accuracy= 0.470625795\n",
      "Epoch: 0020 accuracy= 0.471902936\n",
      "Epoch: 0021 accuracy= 0.478288631\n",
      "Epoch: 0022 accuracy= 0.491060024\n",
      "Epoch: 0023 accuracy= 0.500212855\n",
      "Epoch: 0024 accuracy= 0.507449977\n",
      "Epoch: 0025 accuracy= 0.518518514\n",
      "Epoch: 0026 accuracy= 0.530438481\n",
      "Epoch: 0027 accuracy= 0.557258402\n",
      "Epoch: 0028 accuracy= 0.595146871\n",
      "Epoch: 0029 accuracy= 0.606641121\n",
      "Epoch: 0030 accuracy= 0.634738187\n",
      "Model saved in file: ./models/conv56x56x3-drop-lr.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    n_batches = 54\n",
    "    for epochs in range(30):\n",
    "        acc = 0\n",
    "        for i in range(n_batches):\n",
    "            batch_X = X_train_batch[i, :]\n",
    "            batch_Y = y_batch[i, :]\n",
    "            _, a = sess.run([train_step, accuracy], {X: batch_X, Y: batch_Y})\n",
    "            acc=acc+a\n",
    "        print(\"Epoch:\", '%04d' % (epochs + 1), \"accuracy=\", \"{:.9f}\".format(acc/54))\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    save_path = saver.save(sess, \"./models/conv56x56x3-drop-lr.ckpt\")\n",
    "    print(\"Model saved in file: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model restored.\n",
      "0.60728\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    saver = tf.train.Saver().restore(sess, \"./models/conv56x56x3-drop-lr.ckpt\")\n",
    "    print(\"Model restored.\")\n",
    "    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={X: X_tests, Y: y_test_oh}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The convolutional neural network with 3 convlutional layers and one fully connected layer. It achieved very high accuracy while training on the dataset. However, tested model achieved only 36.6 % accuracy. Hypothesis why training accuracy is so high while test accuracy is low is that the learnt model overfits the training data. The other possibility is that neural network is not deep enought and needs more layers. Adding a dropout for neurons should improve the classifier."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}