Check_Theano.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:b6946f4bce0b49dd86f4e23b917e92be3a7f47abbd83a0bcd9eab8fe729033cb"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Theano Example"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Manual Run"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cPickle\n",
      "import gzip\n",
      "import os\n",
      "import sys\n",
      "import time\n",
      "\n",
      "import numpy\n",
      "\n",
      "import theano\n",
      "import theano.tensor as T"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class LogisticRegression(object):\n",
      "    def __init__(self, input, n_in, n_out, borrow=True):\n",
      "\n",
      "        self.W = theano.shared(value=numpy.zeros((n_in, n_out), dtype=theano.config.floatX), name='W', borrow=borrow)\n",
      "        self.b = theano.shared(value=numpy.zeros((n_out,), dtype=theano.config.floatX), name='b', borrow=borrow)\n",
      "        \n",
      "        self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b)\n",
      "        self.y_pred = T.argmax(self.p_y_given_x, axis=1)\n",
      "        self.params = [self.W, self.b]\n",
      "\n",
      "    def negative_log_likelihood(self, y): \n",
      "        return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y])\n",
      "\n",
      "    def errors(self, y):\n",
      "        # check if y has same dimension of y_pred\n",
      "        if y.ndim != self.y_pred.ndim:\n",
      "            raise TypeError('y should have the same shape as self.y_pred', ('y', y.type, 'y_pred', self.y_pred.type))\n",
      "            \n",
      "        # check if y is of the correct datatype\n",
      "        if y.dtype.startswith('int'):\n",
      "            # the T.neq operator returns a vector of 0s and 1s, where 1\n",
      "            # represents a mistake in prediction\n",
      "            return T.mean(T.neq(self.y_pred, y))\n",
      "        else:\n",
      "            raise NotImplementedError()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def shared_dataset(data_xy, borrow=True):\n",
      "    data_x, data_y = data_xy\n",
      "    shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX), borrow=borrow)\n",
      "    shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX), borrow=borrow)\n",
      "    return shared_x, T.cast(shared_y, 'int32')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "__docformat__ = 'restructedtext en'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "os.path.abspath('__file__')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 46,
       "text": [
        "'/Users/mwarrick/Documents/Neural_Net_Newbies/__file__'"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def load_data(dataset):\n",
      "    data_dir, data_file = os.path.split(dataset)\n",
      "    if data_dir == \"\" and not os.path.isfile(dataset):\n",
      "        # Check if dataset is in the data directory.\n",
      "        new_path = os.path.join(os.path.split('__file__')[0], \"..\", \"data\", dataset)\n",
      "        if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz':\n",
      "            dataset = new_path\n",
      "\n",
      "    if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz':\n",
      "        import urllib\n",
      "        origin = ('http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz')\n",
      "        print 'Downloading data from %s' % origin\n",
      "        urllib.urlretrieve(origin, dataset)\n",
      "        \n",
      "    print '... loading data'\n",
      "\n",
      "    # Load the dataset\n",
      "    f = gzip.open(dataset, 'rb')\n",
      "    train_set, valid_set, test_set = cPickle.load(f)\n",
      "    f.close()\n",
      "    \n",
      "    test_set_x, test_set_y = shared_dataset(test_set)\n",
      "    valid_set_x, valid_set_y = shared_dataset(valid_set)\n",
      "    train_set_x, train_set_y = shared_dataset(train_set)\n",
      "\n",
      "    data_sets = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),\n",
      "            (test_set_x, test_set_y)]\n",
      "    return data_sets"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 47
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "*SGD Opitmization Function code - Need to split out and put some calls in main*"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "learning_rate=0.13\n",
      "n_epochs=100\n",
      "dataset='mnist.pkl.gz'\n",
      "batch_size=600"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 49
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "*Build Model Structure*"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "datasets = load_data(\"data/mnist.pkl.gz\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "... loading data\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "datasets[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "(<TensorType(float64, matrix)>, Elemwise{Cast{int32}}.0)"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_set_x, train_set_y = datasets[0]\n",
      "valid_set_x, valid_set_y = datasets[1]\n",
      "test_set_x, test_set_y = datasets[2]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_set_x"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "<TensorType(float64, matrix)>"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size\n",
      "n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size\n",
      "n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "n_train_batches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "83"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "n_valid_batches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "16"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "index = T.lscalar()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "x = T.matrix('x')\n",
      "y = T.ivector('y')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "classifier = LogisticRegression(input=x, n_in=28 * 28, n_out=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cost = classifier.negative_log_likelihood(y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_model = theano.function(\n",
      "    inputs=[index],\n",
      "    outputs=cost,\n",
      "    updates=updates,\n",
      "    givens={\n",
      "        x: train_set_x[index * batch_size: (index + 1) * batch_size],\n",
      "        y: train_set_y[index * batch_size: (index + 1) * batch_size]\n",
      "    }\n",
      ")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "validate_model = theano.function(inputs=[index], outputs=classifier.errors(y),\n",
      "    givens={\n",
      "        x: valid_set_x[index * batch_size: (index + 1) * batch_size],\n",
      "        y: valid_set_y[index * batch_size: (index + 1) * batch_size]\n",
      "    }\n",
      ")\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test_model = theano.function(inputs=[index], outputs=classifier.errors(y), givens={\n",
      "        x: test_set_x[index * batch_size: (index + 1) * batch_size],\n",
      "        y: test_set_y[index * batch_size: (index + 1) * batch_size]\n",
      "    }\n",
      ")\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test_model"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "<theano.compile.function_module.Function at 0x11295dd90>"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "g_W = T.grad(cost=cost, wrt=classifier.W)\n",
      "g_b = T.grad(cost=cost, wrt=classifier.b)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "updates = [(classifier.W, classifier.W - learning_rate * g_W),\n",
      "           (classifier.b, classifier.b - learning_rate * g_b)]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 33
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "*Train Model*"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "patience = 5000  # look as this many examples regardless\n",
      "patience_increase = 2  # wait this much longer when a new best is\n",
      "                              # found\n",
      "improvement_threshold = 0.995  # a relative improvement of this much is\n",
      "                              # considered significant\n",
      "validation_frequency = min(n_train_batches, patience / 2)\n",
      "                              # go through this many\n",
      "                              # minibatche before checking the network\n",
      "                              # on the validation set; in this case we\n",
      "                              # check every epoch\n",
      "\n",
      "best_validation_loss = numpy.inf\n",
      "test_score = 0.\n",
      "start_time = time.clock()\n",
      "\n",
      "done_looping = False\n",
      "epoch = 0"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "while (epoch < n_epochs) and (not done_looping):\n",
      "    epoch = epoch + 1\n",
      "    for minibatch_index in xrange(n_train_batches):\n",
      "\n",
      "        minibatch_avg_cost = train_model(minibatch_index)\n",
      "        # iteration number\n",
      "        iter = (epoch - 1) * n_train_batches + minibatch_index\n",
      "\n",
      "        if (iter + 1) % validation_frequency == 0:\n",
      "            # compute zero-one loss on validation set\n",
      "            validation_losses = [validate_model(i)\n",
      "                                 for i in xrange(n_valid_batches)]\n",
      "            this_validation_loss = numpy.mean(validation_losses)\n",
      "\n",
      "            print(\n",
      "                'epoch %i, minibatch %i/%i, validation error %f %%' %\n",
      "                (\n",
      "                    epoch,\n",
      "                    minibatch_index + 1,\n",
      "                    n_train_batches,\n",
      "                    this_validation_loss * 100.\n",
      "                )\n",
      "            )\n",
      "\n",
      "            # if we got the best validation score until now\n",
      "            if this_validation_loss < best_validation_loss:\n",
      "                #improve patience if loss improvement is good enough\n",
      "                if this_validation_loss < best_validation_loss *  \\\n",
      "                   improvement_threshold:\n",
      "                    patience = max(patience, iter * patience_increase)\n",
      "\n",
      "                best_validation_loss = this_validation_loss\n",
      "                # test it on the test set\n",
      "\n",
      "                test_losses = [test_model(i)\n",
      "                               for i in xrange(n_test_batches)]\n",
      "                test_score = numpy.mean(test_losses)\n",
      "\n",
      "                print(\n",
      "                    (\n",
      "                        '     epoch %i, minibatch %i/%i, test error of'\n",
      "                        ' best model %f %%'\n",
      "                    ) %\n",
      "                    (\n",
      "                        epoch,\n",
      "                        minibatch_index + 1,\n",
      "                        n_train_batches,\n",
      "                        test_score * 100.\n",
      "                    )\n",
      "                )\n",
      "\n",
      "        if patience <= iter:\n",
      "            done_looping = True\n",
      "            break\n",
      "\n",
      "end_time = time.clock()\n",
      "print(\n",
      "    (\n",
      "        'Optimization complete with best validation score of %f %%,'\n",
      "        'with test performance %f %%'\n",
      "    )\n",
      "    % (best_validation_loss * 100., test_score * 100.)\n",
      ")\n",
      "print 'The code run for %d epochs, with %f epochs/sec' % (epoch, 1. * epoch / (end_time - start_time))\n",
      "print >> sys.stderr, ('The code for file ' + os.path.split('__file__')[1] + ' ran for %.1fs' % ((end_time - start_time)))\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "epoch 4, minibatch 83/83, validation error 9.875000 %\n",
        "     epoch 4, minibatch 83/83, test error of best model 9.833333 %\n",
        "epoch 5, minibatch 83/83, validation error 9.562500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 5, minibatch 83/83, test error of best model 9.479167 %\n",
        "epoch 6, minibatch 83/83, validation error 9.322917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 6, minibatch 83/83, test error of best model 9.291667 %\n",
        "epoch 7, minibatch 83/83, validation error 9.187500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 7, minibatch 83/83, test error of best model 9.000000 %\n",
        "epoch 8, minibatch 83/83, validation error 8.989583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 8, minibatch 83/83, test error of best model 8.958333 %\n",
        "epoch 9, minibatch 83/83, validation error 8.937500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 9, minibatch 83/83, test error of best model 8.812500 %\n",
        "epoch 10, minibatch 83/83, validation error 8.750000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 10, minibatch 83/83, test error of best model 8.666667 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 11, minibatch 83/83, validation error 8.666667 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 11, minibatch 83/83, test error of best model 8.520833 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 12, minibatch 83/83, validation error 8.583333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 12, minibatch 83/83, test error of best model 8.416667 %\n",
        "epoch 13, minibatch 83/83, validation error 8.489583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 13, minibatch 83/83, test error of best model 8.291667 %\n",
        "epoch 14, minibatch 83/83, validation error 8.427083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 14, minibatch 83/83, test error of best model 8.281250 %\n",
        "epoch 15, minibatch 83/83, validation error 8.354167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 15, minibatch 83/83, test error of best model 8.270833 %\n",
        "epoch 16, minibatch 83/83, validation error 8.302083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 16, minibatch 83/83, test error of best model 8.239583 %\n",
        "epoch 17, minibatch 83/83, validation error 8.250000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 17, minibatch 83/83, test error of best model 8.177083 %\n",
        "epoch 18, minibatch 83/83, validation error 8.229167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 18, minibatch 83/83, test error of best model 8.062500 %\n",
        "epoch 19, minibatch 83/83, validation error 8.260417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 20, minibatch 83/83, validation error 8.260417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 21, minibatch 83/83, validation error 8.208333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 21, minibatch 83/83, test error of best model 7.947917 %\n",
        "epoch 22, minibatch 83/83, validation error 8.187500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 22, minibatch 83/83, test error of best model 7.927083 %\n",
        "epoch 23, minibatch 83/83, validation error 8.156250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 23, minibatch 83/83, test error of best model 7.958333 %\n",
        "epoch 24, minibatch 83/83, validation error 8.114583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 24, minibatch 83/83, test error of best model 7.947917 %\n",
        "epoch 25, minibatch 83/83, validation error 8.093750 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 25, minibatch 83/83, test error of best model 7.947917 %\n",
        "epoch 26, minibatch 83/83, validation error 8.104167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 27, minibatch 83/83, validation error 8.104167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 28, minibatch 83/83, validation error 8.052083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 28, minibatch 83/83, test error of best model 7.843750 %\n",
        "epoch 29, minibatch 83/83, validation error 8.052083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 30, minibatch 83/83, validation error 8.031250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 30, minibatch 83/83, test error of best model 7.843750 %\n",
        "epoch 31, minibatch 83/83, validation error 8.010417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 31, minibatch 83/83, test error of best model 7.833333 %\n",
        "epoch 32, minibatch 83/83, validation error 7.979167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 32, minibatch 83/83, test error of best model 7.812500 %\n",
        "epoch 33, minibatch 83/83, validation error 7.947917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 33, minibatch 83/83, test error of best model 7.739583 %\n",
        "epoch 34, minibatch 83/83, validation error 7.875000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 34, minibatch 83/83, test error of best model 7.729167 %\n",
        "epoch 35, minibatch 83/83, validation error 7.885417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 36, minibatch 83/83, validation error 7.843750 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 36, minibatch 83/83, test error of best model 7.697917 %\n",
        "epoch 37, minibatch 83/83, validation error 7.802083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 37, minibatch 83/83, test error of best model 7.635417 %\n",
        "epoch 38, minibatch 83/83, validation error 7.812500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 39, minibatch 83/83, validation error 7.812500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 40, minibatch 83/83, validation error 7.822917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 41, minibatch 83/83, validation error 7.791667 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 41, minibatch 83/83, test error of best model 7.625000 %\n",
        "epoch 42, minibatch 83/83, validation error 7.770833 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 42, minibatch 83/83, test error of best model 7.614583 %\n",
        "epoch 43, minibatch 83/83, validation error 7.750000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 43, minibatch 83/83, test error of best model 7.593750 %\n",
        "epoch 44, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 44, minibatch 83/83, test error of best model 7.593750 %\n",
        "epoch 45, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 46, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 47, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 48, minibatch 83/83, validation error 7.708333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 48, minibatch 83/83, test error of best model 7.583333 %\n",
        "epoch 49, minibatch 83/83, validation error 7.677083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 49, minibatch 83/83, test error of best model 7.572917 %\n",
        "epoch 50, minibatch 83/83, validation error 7.677083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 51, minibatch 83/83, validation error 7.677083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 52, minibatch 83/83, validation error 7.656250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 52, minibatch 83/83, test error of best model 7.541667 %\n",
        "epoch 53, minibatch 83/83, validation error 7.656250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 54, minibatch 83/83, validation error 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 54, minibatch 83/83, test error of best model 7.520833 %\n",
        "epoch 55, minibatch 83/83, validation error 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 56, minibatch 83/83, validation error 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 57, minibatch 83/83, validation error 7.604167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 57, minibatch 83/83, test error of best model 7.489583 %\n",
        "epoch 58, minibatch 83/83, validation error 7.583333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 58, minibatch 83/83, test error of best model 7.458333 %\n",
        "epoch 59, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 59, minibatch 83/83, test error of best model 7.468750 %\n",
        "epoch 60, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 61, minibatch 83/83, validation error 7.583333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 62, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 63, minibatch 83/83, validation error 7.562500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 63, minibatch 83/83, test error of best model 7.510417 %\n",
        "epoch 64, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 65, minibatch 83/83, validation error 7.562500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 66, minibatch 83/83, validation error 7.552083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 66, minibatch 83/83, test error of best model 7.520833 %\n",
        "epoch 67, minibatch 83/83, validation error 7.552083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 68, minibatch 83/83, validation error 7.531250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 68, minibatch 83/83, test error of best model 7.520833 %\n",
        "epoch 69, minibatch 83/83, validation error 7.531250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 70, minibatch 83/83, validation error 7.510417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 70, minibatch 83/83, test error of best model 7.500000 %\n",
        "epoch 71, minibatch 83/83, validation error 7.520833 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 72, minibatch 83/83, validation error 7.510417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 72, minibatch 83/83, test error of best model 7.510417 %\n",
        "epoch 73, minibatch 83/83, validation error 7.500000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     epoch 73, minibatch 83/83, test error of best model 7.489583 %\n",
        "Optimization complete with best validation score of 7.500000 %,with test performance 7.489583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "The code run for 74 epochs, with 1.543878 epochs/sec\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "The code for file __file__ ran for 47.9s\n"
       ]
      }
     ],
     "prompt_number": 50
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Script Run"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%load_ext autoreload"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%autoreload 2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import lib.theano_mnist as th_mnist"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "th_mnist.main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "... load and setup data\n",
        "... loading data\n",
        "... building the model"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "... training the model"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 1, minibatch 83/83, validation error 12.458333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 1, minibatch 83/83, test error of best model 12.375000 %\n",
        "epoch 2, minibatch 83/83, validation error 11.010417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 2, minibatch 83/83, test error of best model 10.958333 %\n",
        "epoch 3, minibatch 83/83, validation error 10.312500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 3, minibatch 83/83, test error of best model 10.312500 %\n",
        "epoch 4, minibatch 83/83, validation error 9.875000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 4, minibatch 83/83, test error of best model 9.833333 %\n",
        "epoch 5, minibatch 83/83, validation error 9.562500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 5, minibatch 83/83, test error of best model 9.479167 %\n",
        "epoch 6, minibatch 83/83, validation error 9.322917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 6, minibatch 83/83, test error of best model 9.291667 %\n",
        "epoch 7, minibatch 83/83, validation error 9.187500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 7, minibatch 83/83, test error of best model 9.000000 %\n",
        "epoch 8, minibatch 83/83, validation error 8.989583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 8, minibatch 83/83, test error of best model 8.958333 %\n",
        "epoch 9, minibatch 83/83, validation error 8.937500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 9, minibatch 83/83, test error of best model 8.812500 %\n",
        "epoch 10, minibatch 83/83, validation error 8.750000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 10, minibatch 83/83, test error of best model 8.666667 %\n",
        "epoch 11, minibatch 83/83, validation error 8.666667 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 11, minibatch 83/83, test error of best model 8.520833 %\n",
        "epoch 12, minibatch 83/83, validation error 8.583333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 12, minibatch 83/83, test error of best model 8.416667 %\n",
        "epoch 13, minibatch 83/83, validation error 8.489583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 13, minibatch 83/83, test error of best model 8.291667 %\n",
        "epoch 14, minibatch 83/83, validation error 8.427083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 14, minibatch 83/83, test error of best model 8.281250 %\n",
        "epoch 15, minibatch 83/83, validation error 8.354167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 15, minibatch 83/83, test error of best model 8.270833 %\n",
        "epoch 16, minibatch 83/83, validation error 8.302083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 16, minibatch 83/83, test error of best model 8.239583 %\n",
        "epoch 17, minibatch 83/83, validation error 8.250000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 17, minibatch 83/83, test error of best model 8.177083 %\n",
        "epoch 18, minibatch 83/83, validation error 8.229167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 18, minibatch 83/83, test error of best model 8.062500 %\n",
        "epoch 19, minibatch 83/83, validation error 8.260417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 20, minibatch 83/83, validation error 8.260417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 21, minibatch 83/83, validation error 8.208333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 21, minibatch 83/83, test error of best model 7.947917 %\n",
        "epoch 22, minibatch 83/83, validation error 8.187500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 22, minibatch 83/83, test error of best model 7.927083 %\n",
        "epoch 23, minibatch 83/83, validation error 8.156250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 23, minibatch 83/83, test error of best model 7.958333 %\n",
        "epoch 24, minibatch 83/83, validation error 8.114583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 24, minibatch 83/83, test error of best model 7.947917 %\n",
        "epoch 25, minibatch 83/83, validation error 8.093750 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 25, minibatch 83/83, test error of best model 7.947917 %\n",
        "epoch 26, minibatch 83/83, validation error 8.104167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 27, minibatch 83/83, validation error 8.104167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 28, minibatch 83/83, validation error 8.052083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 28, minibatch 83/83, test error of best model 7.843750 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 29, minibatch 83/83, validation error 8.052083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 30, minibatch 83/83, validation error 8.031250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 30, minibatch 83/83, test error of best model 7.843750 %\n",
        "epoch 31, minibatch 83/83, validation error 8.010417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 31, minibatch 83/83, test error of best model 7.833333 %\n",
        "epoch 32, minibatch 83/83, validation error 7.979167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 32, minibatch 83/83, test error of best model 7.812500 %\n",
        "epoch 33, minibatch 83/83, validation error 7.947917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 33, minibatch 83/83, test error of best model 7.739583 %\n",
        "epoch 34, minibatch 83/83, validation error 7.875000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 34, minibatch 83/83, test error of best model 7.729167 %\n",
        "epoch 35, minibatch 83/83, validation error 7.885417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 36, minibatch 83/83, validation error 7.843750 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 36, minibatch 83/83, test error of best model 7.697917 %\n",
        "epoch 37, minibatch 83/83, validation error 7.802083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 37, minibatch 83/83, test error of best model 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 38, minibatch 83/83, validation error 7.812500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 39, minibatch 83/83, validation error 7.812500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 40, minibatch 83/83, validation error 7.822917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 41, minibatch 83/83, validation error 7.791667 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 41, minibatch 83/83, test error of best model 7.625000 %\n",
        "epoch 42, minibatch 83/83, validation error 7.770833 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 42, minibatch 83/83, test error of best model 7.614583 %\n",
        "epoch 43, minibatch 83/83, validation error 7.750000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 43, minibatch 83/83, test error of best model 7.593750 %\n",
        "epoch 44, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 44, minibatch 83/83, test error of best model 7.593750 %\n",
        "epoch 45, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 46, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 47, minibatch 83/83, validation error 7.739583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 48, minibatch 83/83, validation error 7.708333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 48, minibatch 83/83, test error of best model 7.583333 %\n",
        "epoch 49, minibatch 83/83, validation error 7.677083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 49, minibatch 83/83, test error of best model 7.572917 %\n",
        "epoch 50, minibatch 83/83, validation error 7.677083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 51, minibatch 83/83, validation error 7.677083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 52, minibatch 83/83, validation error 7.656250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 52, minibatch 83/83, test error of best model 7.541667 %\n",
        "epoch 53, minibatch 83/83, validation error 7.656250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 54, minibatch 83/83, validation error 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 54, minibatch 83/83, test error of best model 7.520833 %\n",
        "epoch 55, minibatch 83/83, validation error 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 56, minibatch 83/83, validation error 7.635417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 57, minibatch 83/83, validation error 7.604167 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 57, minibatch 83/83, test error of best model 7.489583 %\n",
        "epoch 58, minibatch 83/83, validation error 7.583333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 58, minibatch 83/83, test error of best model 7.458333 %\n",
        "epoch 59, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 59, minibatch 83/83, test error of best model 7.468750 %\n",
        "epoch 60, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 61, minibatch 83/83, validation error 7.583333 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 62, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 63, minibatch 83/83, validation error 7.562500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 63, minibatch 83/83, test error of best model 7.510417 %\n",
        "epoch 64, minibatch 83/83, validation error 7.572917 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 65, minibatch 83/83, validation error 7.562500 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 66, minibatch 83/83, validation error 7.552083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 66, minibatch 83/83, test error of best model 7.520833 %\n",
        "epoch 67, minibatch 83/83, validation error 7.552083 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 68, minibatch 83/83, validation error 7.531250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 68, minibatch 83/83, test error of best model 7.520833 %\n",
        "epoch 69, minibatch 83/83, validation error 7.531250 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 70, minibatch 83/83, validation error 7.510417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 70, minibatch 83/83, test error of best model 7.500000 %\n",
        "epoch 71, minibatch 83/83, validation error 7.520833 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 72, minibatch 83/83, validation error 7.510417 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 72, minibatch 83/83, test error of best model 7.510417 %\n",
        "epoch 73, minibatch 83/83, validation error 7.500000 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "epoch 73, minibatch 83/83, test error of best model 7.489583 %\n",
        "Optimization complete with best validation score of 7.500000 %, with test performance 7.489583 %"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "The code run for 74 epochs, with 1.748854 epochs/sec\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "The code for file __file__ ran for 42.3s\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}