mnist_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/device:CPU:0', '/device:GPU:0']\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "def get_available_devices():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos]\n",
    "\n",
    "print(get_available_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using Keras version: 2.2.4 backend: tensorflow\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout, Flatten, MaxPooling2D\n",
    "from keras.layers.convolutional import Conv2D \n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K\n",
    "\n",
    "from distutils.version import LooseVersion as LV\n",
    "from keras import __version__\n",
    "\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import normalize\n",
    "from keras.models import model_from_json\n",
    "\n",
    "import random\n",
    "from keract import get_activations, display_activations\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import norm, Session\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "from sklearn import linear_model\n",
    "print('Using Keras version:', __version__, 'backend:', K.backend())\n",
    "assert(LV(__version__) >= LV(\"2.0.0\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[22. 28.]\n",
      " [49. 64.]]\n"
     ]
    }
   ],
   "source": [
    "# Creates a graph.\n",
    "a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')\n",
    "b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')\n",
    "c = tf.matmul(a, b)\n",
    "# Creates a session with log_device_placement set to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MNIST data loaded: train: 60000 test: 10000\n",
      "X_train: (60000, 28, 28)\n",
      "y_train: (60000,)\n",
      "Y_train: (60000, 10)\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "nb_classes = 10\n",
    "\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "# one-hot encoding:\n",
    "Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "\n",
    "print()\n",
    "print('MNIST data loaded: train:',len(X_train),'test:',len(X_test))\n",
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('Y_train:', Y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (60000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "if K.image_dim_ordering() == 'th':\n",
    "    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)\n",
    "    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "    \n",
    "print('X_train:', X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_6 (Conv2D)            (None, 28, 28, 128)       1280      \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, 28, 28, 128)       147584    \n",
      "_________________________________________________________________\n",
      "conv2d_8 (Conv2D)            (None, 28, 28, 128)       147584    \n",
      "_________________________________________________________________\n",
      "conv2d_9 (Conv2D)            (None, 28, 28, 128)       147584    \n",
      "_________________________________________________________________\n",
      "conv2d_10 (Conv2D)           (None, 28, 28, 128)       147584    \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 100352)            0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               12845184  \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 13,454,602\n",
      "Trainable params: 13,454,602\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# number of convolutional filters to use\n",
    "nb_filters = 128\n",
    "# convolution kernel size\n",
    "kernel_size = (3, 3)\n",
    "# size of pooling area for max pooling\n",
    "pool_size = (2, 2)\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(nb_filters, kernel_size,\n",
    "                 padding='same',\n",
    "                 activation='relu',\n",
    "                 input_shape=input_shape))\n",
    "\n",
    "model.add(Conv2D(nb_filters, kernel_size,\n",
    "                 padding='same',\n",
    "                 activation='relu'))\n",
    "\n",
    "\n",
    "model.add(Conv2D(nb_filters, kernel_size,\n",
    "                 padding='same',\n",
    "                 activation='relu'))\n",
    "model.add(Conv2D(nb_filters, kernel_size,\n",
    "                 padding='same',\n",
    "                 activation='relu'))\n",
    "model.add(Conv2D(nb_filters, kernel_size,\n",
    "                 padding='same',\n",
    "                 activation='relu'))\n",
    "\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation = 'relu'))#model.add(Dropout(0.5))\n",
    "model.add(Dense(128, activation = 'relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(nb_classes , activation = 'softmax'))\n",
    "\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 6.91 µs\n",
      "Epoch 1/5\n",
      " - 14s - loss: 0.2743 - acc: 0.9218\n",
      "Epoch 2/5\n",
      " - 10s - loss: 0.0614 - acc: 0.9835\n",
      "Epoch 3/5\n",
      " - 10s - loss: 0.0423 - acc: 0.9891\n",
      "Epoch 4/5\n",
      " - 10s - loss: 0.0308 - acc: 0.9920\n",
      "Epoch 5/5\n",
      " - 10s - loss: 0.0249 - acc: 0.9933\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "\n",
    "epochs = 5 # one epoch takes about 45 seconds\n",
    "\n",
    "history = model.fit(X_train, \n",
    "                    Y_train, \n",
    "                    epochs=epochs, \n",
    "                    batch_size=128,\n",
    "                    verbose=2)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAADSCAYAAAA/vMlrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHpVJREFUeJzt3XtQVPf9N/D32V2W2wILCAeQS5KKP43wU5+oQSCmWQIkrIiJpPOk82ScNOQymiZp2rRp+hQntCYzuZvY6sNkwnRqm2dirCXK76dJgYTgJdqYuI+XNNGKLgoLCnKHleU8fwALGy67wN7P+zXjCLvn7Od8d9a3n/M95+wRJEmSQEQkYwpPbwARkacxCIlI9hiERCR7DEIikj0GIRHJHoOQiGSPQUheR6fT4fDhw57eDJIRBiERyR6DkIhkj0FIXstsNmPr1q3Izs5GdnY2tm7dCrPZDABoa2vD448/jhUrVmDVqlX48Y9/jKGhIQBAeXk57rjjDixfvhz5+fk4cuSIJ4dBPkDl6Q0gmsqOHTtw8uRJVFZWQhAEbNq0CX/84x/xzDPPoKKiAqIoWkPu5MmTEAQB//73v/GXv/wFH374IURRRGNjozUgiabCjpC81r59+7B582ZER0cjKioKmzdvxkcffQQAUKlUaG1txZUrVxAQEIAVK1ZAEAQolUqYzWacP38eN27cQGJiIpKTkz08EvJ2DELyWi0tLUhISLD+npCQgJaWFgDAI488gpSUFPzkJz9BTk4OysvLAQApKSl44YUX8M477yAzMxM/+9nPYDKZPLL95DsYhOS1YmNjceXKFevvTU1NiI2NBQBoNBo8//zzqK6uxo4dO1BRUWHdTS4sLMT777+P2tpaCIKA1157zSPbT76DQUheS6/XY8eOHWhra0NbWxv+8Ic/oLCwEABQW1uLixcvQpIkhIWFQalUWucIjxw5ArPZDLVajcDAQCgU/JjT9HiwhLzWpk2b0NPTg3Xr1gEA7rnnHmzatAkAcPHiRfzud79DW1sbwsPD8eCDDyIjIwPffPMNXn/9dZw/fx4BAQFYvnw5ysrKPDkM8gECv5iViOSO+wxEJHsMQiKSPQYhEckeg5CIZI9BSESy53Wnz7S2ds14ncjIELS397pga1jfm2vLvb6cxz6b+jExYVM+5xcdoUqlZH0Z1pZ7fTmP3dn1/SIIiYjmgkFIRLLHICQi2WMQEpHs+XwQHj3TjIp9p8FLpolotnw+CM82tONvn56D4fw1T28KEfkonw/C3JVJAIDK+gvsColoVnw+CBNjNMhamoCG5i52hUQ0Kz4fhADwYO5/AGBXSESz4xdBmBIfjhWLYtHQ3IWT7AqJaIb8IggBYF3WTRDArpCIZs5vgjAxRoMVi2JxkV0hEc2Q3wQhABSyKySiWfCrIGRXSESz4VdBCHCukIhmzu+CcH6MBisXj3SF59gVEpF9DgVhXV0d8vPzkZubi/Ly8gnPV1RUoKCgAIWFhdi4cSMuX75sfW7x4sUoKipCUVERnnjiCedt+TQKM9kVEpHj7H5Vv8ViQVlZGSoqKiCKIoqLi6HT6bBgwQLrMosXL8aePXsQHByMv/71r3j11Vfx1ltvAQCCgoJQWVnpuhFMYrQrPHa2BSfPXcOy1HlurU9EvsVuR2gwGJCSkoKkpCSo1Wro9XpUV1fbLJORkYHg4GAAwLJly9Dc3OyarZ0BdoVE5Ci7QWgymRAXF2f9XRRFmEymKZf/8MMPsWbNGuvvAwMDuP/++/GjH/0I//jHP+a4uY6zzhWauvD1uatuq0tEvsepd7GrrKzEqVOnsGvXLutjtbW1EEURRqMRGzduxMKFC5GcnDzla0RGhszqpiyT3aFq49olOP5NC6qOXkLu6pshCMKMX3cu9d3Jk/XlPHZP15fz2J1Z324QiqJos6trMpkgiuKE5Q4fPoydO3di165dUKvVNusDQFJSElatWoUzZ85MG4SzuT1gTEzYpLcBDVYKWLloeK7wkyMXsDw1ZsavPZf67uLJ+nIeu6fry3nss6k/p9t5pqeno6GhAUajEWazGVVVVdDpdDbLnDlzBqWlpdixYweio6Otj3d0dMBsNgMA2tracOLECZuDLO5QmHUz5wqJaFp2O0KVSoXS0lKUlJTAYrFgw4YNSE1NxbZt25CWloacnBy88sor6O3txdNPPw0AiI+Px86dO3H+/Hls2bIFgiBAkiQ8+uijbg/C+fNCrUeQv/7uKpYvdE1XSES+S5C8rE2aTattr0W+crUHv333CyTFarDl4ZVOnyv0tV0Ef6kt9/pyHvts6s9p19gfJMwLxapbRVxq6cbX3/EIMhHZkkUQAjyvkIimJpsgZFdIRFORTRAC7AqJaHKyCsKEeaG4faQr/IpdIRGNkFUQAmPfYv0Ru0IiGiG7IIyPZldIRLZkF4SA7b1NhtgVEsmeLINwtCs0tnTjq2/ZFRLJnSyDEBjpCgXgo0PsConkTrZByK6QiEbJNgiBkfMK2RUSyZ6sg9C2K2z19OYQkYfIOgiBsa6wsr6BXSGRTMk+COOjQ5Fxq4jGVnaFRHIl+yAEgLXsColkjUEIdoVEcscgHFGYdfNIV8gjyERywyAcERcVgoxb49DY2oMT/2JXSCQnDMJxeLUJkTwxCMdhV0gkTw4FYV1dHfLz85Gbm4vy8vIJz1dUVKCgoACFhYXYuHEjLl++bH1u7969yMvLQ15eHvbu3eu8LXeR0a6wkl0hkWzYDUKLxYKysjK8++67qKqqwv79+3Hu3DmbZRYvXow9e/Zg3759yM/Px6uvvgoAuH79OrZv344PPvgAu3fvxvbt29HR0eGakThJXFQIVi+Jw2V2hUSyYTcIDQYDUlJSkJSUBLVaDb1ej+rqaptlMjIyEBwcDABYtmwZmpubAQD19fXIysqCVqtFREQEsrKy8Pnnn7tgGM5lvdqEXSGRLKjsLWAymRAXF2f9XRRFGAyGKZf/8MMPsWbNminXNZlM09aLjAyBSqW0u+HfN93Nm2fzWnfdloSafxpxrqkbWUsT3Fp/NjxZX85j93R9OY/dmfXtBuFMVFZW4tSpU9i1a9esX6O9vXfG68z0jveOyP0f81H7pRF//u8zWBCvgUIQ3Fp/JjxZX85j93R9OY99NvWnC027u8aiKFp3dYHhLk8UxQnLHT58GDt37sSOHTugVqtntK43EsfNFX7JuUIiv2Y3CNPT09HQ0ACj0Qiz2YyqqirodDqbZc6cOYPS0lLs2LED0dHR1sezs7NRX1+Pjo4OdHR0oL6+HtnZ2c4fhYsUZt0EhSDgI15tQuTX7O4aq1QqlJaWoqSkBBaLBRs2bEBqaiq2bduGtLQ05OTk4JVXXkFvby+efvppAEB8fDx27twJrVaLTZs2obi4GACwefNmaLVa147IicTIEKxeIuLQqWZ8+a9WrFwU6+lNIiIXECQvu7nvbOYcXDlXYWrvxW/Kv0B8dAhefGTVpHOFvjZX4i+15V5fzmOfTf05zRHK3WhXePkq5wqJ/BWD0AFrOVdI5NcYhA4QI0OwOm24K/znNy2e3hwicjIGoYPWZo50hYf4LdZE/oZB6KDRrvAKu0Iiv8MgnIHC8V3hELtCIn/BIJyB2MgQZKbFDXeF/2JXSOQvGIQztDYzhV0hkZ9hEM4Qu0Ii/8MgnAV2hUT+hUE4C+O7wuM8gkzk8xiEs2S92uTQBVjYFRL5NAbhLMVqg5GZHoema704dPKy/RWIyGsxCOdg9GqT//vJvzhXSOTDGIRzMNoVGk3dnCsk8mEMwjlam3kTlIrhuUJ2hUS+iUE4R7HaYOhWJKHpWi+OfTP9HfqIyDsxCJ3gR3cvhFIhYB/PKyTySQxCJ4iLDkVmWhy7QiIfxSB0ktG5QnaFRL6HQegkMdpgZI2cV3jsLLtCIl/iUBDW1dUhPz8fubm5KC8vn/D88ePHcd999+HWW2/FgQMHbJ5bvHgxioqKUFRUhCeeeMI5W+2l9KtHjyCzKyTyJXbva2yxWFBWVoaKigqIooji4mLodDosWLDAukx8fDxefvllvPfeexPWDwoKQmVlpXO32kuNdoV1J5tw7KwJGUviPL1JROQAux2hwWBASkoKkpKSoFarodfrUV1dbbNMYmIiFi1aBIWCe9rsCol8j92O0GQyIS5urLMRRREGg8HhAgMDA7j//vuhUqnw2GOP4e677552+cjIEKhUSodff9R0N292h9H6MTFhyFmZjI+/uIizjR344W1Jbq3vCd7y3suxvpzH7sz6doNwrmprayGKIoxGIzZu3IiFCxciOTl5yuXb23tnXGOmd7x3tu/Xv3t5AqqPX8KuA99gcWIEFArBrfXdydveeznVl/PYZ1N/utC0uy8riiKam5utv5tMJoii6HDx0WWTkpKwatUqnDlzxuF1fdU8bTCy0uNhauvFFzyCTOT17AZheno6GhoaYDQaYTabUVVVBZ1O59CLd3R0wGw2AwDa2tpw4sQJm4Ms/mzt6hTOFRL5CLu7xiqVCqWlpSgpKYHFYsGGDRuQmpqKbdu2IS0tDTk5OTAYDHjyySfR2dmJ2tpavPPOO6iqqsL58+exZcsWCIIASZLw6KOPyiYIR7vCupNX8MVZE1bzCDKR1xIkSfKqdmU2cw7eOldxtaMPv/4/RzFPG4zfl6yC0kVH1TlPJc/6ch77bOrPaY6QZm9eRDCy/3N4rvDYGX5fIZG3YhC6mH50rvBwAyxDQ57eHCKaBIPQxdgVEnk/BqEbWLvCQxfYFRJ5IQahG8yLCMYd/xkPU3sfvjjD8wqJvA2D0E0KRrrCfYc4V0jkbRiEbsKukMh7MQjdaPSbadgVEnkXBqEbRUcE4Y6lCTC19+HoaXaFRN6CQehm+oyRuUKeV0jkNRiEbjbaFbawKyTyGgxCD2BXSORdGIQewK6QyLswCD1k9PsK2RUSeR6D0EOiwoOwhl0hkVdgEHqQnlebEHkFBqEHWbvC6+wKiTyJQehh+tUpUCnZFRJ5EoPQw6LCR44gX+/DkVPsCok8gUHoBfQZw13hfh5BJvIIh4Kwrq4O+fn5yM3NRXl5+YTnjx8/jvvuuw+33norDhw4YPPc3r17kZeXh7y8POzdu9c5W+1n2BUSeZbdILRYLCgrK8O7776Lqqoq7N+/H+fOnbNZJj4+Hi+//DLWrl1r8/j169exfft2fPDBB9i9eze2b9+Ojo4O547AT7ArJPIcu0FoMBiQkpKCpKQkqNVq6PV6VFdX2yyTmJiIRYsWQfG921XW19cjKysLWq0WERERyMrKwueff+7cEfiJ8UeQD59q9vTmEMmK3SA0mUyIixu7ObkoijCZHNt9m8u6clQwrisctLArJHIXlac34PsiI0OgUilnvN50N292B2fUj4kJQ37GTag6dAGnL13H3atS3Fp/tvzhvffV+nIeuzPr2w1CURTR3Dy2q2YymSCKokMvLooijh07ZrPuqlWrpl2nvb3Xodceb6Z3vHc2Z9a/a2k8Dh5twF8PfoMlyVqolPaPZ3ly/P703vtafTmPfTb1pwtNu//K0tPT0dDQAKPRCLPZjKqqKuh0OocKZ2dno76+Hh0dHejo6EB9fT2ys7Md3nA5Gp0rbL3ejyOnOVdI5A52g1ClUqG0tBQlJSUoKCjAvffei9TUVGzbts160MRgMGDNmjU4cOAAtmzZAr1eDwDQarXYtGkTiouLUVxcjM2bN0Or1bp2RH5Av/om69UmnCskcj1BkiTJ0xsx3mxabV9r0R3xl4+/RfWJRjx87yLcsTTB7fUd5Y/vva/Ul/PYZ1N/TrvG5BkFo9cg8wgykcsxCL1UZFgg7lw6H1c7+nGE5xUSuRSD0IsNd4UKdoVELsYg9GKRYYG4c1kCrnb082oTIhdiEHq54atNFLzahMiFGIRejl0hkesxCH0Au0Ii12IQ+oDIsED8kF0hkcswCH3EvewKiVyGQegj2BUSuQ6D0IewKyRyDQahD2FXSOQaDEIfU7A6BQEqBb+ZhsiJGIQ+RqsZPq/wWmc/Dv2/Jk9vDpFfYBD6oIKM4a5w/+GL7AqJnIBB6IPYFRI5F4PQR411hQ0w37B4enOIfJrX3cWOHKPVBOKHy+bjk38a8T//938hPjoESTEaJMZqkDjyd0So2tObSeQTGIQ+rCj7ZliGhnCppRsXm7twydRt83x4SADmx2iQZA3HUCREh0IdMPPbpRL5MwahDwsJUuF/5f0HYmLCYDJ1wtTei8bWHjS2dKOxtRvGlm6cvdiOsxfbresIAhAXFTIckDGhSIzVIClGg+iIIAiC4MHREHkOg9BPKBQC4qNDER8dipWLYq2P9w0M4vLV4XA0tnbjcks3jK09aLrWgn9+M7Z+kFpp3aVOjAkd/jlGg5AgfkTI/zn0Ka+rq8PWrVsxNDSEBx54AI899pjN82azGb/85S9x+vRpaLVavPnmm0hMTERjYyMKCgpw8803AwCWLl2KsrIy54+CphQcqMKC+RFYMD/C+pgkSWjrHEBj61jneLm1B/++0olzlzts1o8ODxwXkMN/x0UFQ6ngcTbyH3aD0GKxoKysDBUVFRBFEcXFxdDpdFiwYIF1md27dyM8PByffPIJqqqq8Nprr+Gtt94CACQnJ6OystJ1I6AZEwQB0RFBiI4IwtIF86yP3xgcQtO1HmswGlu70djSjZPnr+Hk+WvW5VRKBRLmhSAxRoNFN0cjMiQAiTGhCA9Vc/eafJLdIDQYDEhJSUFSUhIAQK/Xo7q62iYIa2pq8OSTTwIA8vPzUVZWBi+7XTI5IEClQLIYhmTR9v6vnb1m6y5140g4Xr7ag0umbptrnsNCAqy71ImxoUiK1fDgDPkEu0FoMpkQFxdn/V0URRgMhgnLxMfHD7+gSoWwsDC0tw9P0Dc2NmL9+vXQaDR45plnsGLFimnrRUaGQKWa+T+c6W7e7A7+XD8GwA9Som0eswxJaLrajYtNXbjQ1IGGK5242Nw54eCMQgDi52lwU0I4boof+xMbGQKFwjndoz+/995c25/qu3QmPDY2FrW1tYiMjMSpU6ewefNmVFVVQaPRTLlOe3vvjOvM9I73zibX+oECkLU0AQsTwoDbhh+zHpwZ6RwbRzrJy63dOHTyinXdILUS82NCbc99jAlFSFDAjLZBru+9p2v7Yv3pQtNuEIqiiObmsd0fk8kEURQnLNPU1IS4uDgMDg6iq6sLkZGREAQBavXwSb1paWlITk7GhQsXkJ6e7vDGk2+Z6uBMe9cAjCOn9Yye4nPhShfOX+60WT86PND23MeYUIhRIVApeXCGXMduEKanp6OhoQFGoxGiKKKqqgqvv/66zTI6nQ579+7F8uXLcfDgQWRkZEAQBLS1tSEiIgJKpRJGoxENDQ3WuUaSD0EQEBUehKjwyQ/ODHePIwdnWrthOH8NBpuDMwISokPHAjJ2+PQeXjlDzmI3CFUqFUpLS1FSUgKLxYINGzYgNTUV27ZtQ1paGnJyclBcXIznnnsOubm5iIiIwJtvvgkAOH78ON5++22oVCooFAq8+OKL0Gq1Lh8U+QZ7B2caxx25vny1B5daunHk9NhymuAAJIlh0ASpEBkWaP0TFRYEbZgaWk0gO0lyiCB52eHd2cw5+NpchT/Vd1ftoSEJLdf7huccrbvY3bjWOYChock/wgKA8FC1TUiOBWUgosICoQ0LROAcjmrL4b33l/pzmiMk8gYKhYC4qBDERYVgxbgrZ6KiNTjfcA3tXQNo7+of+dv2T2NrDxqap/4HE2rtKIO+F5iB1sAMDlTxHEk/xiAkn6ZUCNbgAsInXUaSJHT33Zg0JNu7+tHWNYBrnf1obO2Zsk5ggHJCZxkZFoib5muhlCREhgVCExIABcPSJzEIye8JgoCwEDXCQtQT5iPH6xsYHA7H7gG0d07sMNu6BtDcNvXpXeNDeexPkE1nGaFR8/JEL8QgJBoRHKhCcKAKCfNCp1zmxqAF7d1mtHcOh+QNCTA2dVqD8nr3AM5d7sBUM++CMDxvGTXNrnikJpBX47gZg5BoBgJUSsRqgxGrDQYw+YS9ZWgIHd3mcZ3lwLhOc3hX3NjSjQtNU89baoIDJnaXmkBEho91meQ8DEIiJ1MqFNbzJpEw+TKSJKGr78ZYUHaP7Ip3jv48gJb2Phhbuid/AQCBaiWC1EoEq1UIDlQiSD3c0QarlQgKHH4sWK0a/nn0MbUSwYEqm595ihGDkMgjBEFAeIga4SFqpMRNPm8pSRL6BiyThmR71wB6BgbR1WNGb/8NXOvsx43B2d3RUKVUjAvN0WAd+9nmsZHwDFYrcb1/EP29ZmvIqlUKnz2yziAk8lKCICAkSIWQIBXmTzJv+f3d8kHLEPrNFvQNDKJvYHDsZ/Mg+gcs6DMPom/Agv7JHjMPr9PRa8aAeXY3A1MIwrjOdLQDVdk8Nr5DtQ3bsccC1Uq3H31nEBL5CZVSAU2wAprgmX1xxfcNDUnoN4+FY595NDyHg3X0Z0GpQNv1XvSNBGq/9e9BtHUOoM/cM+VBI3tGO8/xHehosAYFKhESqELhnQvgrENKDEIisqFQjHWi07F3ZYckSTDfGBrpOsd1qAOTh2z/wPcfG0RX7w20tPfBMsnVQwFqFfS3J895vACDkIhcRBAEBKqVCFQrodXM7Sj3jcEha7fZN2DBwA0LbkuLR1dHn1O2lUFIRF4vQKVAgGr44NKoILUKzrrSmcfNiUj2GIREJHsMQiKSPQYhEckeg5CIZM/rvqGaiMjd2BESkewxCIlI9hiERCR7DEIikj0GIRHJHoOQiGTPp4Kwrq4O+fn5yM3NRXl5+YTnzWYznnnmGeTm5uKBBx5AY2OjW+v/7W9/Q0ZGBoqKilBUVITdu3c7rfavf/1rrF69GmvXrp30eUmS8Pvf/x65ubkoLCzE6dOn3Vb7iy++wG233WYd9/bt251WGwCamprw0EMPoaCgAHq9Hn/6058mLOOq8TtS25XjHxgYQHFxMdatWwe9Xo+33357wjKu/Nw7Ut+Vn3sAsFgsWL9+PR5//PEJzzlt7JKPGBwclHJycqRLly5JAwMDUmFhofTdd9/ZLLNr1y7pt7/9rSRJkrR//37p6aefdmv9PXv2SC+++KLTao537Ngx6dSpU5Jer5/0+U8//VR65JFHpKGhIemrr76SiouL3Vb76NGj0mOPPea0et9nMpmkU6dOSZIkSV1dXVJeXt6E995V43ektivHPzQ0JHV3d0uSJElms1kqLi6WvvrqK5tlXPm5d6S+Kz/3kiRJ7733nvTss89O+h47a+w+0xEaDAakpKQgKSkJarUaer0e1dXVNsvU1NTgvvvuAwDk5+fjyJEjkJx0vrgj9V1p5cqViIiImPL56upqrF+/HoIgYNmyZejs7ERLS4tbartabGwslixZAgDQaDS45ZZbYDKZbJZx1fgdqe1KgiAgNHT4a/oHBwcxODg44b4grvzcO1LflZqbm/Hpp5+iuLh40uedNXafCUKTyYS4uDjr76IoTvhAmkwmxMfHAwBUKhXCwsLQ3t7utvoA8PHHH6OwsBBPPfUUmpqanFJ7NtsXFxfn1n+wX3/9NdatW4eSkhJ89913LqvT2NiIs2fPYunSpTaPu2P8U9UGXDt+i8WCoqIiZGZmIjMzc9Kxu+pz70h9wHWf+5deegnPPfccFIrJo8pZY/eZIPQFd911F2pqarBv3z5kZmbiV7/6lac3yS2WLFmCmpoafPTRR3jooYewefNml9Tp6enBU089hRdeeAEajcYlNWZT29XjVyqVqKysxGeffQaDwYBvv/3Wqa8/1/qu+tzX1tYiKioKaWlpTnm96fhMEIqiiObmZuvvJpMJoihOWGb0f6PBwUF0dXUhMjLSbfUjIyOhVg9/g+4DDzzg1AMWM92+5ubmCdvnKhqNxrr7dOedd2JwcBBtbW1OrXHjxg089dRTKCwsRF5e3oTnXTl+e7XdMX4ACA8Px+23347PP//c5nFXfu4dqe+qz/2JEydQU1MDnU6HZ599FkePHsUvfvELm2WcNXafCcL09HQ0NDTAaDTCbDajqqoKOp3OZhmdToe9e/cCAA4ePIiMjAynzWc4Un/8nFRNTQ1+8IMfOKW2I3Q6Hf7+979DkiR8/fXXCAsLQ2xsrFtqt7a2WudlDAYDhoaGnPoPUZIk/OY3v8Ett9yChx9+eNJlXDV+R2q7cvxtbW3o7OwEAPT39+Pw4cO45ZZbbJZx5efekfqu+tz//Oc/R11dHWpqavDGG28gIyMDr732ms0yzhq7z9yzRKVSobS0FCUlJbBYLNiwYQNSU1Oxbds2pKWlIScnB8XFxXjuueeQm5uLiIgIvPnmm26t/+c//xk1NTVQKpWIiIjAyy+/7LT6zz77LI4dO4b29nasWbMGP/3pTzE4OAgAePDBB3HnnXfis88+Q25uLoKDg/HSSy+5rfbBgwfx/vvvQ6lUIigoCG+88YZTJ9S//PJLVFZWYuHChSgqKrJu05UrV6zb4KrxO1LbleNvaWnB888/D4vFAkmScM899+Cuu+5y2+fekfqu/NxPxhVj59dwEZHs+cyuMRGRqzAIiUj2GIREJHsMQiKSPQYhEckeg5CIZI9BSESyxyAkItn7//YqZlgvsk22AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAADSCAYAAAA/vMlrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XlUlOfZP/DvDMOwDavigMqigkvdFRUFsaLIFhyNmNq38fVXa5OoqVncGpvEU5uYWItp0/R9W48n/vprT2yzGBHQuKGQKIuokaBGxSo7gwiyDMx+//4AJowsgzDPbM/1OYcj8DzMdd9zhsvruR/mvgSMMQZCCOExobUHQAgh1kaJkBDCe5QICSG8R4mQEMJ7lAgJIbxHiZAQwnuUCAkhvEeJkBDCe5QIicNhjEGv11t7GMSOUCIknDl48CCWLl2KmTNnIikpCWfOnDEc+/TTT5GYmGg4duPGDQBATU0NXn75ZURGRmLevHnYs2cPAODPf/4ztm3bZvj5yspKTJgwAVqtFgCwdu1afPDBB1izZg2mT5+OiooKfPHFF4YYS5Yswb/+9S+j8Z09exYymQyzZs3C0qVLkZubi5MnT+LZZ581Ou/w4cPYuHEjJ88RsRGMEI6cOHGC1dbWMp1Ox7Kystj06dOZXC5nJ06cYNHR0ez69etMr9ezBw8esMrKSqbVallKSgp79913mUKhYEqlkl2+fJkxxtiHH37Itm7danjsiooKNn78eKbRaBhjjD3//PNs0aJF7M6dO0yj0TC1Ws3Onz/PysrKmF6vZwUFBWzatGmspKSEMcbY9evX2axZs9g333zDdDodq62tZaWlpUylUrE5c+aw0tJSQyyZTMa++uorCz5zxNKoIiScSUxMhFQqhVAoRFJSEkJCQlBcXIzPP/8cGzZswLRp0yAQCBASEoJRo0ahuLgYdXV12LFjB9zd3eHi4oKIiIgBx1u5ciXCw8MhEong7OyMH//4xwgODoZAIMDcuXMRFRWFoqIiAMDnn3+OVatWISoqCkKhEFKpFOPGjYNYLEZiYiKOHz8OALh79y6qqqqwePFiTp4jYhsoERLOHDt2DDKZDBEREYiIiMDdu3fR2NiImpoaBAcH9zi/pqYGI0eOhEgkGlS8wMBAo69zcnLw3HPPYe7cuYiIiEBubi4aGxsNsXobA9CRUDMyMsAYQ3p6OhITEyEWiwc1JmIfKBESTlRVVeHNN9/EW2+9hYKCAhQVFSE8PBxAR8IqLy/v8TOBgYGoqakxrPt15+bmBqVSafi6vr6+xzkCgcDwuVqtxpYtW7B+/XpcvHgRRUVFiImJAevcbKmvMQDAjBkz4OzsjKKiImRmZmL58uVPN3lidygREk60t7dDIBDAz88PAPDFF1/g7t27AIDU1FR8/PHHKCkpAWMMZWVlqKqqwrRp0+Dv74+0tDS0tbVBpVLhypUrAIBJkybh8uXLqK6uRktLC/72t7/1G1+tVkOtVsPPzw8ikQg5OTm4ePGi4XhqaiqOHj2KvLw86PV6yOVy3Lt3z3B8xYoV2LNnD0Qi0VNdnhP7RImQcCIsLAzr16/HmjVrsGDBAty5cwezZs0C0LF2+NJLL2Hr1q2YNWsWNm/ejKamJjg5OeGvf/0rysrKsHjxYsTExODkyZMAgKioKCQlJWH58uV49tlnTa7ZSSQSvPnmm3j11VcxZ84cZGZmIjY21nB82rRpeO+997B3717Mnj0bzz//PKqrqw3HZTIZ7t69S9UgTwgYo41ZCXmSUqnE/Pnz8eWXXyI0NNTawyEco4qQkF4cOXIEU6dOpSTIE4O7PUeIA4uNjQVjDH/5y1+sPRRiIXRpTAjhPbo0JoTwHiVCQgjv2dwa4cOHLU/9M76+7mhsbONgNBTflmPzPT6f5z6Y+P7+nn0ec4iKUCRyovg8jM33+Hyeu7njO0QiJISQoaBESAjhPUqEhBDeo0RICOE9m7trTAhxfHo9g0qjg1Ktg7rzX5Wm86Pzc8P3ejmm0erxXNwEjPZzM8t4KBESQvqk0+uhUuuNEpFSrYVKo4dLdTMe1is6k1bH93pLWr0lO4126M215te1UCIkhPxAq9P3X1l1//6TycnomB4qtbbzcfTQ6oaesEROQrg4C+EqdoKnuzP8xa5wcXbq+BAb/+va9fUT3xN3P9b5b2CA96D+7rjXMZrlUQghZqVUa9GkUKOpVY1mhbrjc4UKTa1dn6uhUGrRrtRApdFBqxv6lgHOIqEhCflIXHpJPkK4OIsMnw/384BapYGrWNT5c0K4ikUQd/7r4iyE2NkJIifbvxVBiZAQC9Hq9D8ktdbOxNb5dXNrt2SnUEOt6b8SE4uE8PZ0gY+niyF5PVkx9VppdTvWvdISOwvhJHy6hOXv72m2iszaBpQIc3Nz8e6770Kv12P16tV44YUXjI5XVVVh165daGhogI+PD/bv34+AgAAAwP79+5GTkwMA2LRpE5KSksw8BUKsR69naG7rmciMKzk1mlpVUCh79mLpTigQwMvDGYF+HvCWiOHlIYZ314fExfC5l4cYrmInjBjh5TCJyNpMJkKdToc9e/bg8OHDkEqlSE1NRWxsLMLCwgzn7Nu3DytWrMDKlSuRl5eHtLQ07N+/HxcuXMDNmzdx7NgxqNVqrF27FjExMZBIJJxOipChYIxBqdb1SGSG6q1bRdfcpoFe3/9lqYerCN4SFwRLPQ2JzFvSleQ6EpyXRAyJmzOE3RpQEcsxmQiLi4sREhKCoKAgAEBycjLOnTtnlAjv3buHN954AwAQGRmJzZs3AwBKS0sREREBkUgEkUiECRMmIDc3l6pCYhVmvzSViDEh2BfuYid4SbpVbx4uhkTn6S6Gs8j218j4zmQilMvlhstcAJBKpSguLjY6Z+LEiTh9+jTWrVuHM2fOQKFQoLGxERMnTsRHH32E9evXo729HQUFBUYJtDe+vu6DejN1fztLWAKf41sztl7PIHYTo7FFhcZmJRpbVHjcouz8WoXGlh++19Km6fexhEIBfCQuCJJ6wtfTFb6da3C+nq7w9XIx+p6bi8iofai18Pl1Z874ZrlZsmPHDvzud7/Dl19+iYiICEilUjg5OSE6Ohrfffcd1qxZAz8/P8yYMQNCEwuyg9nWx9qLtnyOb6nYjDHUNylRLm9FRV0LKupaUfmwFY+aVQO+NB3tLzHDpSmDokUJReeU+fDcO0r8/pKmyUQolUpRW1tr+Foul0MqlfY456OPPgIAKBQKnD59Gl5eXgCAjRs3YuPGjQCArVu3YsyYMQMeOOEnlUaHqocKQ8LrSnrtKp3ReRI3Z4wP8oGHi4guTcmQmEyEU6dOxYMHD1BRUQGpVIqsrCykpaUZndN1t1goFOLgwYNYtWoVgI4bLc3NzfD19cX333+P27dvIyoqipuZELvDGENji8qQ7Lo+5I1t6N5JRyAAAvzcMXWsBEEjJAga4YmgERL4SMR055SYhclEKBKJ8Pbbb2PDhg3Q6XRYtWoVwsPD8ac//QlTpkzBkiVLUFhYiAMHDkAgECAiIgK7d+8GAGi1WvzsZz8D0NFwe//+/RCJ6E8X+Uij1aO6XtEt4XVUe0/+SYmbiwjho306E17Hx6jhHhA7W3cTUOLYbK6L3WD+d7e3tQpHit9b7CaF2uiytqKuFbWP2qDrtpYnAODv62aU8IJGSDDMy/WpbkLQc8/PuQ8m/pDWCAnpi1anR1lNM67flhslvWaF2ug8F2cnhAZ6Gi5pg0ZIMNrfA65ievkR20CvRDIgre2aHpe11fWKHu9xHeblihlhw3+o8qQS+Pu40R8KE5tGiZAY0esZ5I1tPW5gNLaojM5zFgkx2l+C8GBf+Hu5GBKfu6uzlUZOyOBRIuSxdpW2R8Krqm/t8a4KH4kYU8cOM1rLk/q5wUkotPo6ESHmQImQBxhjeNikREW3P0auqGtFfZPS6DwnoQAjh3v0uIHh6S620sgJsQxKhA5GpdGh8qFxlVdZ1wql2viPkT3dnfGjUN9uCc8TgcPc7WLvOELMjRKhnbtV1ojs69X4/n4DKupaUdfQhu63L4QCAQKGufeo8rw9xDbxXllCbAElQjv2TXENPj5xy/C1u4sI44N8jO7YjhxGf4xMiCmUCO2UVqfH8Yv3IXISYvvzs+Hn7gw/Lxeq8ggZBFoQslOXSmpR36TEohkjsWDaSAzzfrp3ZBBCfkCJ0A5pdXpkXnoAkZMASZEh1h4OIXZvQIkwNzcX8fHxiIuLw8GDB3scr6qqwrp165CSkoK1a9cabdv1+9//HsnJyUhMTMQ777wDG3trs13Ku9FRDcZMHwlfTxdrD4cQu2cyEXb1LDl06BCysrKQmZmJ0tJSo3O6epZkZGRg06ZNhm26rl69iqtXr+L48ePIzMzEd999h8LCQm5mwhM6vR5Zl8qoGiTEjEwmwu49S8RisaFnSXf37t1DZGQkgI6eJV3HBQIB1Go1NBqN4d/hw4dzMA3+yL8hR93jdiycNhJ+Xq7WHg4hDsFkIuytZ4lcLjc6p6tnCQCjniUzZ87EvHnzEB0djejoaCxcuBDjxo0z8xT4Q6fXI+PSAzgJqRokxJw47VlSVlaGe/fuGfoar1+/HkVFRYiIiOjzsah5U9+yiypQ19iOxPmhmBjmb/H4feHDc2+r8fk8d3PG57Rnyaefforp06fDw8MDALBw4UJcu3at30RIzZt6p9czfHLqezgJBYidMdIoHm0Oys/4fJ77YOL3lzRNXhp371miVquRlZWF2NhYo3MaGhqg13fsWNK9Z8nIkSNx+fJlaLVaaDQaXL58mS6NB6nglhzyhjZETwvEMG9aGyTEnDjtWRIfH4/8/HykpKRAIBBg4cKFPZIoMU2vZ8i42LE2mExrg4SYHfUsMQOu4+ffrMXB4zcRMz0Q/ydxksXj98fRn3tbjs/nuQ8m/pAujYl1GVWD80OtPRxCHBIlQhtXdLsONY/aMH9KAPx93Kw9HEIcEiVCG6ZnHdWgUCDAMwtCrT0cQhwWJUIbduX2Q1TVKzB/ihQjqBokhDOUCG2UnjEcv3ifqkFCLIASoY26evshqh4qEDlZCqmvu7WHQ4hDo0RogzqqwQcQCIAUqgYJ4RwlQht07U49Kh+2IvJHUkj9qBokhGuUCG0MYwwZF+9DIACtDRJiIZQIbcy3d+tRXteKeZOkCBzmYe3hEMILlAhtCGMM6RfvQwCqBgmxJEqENuR66SOUy1sxZ9IIjBxO1SAhlsJp86b8/HzIZDLDx9SpU3H27FnzzsBBdK8GU6LGWHs4hPCKyW24upo3HT58GFKpFKmpqYiNjUVYWJjhnK7mTStXrkReXh7S0tKwf/9+REZGIj09HQDw+PFjLFu2DFFRUdzNxo4V33uEstoWzJk4AqOoGiTEojht3tTdqVOnsHDhQri50VvFnsQ630UCAClRodYdDCE8xGnzpu6ysrLwzDPPmGPMDue7/zTgfk0LIib4Y7S/xNrDIYR3OG3e1KWurg537txBdHS0ycfiW/MmxhhOfnINALAuZcqgH4caCPEzPp/nbs74nDZv6nLy5EnExcXB2dnZ5ID41ryp5D+PcLu8EbPH+8NDJLC7Hbrt+bm39/h8nvtg4luteVOXrKwsJCcnD3jAfNF1pxigtUFCrMlkIuzevCkpKQmJiYmG5k1dN0UKCwuRkJCA+Ph41NfXY+PGjYafr6ysRE1NDebOncvdLOzUzQeNuFfVjJnhwxEste4lBiF8NqA1wkWLFmHRokVG33vllVcMnyckJCAhIaHXnx09ejS+/vrrIQzRMXWvBpfT3w0SYlX0zhIruVXWiNLKJswIG46QAKoGCbEmSoRWwBjD8W86q8HoUOsOhhBCidAavi9/jDuVTZg+bhhCA7xM/wAhhFOUCK0g3VAN0togIbaAEqGFfV/WiDsVjzFt3DCMCaRqkBBbQInQwo7TnWJCbA4lQgu6Xd6I78sfY8pYP4wdSdUgIbaCEqEFHb/4AAAgo2qQEJtCidBC7lQ8xq2yRkwe44dxo7ytPRxCSDeUCC2ka22QqkFCbA8lQgsorWzCzQeN+FGoL8JGUzVIiK2hRGgB9J5iQmwbp82bAKC6uhrr169HYmIikpKSUFlZab7R24F7VU24cb8Bk0J8MT7Ix9rDIYT0gtPmTQCwc+dOvPTSS4iKioJCoYBQyK8i9IdqMNS6AyGE9InT5k2lpaXQarWGznUeHh68at70n+pmlPynARODfTAh2NfawyGE9MFkRdhb86bi4mKjc7qaN61bt86oedODBw/g5eWFl19+GZWVlZg/fz62bdtm1M/kSY7Us+R/0m8AAP47eTLn46O+GfyMz+e5mzM+p82btFotioqKcOzYMQQGBuK1117D0aNHsXr16j4fy1F6ltyvaUbRLTnGB/kgwNuF0/FR3wx+xufz3AcTv7+kyWnzpoCAAEyaNAlBQUEAgCVLluD69esDHrg969pvUEZrg4TYPE6bN02dOhXNzc1oaGgAABQUFBjdZHFUD2qbcf3eI4SP9sbEEFobJMTWmawIuzdv0ul0WLVqlaF505QpU7BkyRIUFhbiwIEDEAgEiIiIwO7duwEATk5O2LlzJ9atWwcAmDx5cr+XxY7i+DcPAHTsNygQCKw7GEKISQLGGLP2ILqzt76+T8Yvq23Bb//vZYSN8sYbz8+ySCKkdSp+xufz3AcTf0h9jcnTMew3GB1K1SAhdoISoRmVy1tw7W49xo30wuRQP2sPhxAyQJQIzSijc79BWhskxL5QIjSTirpWXLnzEGMCvTBlDFWDhNgTSoRmktG13yCtDRJidygRmkFZTTOKbj9EaIAnpo4dZu3hEEKeEiVCM/jXmdsAABmtDRJilygRDlFVvQIXi6sRGuCJaeOoGiTEHlEiHKKMi/fBWMfu01QNEmKfKBEOQXW9Apdv1WHcaG9MD6NqkBB7RYlwCDIvPQADsCZuAlWDhNgxznuWTJo0CTKZDDKZDC+99JL5Rm5lNY8UKLglR/AICeZNDjD9A4QQm8V5zxJXV1ekp6dzNwMrybz0AIwBKbQ2SIjd47RniaOqbWhD/k05RvtLMHP8cGsPhxAyRCYTYW89S+RyudE5XT1LABj1LAEAlUqFZ599Fs899xzOnj1rzrFbTVc1uDwqFEKqBgmxe5z2LAGA8+fPQyqVoqKiAuvWrcP48eMRHBzc52PZevOm6vpW5N+UIyTAE/FRYyEUCiwavy/UQIif8fk8d3PG57RnSdcxAAgKCsLcuXNx8+bNfhOhrTdv+n9ZN6HXMyRFhuDRo1aLx+8NbQ7Kz/h8nvtg4g9pY9ah9CxpamqCWq02nHP16lW77llS19iGvBI5Rg33wOwJ/tYeDiHETDjtWXLv3j3s3r0bAoEAjDH88pe/tOtEmJlXBj1jSKG1QUIcCvUsGaCHj9ux62A+Rvi64Xe/mGdYG7RU/P7Q5Rk/4/N57oOJTz1LzCAr7wF0+s5qUEjVICGOhBLhANQ/bsfF72oROMwdcydKTf8AIcSuUCIcgKz8Muj0DM8soGqQEEdEidCER01KfFNcA6mfO+ZNomqQEEdEidCErmowZUEIVYOEOChKhP1oaFbi6+vVGOHrhnk/omqQEEdFibAfP1SDoXAS0lNFiKOi3+4+GKpBHzdETqZqkBBHRomwDyfyy6DVMSQvCKFqkBAHR7/hvWhsUSH3ejWGe7tiPu0+TYjDo0TYi65q8JkFoRA50VNEiKOj3/InNLaokPNtRzW4YApVg4TwAefNmwCgtbUVMTEx2LNnj3lGzaGTBWXQ6vRInh9C1SAhPGHyN72redOhQ4eQlZWFzMxMlJaWGp3T1bwpIyMDmzZtQlpamtHxP/7xj5gzZ455R86Bx60d1eAwLxdETQ209nAIIRbCefOmkpISPHr0CFFRUWYeuvl9VVAOjVaP5Pm0NkgIn3DavEmv12Pfvn3YuXOnmYdtfk0KNS5cq4Kflwuip1E1SAifcNq86ZNPPkFMTIxRIjXFWs2bjueVQK3V4ydxExAY4G3x+ENFDYT4GZ/PczdnfE6bN127dg1XrlzBkSNHoFAooNFo4O7ujm3btvUZzxrNm5oVapy4eB++ni6YMcbvqR/L3nbqdZTYfI/P57kPJn5/SdNkIuzevEkqlSIrK6vHzZCGhgb4+PhAKBQaNW/qft7Ro0dRUlLSbxK0lq8Ky6HW6rE6MgTOIlobJIRvTP7Wd2/elJSUhMTEREPzpq6bIoWFhUhISEB8fDzq6+uxceNGzgduLs1tamRfrYSPRIyY6bQ2SAgf8b5502cXSnEyvxz/tTQcSyOCBvUY9naJ4Cix+R6fz3MfTHxq3tSHljY1sq9UwVsixqIZI609HEKIlfA6EZ6+XAGVRoekeSFwHsSdakKIY+BtImxt1+DslUp4e1A1SAjf8TYRnr5cDpVah8R5wRA7UzVICJ/xMhG2tmtwtqgSXh5iLJo5ytrDIYRYGS8T4ZnLFVCqdUiYGwwXqgYJ4T3eJUKFUoOzVyrg6e6MxVQNEkLAw0R45nIF2lU6JMwLhouYqkFCCM8SYZtSgzNFlZC4OSN25mhrD4cQYiN4lQjPFlWiXaWlapAQYoQ3ibBNqcXpyxUd1eAsWhskhPyAN4nw3JUKtKm0iJ8bBFexWbZhJIQ4CE6bN1VVVWHlypWQyWRITk7GkSNHzDv6AWpXdVSDHq4ixM6itUFCiDGTpVFX86bDhw9DKpUiNTUVsbGxCAsLM5zT1bxp5cqVyMvLQ1paGvbv3w9/f3/8+9//hlgshkKhQEpKCmJjY3ts7Mq1c1cqoVBqsTJmLNxcqBokhBjjtHmTWCyGWCwGAKjVauj1enOP36R2lRanCsvh4SrC0tlUDRJCejJZHvXWvKm4uNjonK7mTevWrTNq3uTr64uamhq88MILKC8vx44dO0xWg+buWfLZuTtQKLX4WcJEBI/2ferHHWp8S6G+GfyMz+e5mzM+p82bACAwMBAZGRmQy+XYvHkz4uPjMXz48D4fy5w9S5RqLY6eL4W7iwjzJ47gbBNJe9ug0lFi8z0+n+c+mPhD6lkylOZNT54THh6OoqIiJCQkDHjwQ3H+ahVa2zWQRY+BuyutDRJCemdyjbB78ya1Wo2srCzExsYandPQ0GBY/+vevKm2thZKpRIA0NTUhKtXr2LMmDHmnkOvVGodviosh5uLCHERtDZICOmbyTKpe/MmnU6HVatWGZo3TZkyBUuWLEFhYSEOHDgAgUCAiIgI7N69G0DHTZT3338fAoEAjDGsX78eEyZM4HxSAHD+WhVa2jRYHhUKd1dni8QkhNgnh2zepNLosPN/L0Gj0+P3GxfAg+NEaG9rJY4Sm+/x+Tz3wcTnXfOmC9eq0NymwdLZQZwnQUKI/XO4RKjS6HCyoByuYifEzRlce05CCL84XCLM+bYazQo1lkaMhsSNqkFCiGkOlQjVGh1O5pfBReyEZXOCrT0cQoidcKhEmHO9Gk0KNZbOpmqQEDJwDpMINdrOatDZCctobZAQ8hQcJhHmXq/B41Y1YmePgqe72NrDIYTYEYdIhBqtDifyyyB2FiJ+Lq0NEkKejkMkwtMF5WhsUSF21mh4UTVICHlKdp8INVo9Pj93B2JnIRKoGiSEDILdJ8JvvqtBfZMSi2eOgpcHVYOEkKdn94mwtPIx3FxESJgXYu2hEELsFKfNm27duoWf/OQnSE5ORkpKCk6cOGHe0QP4Wdx4/M+OWHhTNUgIGSSTibCredOhQ4eQlZWFzMxMlJaWGp3T1bwpIyMDmzZtQlpaGgDA1dUV+/btQ1ZWFg4dOoS9e/eiubnZrBNwd3XGcB83sz4mIYRfOG3eNGbMGISGhgLo2KHaz88PDQ0NZp4CIYQMDefNm7oUFxdDo9EgOLj/O7vmbt5kKXyOz+e5Wzs+n+duzvicN28CgLq6Omzfvh379u2DUNh/EWrO5k2Wwuf4fJ67tePzee6DiW/V5k2tra148cUX8dprr2HGjBkDHjQhhFgKp82b1Go1Nm/eDJlMZrHOdYQQ8rQG1LMkJycHe/fuNTRv2rhxo1Hzpq+++qpH8yaxWIz09HTs2rULYWFhhsd6//33MWnSJE4nRQghT8PmmjcRQoil2f07SwghZKgoERJCeI8SISGE9ygREkJ4jxIhIYT37CoRmtoFR61W49VXX0VcXBxWr16NyspKi8Y/evQoIiMjIZPJIJPJ8Nlnn5kt9htvvIH58+fjmWee6fU4YwzvvPMO4uLikJKSghs3blgsdkFBAWbPnm2Yd9cf15tLTU0N1q5di6SkJCQnJ+Pvf/97j3O4mv9AYnM5f5VKhdTUVCxfvhzJycn48MMPe5zD5et+IPG5fN0DHRu/rFixAi+++GKPY2abO7MTWq2WLVmyhJWXlzOVSsVSUlLY3bt3jc755z//yd566y3GGGOZmZnslVdesWj8L774gv32t781W8zuCgsLWUlJCUtOTu71+IULF9gvfvELptfr2bVr11hqaqrFYufn57MXXnjBbPGeJJfLWUlJCWOMsZaWFrZs2bIezz1X8x9IbC7nr9frWWtrK2OMMbVazVJTU9m1a9eMzuHydT+Q+Fy+7hlj7OOPP2avv/56r8+xueZuNxXhQHbByc7OxsqVKwEA8fHxyMvLAzPTn0kOJD6X5syZA29v7z6Pnzt3DitWrIBAIMCMGTPQ3NyMuro6i8Tm2ogRIzB58mQAgEQiwdixYyGXy43O4Wr+A4nNJYFAAA8PDwCAVquFVquFQCAwOofL1/1A4nOptrYWFy5cQGpqaq/HzTV3u0mEve2C8+QLUi6XIzAwEAAgEong6emJxsZGi8UHgNOnTyMlJQVbtmxBTU2NWWIPZnwBAQEW/YX99ttvsXz5cmzYsAF3797lLE5lZSVu3bqF6dOnG33fEvPvKzbA7fx1Oh1kMhkWLFiABQsW9Dp3rl73A4kPcPe637t3L7Zv397nZi3mmrvdJEJ7sHjxYmRnZyMjIwMLFizAzp07rT0ki5g8eTKys7Nx/PhxrF27Fps3b+YkjkKhwJYtW7Br1y5IJBJOYgwmNtfzd3JyQnp6OnJyclBcXIw7d+6Y9fGHGp+r1/358+fh5+eHKVOmmOXayvfVAAACJUlEQVTx+mM3iXCgu+B0/W+k1WrR0tJitCci1/F9fX0hFne0DFi9erVZb1g87fhqa2t7jI8rEonEcPm0aNEiaLVas2/Aq9FosGXLFqSkpGDZsmU9jnM5f1OxLTF/APDy8sK8efPw9ddfG32fy9f9QOJz9bq/evUqsrOzERsbi9dffx35+fnYtm2b0TnmmrvdJMKB7IITGxuLL7/8EgBw6tQpREZGmm09YyDxu69JZWdnY9y4cWaJPRCxsbE4duwYGGP49ttv4enpiREjRlgk9sOHDw3rMsXFxdDr9Wb9RWSM4Te/+Q3Gjh2Ln//8572ew9X8BxKby/k3NDQY2lsolUpcunQJY8eONTqHy9f9QOJz9brfunUrcnNzkZ2djQMHDiAyMhJ/+MMfjM4x19zNsjGrJYhEIrz99tvYsGGDYRec8PBwo11wUlNTsX37dsTFxcHb2xsffPCBReP/4x//QHZ2NpycnODt7Y333nvPbPFff/11FBYWorGxETExMfjVr34FrVYLAPjpT3+KRYsWIScnB3FxcXBzc8PevXstFvvUqVM4cuQInJyc4OrqatiJyFyuXLmC9PR0jB8/HjKZzDCm6upqwxi4mv9AYnM5/7q6Ovz617+GTqcDYwwJCQlYvHixxV73A4nP5eu+N1zMnXafIYTwnt1cGhNCCFcoERJCeI8SISGE9ygREkJ4jxIhIYT3KBESQniPEiEhhPcoERJCeO//A850xhWhxCGjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,3))\n",
    "plt.plot(history.epoch,history.history['loss'])\n",
    "plt.title('loss')\n",
    "\n",
    "plt.figure(figsize=(5,3))\n",
    "plt.plot(history.epoch,history.history['acc'])\n",
    "plt.title('accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 99.19%\n",
      "Saved model to disk\n",
      "CPU times: user 1.01 s, sys: 372 ms, total: 1.38 s\n",
      "Wall time: 1.28 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scores = model.evaluate(X_test, Y_test, verbose=2)\n",
    "print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))\n",
    "\n",
    "\n",
    "\n",
    "# serialize model to JSON\n",
    "model_json = model.to_json()\n",
    "with open(\"model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "# serialize weights to HDF5\n",
    "model.save_weights(\"model.h5\")\n",
    "print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Load the model\n",
    "#\n",
    "#with open('model.json','r') as f:\n",
    "#    json = f.read()\n",
    "#loaded_model = model_from_json(json)\n",
    "\n",
    "#loaded_model.load_weights(\"model.h5\")\n",
    "\n",
    "#loaded_model.compile(loss='categorical_crossentropy',\n",
    "#              optimizer='rmsprop',\n",
    "#              metrics=['accuracy'])\n",
    "\n",
    "#scores = loaded_model.evaluate(X_test, Y_test, verbose=2)\n",
    "#print(\"%s: %.2f%%\" % (loaded_model.metrics_names[1], scores[1]*100))\n",
    "\n",
    "#model=loaded_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#from keract import get_activations, display_activations\n",
    "\n",
    "\n",
    "def print_names_and_shapes(activations):  # dict\n",
    "    for layer_name, layer_activations in activations.items():\n",
    "        print(layer_name)\n",
    "        print(layer_activations.shape)\n",
    "        print('')\n",
    "\n",
    "\n",
    "def print_names_and_values(activations):  # dict\n",
    "    for layer_name, layer_activations in activations.items():\n",
    "        print(layer_name)\n",
    "        print(layer_activations)\n",
    "        print('')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#print_names_and_shapes(activations)\n",
    "#display_activations(activations)\n",
    "\n",
    "\n",
    "###get number of total conv_filters###\n",
    "total_filters = nb_filters*5\n",
    "#for layer_name, layer_activations in activations.items():\n",
    "#    if( layer_name.startswith('conv') ):\n",
    "#        total_filters += layer_activations.shape[3] \n",
    "\n",
    "\n",
    "###response matrix X : (example * filters)###       \n",
    "image_responses = np.empty((60000 ,total_filters))\n",
    "init_row = 0\n",
    "\n",
    "for i in range(30):\n",
    "    print(i)\n",
    "    activations = get_activations(model, X_train[init_row:init_row+2000,:,:])\n",
    "    init = 0\n",
    "    for layer_name, layer_activations in activations.items():\n",
    "        #print(layer_name)\n",
    "        if( layer_name.startswith('conv') ):\n",
    "            num_filters = layer_activations.shape[3]\n",
    "            with tf.Session() as sess:        \n",
    "                ###L2 norms of filter responses###\n",
    "                layer_activations = tf.norm(layer_activations, ord =2, axis=2)\n",
    "                layer_activations = tf.norm(layer_activations, ord=2, axis=1)\n",
    "                result = sess.run(layer_activations)\n",
    "\n",
    "                ###L1 normalize layer wise###\n",
    "                layer_activations_normalized = normalize(result, axis=1, norm='l1')\n",
    "\n",
    "                #print(layer_activations_normalized)\n",
    "                upper_limit = init + num_filters\n",
    "                image_responses[init_row:init_row+2000, init:upper_limit] = layer_activations_normalized\n",
    "                init += num_filters\n",
    "    init_row+= 2000\n",
    "    \n",
    "        \n",
    "print(image_responses.shape)        \n",
    "        \n",
    "###norm sanity check###\n",
    "#layer_acts2 = activations[\"conv2d_1/Relu:0\"]\n",
    "#act0 = layer_acts2[0,:,:,1]\n",
    "#print(np.linalg.norm(act0))\n",
    "#plt.imshow(act0.reshape(28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spgl1 import spgl1, spg_lasso, spgSetParms\n",
    "\n",
    "# % -----------------------------------------------------------\n",
    "    # % Solve LASSO problem for ||x||_1 <= tau\n",
    "    # \n",
    "    #   argminW ||XW-L||_2 subject to ||x||_1 <= tau.\n",
    "    # \n",
    "    #    Where \n",
    "    #        X: (example * filters) matrix\n",
    "    #        W: (filter * class) matrix\n",
    "    #        L: (example * class) matrix\n",
    "    #\n",
    "    # % ----------------------------------------------------------\n",
    "\n",
    "\n",
    "#L = Y_train[:10000,:]\n",
    "#clf = linear_model.Lasso(alpha=0.00001, normalize=False)\n",
    "#clf.fit(image_responses,L)\n",
    "#W = clf.coef_\n",
    "#print(W)\n",
    "#np.savetxt(\"sparse_features.csv\", W, delimiter=\",\")\n",
    "\n",
    "\n",
    "\n",
    "X_t = image_responses\n",
    "L_t = Y_train[:, :]\n",
    "\n",
    "print(\"x.t shape\", X_t.shape)\n",
    "print(\"l.t shape\", L_t.shape)\n",
    "\n",
    "W_opt = np.empty((X_t.shape[1], L_t.shape[1]))\n",
    "\n",
    "w = np.empty((X_t.shape[1],))\n",
    "\n",
    "for i in range(W_opt.shape[1]):\n",
    "    b = L_t[:,i]\n",
    "    tau = 10\n",
    "    w ,resid,grad,info = spg_lasso(X_t, b, tau)\n",
    "    W_opt[:,i] = w\n",
    "    \n",
    "    \n",
    "print(W_opt)\n",
    "print(W_opt.shape)\n",
    "\n",
    "W = W_opt.T\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##ONLY CONV\n",
    "\n",
    "# load json and create model\n",
    "\n",
    "##### Load the model\n",
    "\n",
    "with open('model.json','r') as f:\n",
    "    json = f.read()\n",
    "loaded_model = model_from_json(json)\n",
    "\n",
    "loaded_model.load_weights(\"model.h5\")\n",
    "\n",
    "\n",
    "\n",
    "z = np.zeros((3,3))\n",
    "def only_conv(model):\n",
    "    init = 0\n",
    "    for layer in model.layers:\n",
    "        if( layer.name.startswith('conv') ):\n",
    "            print(layer.name)\n",
    "\n",
    "            layer_weights = layer.get_weights()\n",
    "            layer_weights_copy = layer.get_weights()\n",
    "\n",
    "            num_filters = layer.get_weights()[0].shape[3]\n",
    "            upper_limit = init + num_filters\n",
    "\n",
    "            W_layer = W[:, init:upper_limit]\n",
    "            nonzero = np.nonzero(W_layer)\n",
    "         \n",
    "            dropped_filters = list(set(nonzero[1]))\n",
    "            print(dropped_filters)\n",
    "            print(\"Number of filters dropped: \", len(dropped_filters))\n",
    "            \n",
    "            #print(z.shape)\n",
    "            for filt in dropped_filters:\n",
    "                #print(filt)\n",
    "                layer_weights_copy[0][:,:,0,filt] = 0\n",
    "                layer_weights_copy[1][filt] = 0 \n",
    "            #for i in range(31):\n",
    "            #    print(layer_weights_copy[1])\n",
    "\n",
    "            layer.set_weights(layer_weights_copy)\n",
    "            init += num_filters\n",
    "            #print(layer_weights_copy)\n",
    "only_conv(loaded_model)\n",
    "  \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "scores = loaded_model.evaluate(X_test, Y_test, verbose=2)\n",
    "print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#RANDOM CONV NO AMOUNT CONTROL\n",
    "\n",
    "##### Load the model\n",
    "\n",
    "with open('model.json','r') as f:\n",
    "    json = f.read()\n",
    "loaded_model = model_from_json(json)\n",
    "\n",
    "loaded_model.load_weights(\"model.h5\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#z = np.zeros((3,3))\n",
    "def only_conv_random(model):\n",
    "    init = 0\n",
    "    for layer in model.layers:\n",
    "        if( layer.name.startswith('conv') ):\n",
    "            print(layer.name)\n",
    "            #print(layer.get_weights()[0].shape)\n",
    "\n",
    "            layer_weights = layer.get_weights()\n",
    "            layer_weights_copy = layer.get_weights()\n",
    "\n",
    "            num_filters = layer.get_weights()[0].shape[3]\n",
    "            upper_limit = init + num_filters\n",
    "\n",
    "            num_drop_filters = np.random.randint(0, high=num_filters)\n",
    "       \n",
    "            dropped_filters = np.random.randint(0, high=num_filters, size=(num_drop_filters,))\n",
    "          \n",
    "            dropped_filters = list(set(dropped_filters))\n",
    "            print(dropped_filters)\n",
    "            print(\"Number of filters dropped: \", len(dropped_filters))\n",
    "            z = np.zeros(layer_weights_copy[0][:,:,0,0].shape)\n",
    "\n",
    "            for filt in range(0, 83): \n",
    "                layer_weights_copy[0][:,:,0,filt] = 0\n",
    "                layer_weights_copy[1][filt] = 0 \n",
    "\n",
    "            layer.set_weights(layer_weights_copy)\n",
    "            init += num_filters\n",
    "            \n",
    "only_conv_random(loaded_model)\n",
    "\n",
    "loaded_model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "scores = loaded_model.evaluate(X_test, Y_test, verbose=2)\n",
    "print(\"%s: %.2f%%\" % (loaded_model.metrics_names[1], scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RANDOM CONV ABLATION WITH AMOUNT CONTROLLED\n",
    "##### Load the model\n",
    "with open('model.json','r') as f:\n",
    "    json = f.read()\n",
    "loaded_model = model_from_json(json)\n",
    "\n",
    "loaded_model.load_weights(\"model.h5\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#z = np.zeros((3,3))\n",
    "def only_conv_random_c(model):\n",
    "        init = 0\n",
    "        for layer in model.layers:\n",
    "                if( layer.name.startswith('conv') ):\n",
    "                    print(layer.name)\n",
    "                    #print(layer.get_weights()[0].shape)\n",
    "\n",
    "                    layer_weights = layer.get_weights()\n",
    "                    layer_weights_copy = layer.get_weights()\n",
    "\n",
    "                    num_filters = layer.get_weights()[0].shape[3]\n",
    "                    upper_limit = init + num_filters\n",
    "\n",
    "                    W_layer = W[:, init:upper_limit]\n",
    "                    nonzero = np.nonzero(W_layer)\n",
    "\n",
    "                    num_drop_filters = len(list(set(nonzero[1])))\n",
    "\n",
    "                    samp = random.sample(range(nb_filters), num_drop_filters)\n",
    "                    #print(samp)\n",
    "                    dropped_filters = samp\n",
    "\n",
    "                    print(dropped_filters)\n",
    "                    print(\"Number of filters dropped: \", len(dropped_filters))\n",
    "                   \n",
    "\n",
    "                    for filt in dropped_filters: \n",
    "                        layer_weights_copy[0][:,:,0,filt] = 0\n",
    "                        layer_weights_copy[1][filt] = 0 \n",
    "\n",
    "                    layer.set_weights(layer_weights_copy)\n",
    "                    init += num_filters\n",
    "\n",
    "           \n",
    "        \n",
    "only_conv_random_c(loaded_model)\n",
    "\n",
    "\n",
    "loaded_model.compile(loss='categorical_crossentropy',\n",
    "                    optimizer='rmsprop',\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "scores = loaded_model.evaluate(X_test, Y_test, verbose=2)\n",
    "print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))\n",
    "scores.append(scores[1]*100)    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "scores = loaded_model.evaluate(X_test, Y_test, verbose=2)\n",
    "print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}