03 Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## SET UP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"classification\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'COL_NAMES': ['label', 'data'],\n",
       " 'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ..., \n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),\n",
       " 'target': array([ 0.,  0.,  0., ...,  9.,  9.,  9.])}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the data samples\n",
    "from sklearn.datasets import fetch_mldata\n",
    "\n",
    "mnist = fetch_mldata('MNIST original')\n",
    "mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = mnist[\"data\"], mnist[\"target\"]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD/CAYAAADxA2MgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAABmZJREFUeJzt3T1rFA0YheGNBAtJETRVEBIEY2Mh/o0gdmqjdlaKEC2s\nbNIIop2toFhpISJaphALsQtaBfEbA8IKsk0Kdd/6FeaZuOtmTc51tcfJDCi3AzO7mej3+x0g165x\nXwAwXiIA4UQAwokAhBMBCCcCEE4EIJwIQDgRgHCTYzqv1xRh9CY284fcCUA4EYBwIgDhRADCiQCE\nEwEIJwIQTgQgnAhAOBGAcCIA4UQAwokAhBMBCCcCEE4EIJwIQDgRgHAiAOFEAMKJAIQTAQgnAhBO\nBCCcCEA4EYBwIgDhRADCiQCEEwEIJwIQTgQgnAhAOBGAcCIA4UQAwokAhBMBCDc57guAYd2/f7/c\nX7161bjdvXv3b1/O/3z48GGkP/9vcCcA4UQAwokAhBMBCCcCEE4EIJwIQDjvCbAler1e4/b8+fPy\n2OXl5XJ/8eJFuU9MTJR7OncCEE4EIJwIQDgRgHAiAOFEAMJ5RBjix48f5b6+vj7Uz297jPfu3bvG\nbWVlZahzj9LMzEy5nzp1aouuZHTcCUA4EYBwIgDhRADCiQCEEwEIJwIQznsCIdreA5ifny/3fr9f\n7v/yx3WPHDnSuJ0+fbo8dnFxsdwPHjw40DX9S9wJQDgRgHAiAOFEAMKJAIQTAQgnAhDOewIhLl++\nXO5t7wG07W1mZ2cbt3PnzpXHXr16dahzU3MnAOFEAMKJAIQTAQgnAhBOBCCcCEA47wnsILdv327c\nnj59Wh477PcBtB3f7XYbt7bfibC2tlbuCwsL5U7NnQCEEwEIJwIQTgQgnAhAOBGAcCIA4SaG/Zz4\ngMZy0u2ueg+g0+l0lpaWGrderzfUucf5ewfm5ubK/e3btyM79za3qb8UdwIQTgQgnAhAOBGAcCIA\n4UQAwnlEuI20PSr7/PnzwD97enq63Kempsp91676/5ONjY3G7evXr+WxbX7+/DnU8TuYR4RAOxGA\ncCIA4UQAwokAhBMBCCcCEM5Xjm8jx48fL/dbt241bmfPni2PPX/+fLkfPXq03Nusr683bouLi+Wx\nq6urQ52bmjsBCCcCEE4EIJwIQDgRgHAiAOFEAML5PgG2xJcvXxq3Yd8T+PXr10DXFMD3CQDtRADC\niQCEEwEIJwIQTgQgnAhAON8n8JtPnz6V+549exq3ffv2/e3L2TGqZ/1tv9a8bX/06FG5t30PQzp3\nAhBOBCCcCEA4EYBwIgDhRADCiQCEi3tP4Nq1a+V+586dct+9e3fjduDAgfLYhw8flvt21u12y/3K\nlSuN2+vXr8tj5+fnB7kkNsmdAIQTAQgnAhBOBCCcCEA4EYBwcY8IX758We5ra2sD/+yPHz+W+6VL\nl8r9xo0bA5971No+Yv3kyZNyrx4DTk7W/wwPHz5c7j4qPBx3AhBOBCCcCEA4EYBwIgDhRADCiQCE\ni3tPYJSmp6fL/V9+D6DNxYsXy73ta78rs7OzI/vZtHMnAOFEAMKJAIQTAQgnAhBOBCCcCEC4uPcE\n2r6+empqqtx7vV7jduzYsUEuaUucPHmy3B88eFDu/X6/3Nt+fXjl+vXrAx/L8NwJQDgRgHAiAOFE\nAMKJAIQTAQgnAhAu7j2BmzdvlvubN2/Kvfp+/Y2NjfLYtmfxbZaXl8v9+/fvjdu3b9/KY9ue8x86\ndKjcz5w5M/C+d+/e8lhGy50AhBMBCCcCEE4EIJwIQDgRgHATbR8RHZGxnHQzVlZWyn1paalxqz5m\n3Ol0Ou/fvy/3UX5cd2FhodxnZmbK/d69e+U+Nzf3x9fEyG3qH4w7AQgnAhBOBCCcCEA4EYBwIgDh\nRADCeU/gD3W73cat7eO6q6ur5f7s2bNyf/z4cblfuHChcTtx4kR57P79+8udbcl7AkA7EYBwIgDh\nRADCiQCEEwEIJwIQznsCsHN5TwBoJwIQTgQgnAhAOBGAcCIA4UQAwokAhBMBCCcCEE4EIJwIQDgR\ngHAiAOFEAMKJAIQTAQgnAhBOBCCcCEA4EYBwIgDhRADCiQCEEwEIJwIQTgQgnAhAOBGAcCIA4UQA\nwokAhBMBCCcCEE4EIJwIQDgRgHAiAOEmx3TeiTGdF/iNOwEIJwIQTgQgnAhAOBGAcCIA4UQAwokA\nhBMBCCcCEE4EIJwIQDgRgHAiAOFEAMKJAIQTAQgnAhBOBCCcCEA4EYBwIgDhRADCiQCEEwEIJwIQ\n7j/CefnF56ivEAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18ebfecb4e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# show data\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "some_digit = X[36000]\n",
    "some_digit_image = some_digit.reshape(28,28)\n",
    "\n",
    "plt.imshow(some_digit_image, cmap = matplotlib.cm.binary, interpolation = \"nearest\")\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[36000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare test set and train set\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# shuffle the training set to guarantee all cross-validation folds will be similar \n",
    "\n",
    "import numpy as np\n",
    "\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BINARY CLASSIFIER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# training a binary classfier for digit 5\n",
    "y_train_5 = (y_train == 5)\n",
    "y_test_5 = (y_test == 5)\n",
    "y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# use Stochastic Gradient Descent (SGD) : handling very large dataset and suited for online learning\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(random_state=42)\n",
    "sgd_clf.fit(X_train, y_train_5)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cross-validation evaluate model\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring=\"accuracy\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# a better cross-validation using StratifiedKFold\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.base import clone\n",
    "\n",
    "skfolds = StratifiedKFold(n_splits = 3, random_state = 42)\n",
    "\n",
    "for train_index, test_index in skfolds.split(X_train, y_train_5):\n",
    "    clone_clf = clone(sgd_clf)\n",
    "    X_train_folds = X_train[train_index]\n",
    "    y_train_folds = y_train_5[train_index]\n",
    "    X_test_folds = X_train[test_index]\n",
    "    y_test_folds = y_train_5[test_index]\n",
    "    \n",
    "    \n",
    "    clone_clf.fit(X_train_folds, y_train_folds)\n",
    "    y_pred = clone_clf.predict(X_test_folds)\n",
    "    n_correct = sum(y_pred == y_test_folds)\n",
    "    print(n_correct / len(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# VERY GOOD, > 95% ???\n",
    "# Let see above\n",
    "\n",
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self, X, y = None):\n",
    "        pass\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype=bool)\n",
    "    \n",
    "\n",
    "never5_clf = Never5Classifier()\n",
    "cross_val_score(never5_clf, X_train, y_train_5, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# It still over 90%, because only 10% images are 5s, so if you always guess that an image is not 5, you will still\n",
    "# get accuracy over 90%.\n",
    "\n",
    "# This demonstrates why accuracy is generally not the preferred performance measure for classifiers, especially\n",
    "# for SKEWED-DATASET.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# CONFUSION MATRIX is a better measurement.\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_train_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_5, y_train_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Each row in confusion matrix represents an actual class, which each column represents a predicted class \n",
    "# 53682 non-5 images correct classified to non-5 digit -> true negative\n",
    "# 897  non-5 images wrong classified to 5 digit -> false postive\n",
    "# 1374 5 images wrong classified to non-5 digit -> false negative\n",
    "# 4047 5-images correct classified to 5 digit -> true positive\n",
    "\n",
    "# PERPECT classifier should as\n",
    "y_train_perfect_prediction = y_train_5\n",
    "confusion_matrix(y_train_5, y_train_perfect_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# More concise metric\n",
    "# precision = TP / (TP + FP)  : accuracy of postitive prediction\n",
    "# recall = TP / (TP + FN)     : sensitivity or true positive rate\n",
    "\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "precision_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "recall_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# F1-score = 2 / [precision^-1 + recall^-1]   is a tradeoff between precision and recall\n",
    "# In some contexts you mostly care about precision but some contexts you mostly care recall\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# PRECISION and RECALL has a tradeoff. \n",
    "# For example SGDClassifier has decision_function to decice the instance is TRUE or FALSE\n",
    "# To select suitable threshold for decision_function, you should get score of all train instance\n",
    "# and base on precision recall curve plot to decice the threshold\n",
    "\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "threshold = 0\n",
    "\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "threshold = 70000\n",
    "\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Raise threshold decreases the recall."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get score of all instances\n",
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3, method='decision_function')\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC CURVE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Compute the precision and recall for all posible thresholds using the precision_recall_curve\n",
    "\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], 'b--', label=\"Precision\")\n",
    "    plt.plot(thresholds, recalls[:-1], 'g-', label=\"Recall\")\n",
    "    plt.xlabel(\"Threshold\")\n",
    "    plt.legend(loc=\"upper left\")\n",
    "    plt.ylim([0, 1])\n",
    "    \n",
    "plt.figure(figsize=(10,6))\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(recalls, precisions)\n",
    "plt.ylabel(\"Precision\")\n",
    "plt.xlabel(\"Recall\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Suppose you want to select 90% precision. Use the plot you can see that threshold is 150000.\n",
    "\n",
    "y_train_pred_90 = (y_scores > 150000)\n",
    "\n",
    "precision_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "recall_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ROC curve is another common tool used with binary classifiers. ROC curve plots true positive rate (recall) versus \n",
    "# false positive rate.\n",
    "# FPR = 1 - true negative rate.\n",
    "# TNR is also called specificity\n",
    "# ROC curve plots sensitivity (recall) versus 1 - specificity\n",
    "\n",
    "\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel(\"False positive rate\")\n",
    "    plt.ylabel(\"True positive rate\")\n",
    "    \n",
    "plt.figure(figsize=(6,6))    \n",
    "plot_roc_curve(fpr, tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Once again there is a tradeoff: the higher the call (TPR), the more false positive the classifier produces. The dotted line\n",
    "# represents the ROC curve of a purely random classifier; a good classifier stays as far away from the line posible.\n",
    "\n",
    "# One way to compare classifiers is to measure the area under the curve (AUC). Perfect classifier will have AUC = 1, \n",
    "# purely random classifier has AUC = 0.5\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We have two curve: PR (precision/recall) and ROC. \n",
    "# We use PR for case positive class is rare or when you care more about the false positive than the false negative.\n",
    "# We use ROC otherwise.\n",
    "\n",
    "# For example, in the ROC above, you may think the classifier is good because positve class is small, but in PR graph,\n",
    "# make it clear that the classifier has room for improvement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use RandomForestClassifier to compare ROC and PR with SGDClassifier\n",
    "\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = RandomForestClassifier(random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3, method='predict_proba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# To plot ROC curve, we need scores, not probabilities. We use positive class's probability as the score\n",
    "y_score_forest = y_probas_forest[:, -1]\n",
    "\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5, y_score_forest)\n",
    "\n",
    "plt.figure(figsize=(6,6))    \n",
    "plt.plot(fpr, tpr, \"b:\", label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"Random forest\")\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "roc_auc_score(y_train_5, y_score_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# From the graph we see that the Random Forest Classifier is much better than SGD Classifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MULTICLASSIFICATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MULTICLASSIFICATION has some type: direct classifiers (Random Forest, Naive Bayes), one-vs-one (SVM), \n",
    "# one-vs-all (otherwise binary classifiers)\n",
    "\n",
    "# One-vs-one is better for train the classifiers faster to train many classifiers on small training sets than\n",
    "# training few classifiers on large training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for SGD classifier\n",
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# the decision score return 10 scores than one score as in binary classifier\n",
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# the highest score is indeed the one corresponding to class 5\n",
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sgd_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sgd_clf.classes_[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# If you want to force Scikit learn to use OvO or OvA\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(ovo_clf.estimators_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Random Forest not need run OvO or OvA\n",
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Evaluate the classifiers\n",
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cross_val_score(forest_clf, X_train, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# simply scaling the inputs increases the accuracy above 90%\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "\n",
    "cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ERROR ANALYSIS : to improve model\n",
    "\n",
    "# Way 1: confusion matrix\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Change to image to more visual\n",
    "plt.matshow(conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This confusion matrix look good because most images are on the main diagonal, which mean that classify correctly. \n",
    "# The 5s look slightly darker than other because fewer images of 5s in the dataset or that the classifier does not\n",
    "# good for 5s as on other digits.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Focus the plot on the errors. Devide each value on confusion matrix by the number of images in the corresponding \n",
    "# class\n",
    "\n",
    "row_sums = conf_mx.sum(axis=1, keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.matshow(norm_conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remember that rows represent actual classes while columns represent predicted classes.\n",
    "# The columns 8 and 9 are quite bright, which tells that many images get missclassified as 8s or 9s.\n",
    "# Similar, rows for class 8 and 9 are also quite bright, telling you that 9s and 8s are often confused \n",
    "# with other digits.\n",
    "# The error not symetric and some digits have less error than such as 1.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Analyzing the confusion matrix can often give you insights on ways to improve your classifier. Looking this\n",
    "# plot, you should improve classification of 8s and 9s, as well as fixing the specifix of 3/5 confusion.\n",
    "\n",
    "# Some way:\n",
    "#      - Gather more training data\n",
    "#      - Engineer new features\n",
    "#      - Preprocess images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cl_a = 3\n",
    "cl_b = 5\n",
    "X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
    "X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
    "X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
    "X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "# EXTRA\n",
    "def plot_digits(instances, images_per_row=10, **options):\n",
    "    size = 28\n",
    "    images_per_row = min(len(instances), images_per_row)\n",
    "    images = [instance.reshape(size,size) for instance in instances]\n",
    "    n_rows = (len(instances) - 1) // images_per_row + 1\n",
    "    row_images = []\n",
    "    n_empty = n_rows * images_per_row - len(instances)\n",
    "    images.append(np.zeros((size, size * n_empty)))\n",
    "    for row in range(n_rows):\n",
    "        rimages = images[row * images_per_row : (row + 1) * images_per_row]\n",
    "        row_images.append(np.concatenate(rimages, axis=1))\n",
    "    image = np.concatenate(row_images, axis=0)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary, **options)\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.subplot(221); plot_digits(X_aa[:25], images_per_row = 5)\n",
    "plt.subplot(222); plot_digits(X_ab[:25], images_per_row = 5)\n",
    "plt.subplot(223); plot_digits(X_ba[:25], images_per_row = 5)\n",
    "plt.subplot(224); plot_digits(X_bb[:25], images_per_row = 5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The two 5x5 block on the left show digits classified as 3s, the two blocks on the right show digits classified as 5s.\n",
    "# Some images was classified wrong on bottom-left and top-right blocks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# You can improve classifiers by preprocessing digit images by rotating"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MULTIPLE LABEL CLASSIFICATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# MULTIPLE LABEL CLASSIFICATION\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train_large = (y_train >=7)\n",
    "y_train_odd = (y_train % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Correct: 5 is not large and is odd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# There are many ways to evaluate a multilabel classifier, and selecting the right metrics depend on your project.\n",
    "# For example, one approach is to measure F1 score for each individual label then simply compute average score.\n",
    " \n",
    "y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_train, cv=3)\n",
    "f1_score(y_train, y_train_knn_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This asume that all labels are equal important.\n",
    "# You can give more weight to the classifier's score."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MULTIOUTPUT CLASSFICIATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_digit(data):\n",
    "    image = data.reshape(28, 28)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary,\n",
    "               interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "some_index = 5500\n",
    "plt.subplot(121); plot_digit(X_test_mod[some_index])\n",
    "plt.subplot(122); plot_digit(y_test_mod[some_index])\n",
    "save_fig(\"noisy_digit_example_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "knn_clf.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "plot_digit(clean_digit)\n",
    "save_fig(\"cleaned_digit_example_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# EXERCISES"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 1: MNIST classifier with over 97% accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "param_grid = [{'weights' : ['uniform', 'distance'], 'n_neighbors' : [3, 4, 5]}]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "grid_search = GridSearchCV(knn_clf, param_grid, cv=5, verbose=3)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = grid_search.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Excercise 2: Data augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.ndimage.interpolation import shift\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shift_image(image, dx, dy):\n",
    "    image = image.reshape((28, 28))\n",
    "    shifted_image = shift(image, [dy, dx], cval = 0, mode='constant')\n",
    "    return shifted_image.reshape([-1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqMAAADWCAYAAADl74szAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHxZJREFUeJzt3Xu0VeV57/HfD/GCVyJQlEQhxiiKmqRurKRqbEOOxaGx\nhp7TeksI8eixxzFihncEg5HE0YycVj1HUqk2EEcSQxo8UTuwDUZ0GGyabQ1piNYrJEYh23oDBIHD\nc/6Yk7jd612w1l5z7bku388Ye8h61rve+U73evb7rLnmO6cjQgAAAEAZhpU9AAAAAHQvilEAAACU\nhmIUAAAApaEYBQAAQGkoRgEAAFAailEAAACUhmK0BdlebfuKOl8Ttv+s4HHMtf2LIvsEWkktuTaw\nje2DbP+z7Y22G742nu0Ntmc02g/Q7YY6nwc5V59l+xnb22wvrOe1nYxitElsv9f2Atsv2t5i+ze2\n/872+2p4+WRJ8+vc5MGS7qt/pEBnsj3G9vx8wnjb9jrbD9r+RJ1dDczHKySNk/RhZXnXlA+DAN7R\nQfl8p6TvSxov6fO2Z9je0KRttY3hZQ+gE9l+v6QVkl6Q9BlJz0j6gKQvS/qp7SkRsTrxuj0iYktE\n9NW7zYhY29iogY7zfUl7S/qcpGcl/Z6kj0kaVU8niXw8XNLjEfFMEYMEUJO2z2fbI5WN958i4jd5\nrNmbbQscGW2O2yRtlzQ1Ih6MiF9FxEOSpubx2yTJ9nLbX7f9Ndt9kn6cxwd+jXCE7Ydtb7b9pO0/\nGfjVXv9PcrYn5I+n2/6h7bds/7L/J0jbu9m+0/YLtjflXxtcZZv3BNpe/kf/ZEnX5Dm4JiJ+GhFf\ni4i7BzTfy/bttt/Mv8m4ckBfv8tH26slnSXp03mOLcxjkvS9PLZaVdg+PM/7zbb/w/YZiTbH2l6W\n5+Wr+TYOyJ+bmG/joPzx3vlRogf6vf5C28/m/97l3wKg1bVqPifGeUD+jehvba/P5+2e/LlTJb2W\nN/1R3vepkr4haZ/8cdieW/v/mc5B4VEw2wdK+hNJt0XEW/2fyx/PlzTN9nvy8PmSrCzRPp3ob5ik\neyRtk3SipJmSbpC0Zw3D+bKkWyV9SNJPJd1te9/8uWGSfiPpv0k6StJ1kmZJ+myt+wq0sA35zydt\n77WLtl+Q9O+Sfl/SX0n6qu0pVdpOlrRM0mJlX+l9Po9J0n/PY5NTL+yXy8MkTVGWy3PVL5dt7yPp\nn/KxnyDpbEkflfT3khQRT0laK+nU/CUflfSmpD+0veObrlMlLR+w+Z39LQBaXcvl80DODnH+o6T3\nSjpD0kckPaKs8DxY2belk/Lm0/O+V0i6TNJb+eODJX2tlu11GorR4n1QWXH5ZJXnf5k//8H88QsR\ncXlEPBURqdd8QtKRkj4dET+LiMeUJVstp1j8TUTcl3/9MEvSgcrOi1FEbI2I6/NPl6sjYrGkv5V0\nTo37CbSsiNgmaYayD3uv234s/wbiDxLN/zki/k9EPBsR/1vZV4Afr9Jvn6S3JW2KiLUR8Ua/r/1e\nz2PVTrOZKuloSedHxBMR8WNlE1H/XD5X0j6SLoiIf4+IhyVdJOlTtg/P2zws6Y/yf58q6R8k/afe\nmTQ/pspitOrfAqDVtWg+D/RHynLqzyLiX/Ptz5H0vLJ83iLpt3nbV/O+t0h6IxtKrM1/uvL8UYrR\n8j2+i+cnSnppx/kluZ8q+7p/V37e798v5f/9vR0B2//Ddq/tvvwE6i9IOrSGfoGWFxHfV7Yw4UxJ\nS5UdRfwX27MGNP35gMcvqV+eFOgoSb+JiF/1i/1E787loyT9PCLW94utyNscnT9erneOjJ4q6aEd\nsbxgfZ8qi9Gd/i0AWl0L5vNAxys7p7UvP41uQz6vHqNszQh2gmK0eM9KCr0zcQx0dP78s/njjU0c\ny9Yd/4iIHZesGCZJtv9c0s2SFko6TdknuvmS9mjieIAhFRGbI+KHEfGliPiospWsc233f59vHfgy\ntebfxh05vFzSEXnh2ZM/Xq7syMypkp6LiBcHvLbq3wKgXbR4Pg+TtE7ZXNr/Z6KkOUOw/bbGH6OC\nRcR/Kjvn6y9t793/ufzx/5S0NCJerbHLpySNsz2uX6xHjf/uTpL0k/zrjH+LiGfFpzd0vl8q+1p8\nV+ed1WurpN120eZJSe+1fUi/2Al6dy4/KelY2/v1i300b/Ok9K7zRq9TVnj+Vlkx+ofKTutZPui9\nANpLmfk80L9JGitpe/4Vff+f3+7kdVsGsa2OQzHaHJcqS5Bltv/Y9iH5qrkfKjtf9NI6+vqhpP+Q\ntMj2h2yfKOmvlS1oauSC209L+n3b02x/0PYcZeeaAW3P9ijbP7J9vu3jbL/f9n+VdJWkByPizYI3\nuVrSx51dQPs9VdosU/bh8pu2P5wvqvgbZbm8w7eULWb4Zr6q/hRJt0takn9g3OFhZefPPSRJ+aXi\n+iR9ShSj6DAtms8DLVN2RZwf5PPq+21PsX2D7ZN3sa29bH/C9uiBB7G6BcVoE0TEc8qOXq6SdJey\nE5i/rezIxuSIeKGOvrYrW1G7p6R/lbRI0leUFaKbGxjm7cpWEH5b2TmoEyT9rwb6A1rJBkn/omx1\n7MPKcvEryt7vf96E7V2u7GvyX0t6ItWgXy4PU3au6DclzVO2gGJHm7eUnTazv7J8/4Gkx5StvO9v\nubIPvMt3EQM6Qcvl80D56S+nS/qRpL9TdhBpsbIFyC/t5HUrlC0e/o6yD5RXNTLwduV3Th9Cu7D9\nIUk/k9QTEbtaAAUAANCyKEbbgO2zlS10ekbZEcy/VvZ1/0eCXyAAAGhj3A60Peyn7OK9hyi7g8Ny\nSV+gEAUAAO2OI6MAAAAoDQuYAAAAUJpCilHbB9q+x/ZG22tsn1tEvwCKR74C7YN8RTco6pzR25Rd\nuHWssjsO/KPtlRGxKtV49OjRMWHChII2DTTm8ccffyUixpQ9jiFUV75K5Cxax+rVq/XKK6+47HEM\nIfIVbavWfG24GLW9j6Tpko6JiA2SHrX9A0kXSLom9ZoJEyaot7e30U0DhbC9puwxDJXB5KtEzqJ1\n9PT0lD2EIUO+ot3Vmq9FfE1/hKRtEfF0v9hKSZMK6BtAschXoH2Qr+gKRRSj+0oaeCuuN5Vdjuh3\nbF9ku9d2b19fXwGbBTAINeWrRM4CLYB8RVcoohjdoOzWdf0dIGl9/0BELIiInojoGTOmm07PA1pK\nTfkqkbNACyBf0RWKKEafljTc9gf7xT6k7N6xAFoL+Qq0D/IVXaHhYjQiNkpaIulLtvexfZKkT0q6\nq9G+ARSLfAXaB/mKblHURe//UtIISb+V9G1Jl+zsshMASkW+Au2DfEXHK+Q6oxHxqqQ/LaIvAM1F\nvgLtg3xFN+B2oAAAACgNxSgAAABKQzEKAACA0lCMAgAAoDQUowAAACgNxSgAAABKQzEKAACA0lCM\nAgAAoDQUowAAACgNxSgAAABKQzEKAACA0lCMAgAAoDQUowAAACgNxSgAAABKQzEKAACA0lCMAgAA\noDQUowAAACgNxSgAAABKQzEKAACA0gwvohPbyyWdKGlbHvpNRBxZRN9oLdu3b0/Gn3/++WT88MMP\nb+ZwMAjkK9A+yNfyMN8NnSKPjF4aEfvmPyQK0NrIV6B9kK/oaHxNDwAAgNIUWYzeZPsV2z+2fWqB\n/QIoHvkKtA/yFR2tqGL0akmHSXqvpAWS7rP9gf4NbF9ku9d2b19fX0GbBTAIu8xXiZwFWgT5io5X\nSDEaET+JiPUR8XZELJL0Y0mnD2izICJ6IqJnzJgxRWwWwCDUkq95O3IWKBn5im5QyGr6hJDkJvWN\n3Lp165Lx22+/PRmfP39+08Yybdq0ZHzOnDnJ+GGHHda0saBu5CvQProyX+uZ75o510nMd83Q8JFR\n2yNtn2Z7L9vDbZ8n6RRJDzQ+PABFIl+B9kG+olsUcWR0d0nzJE2U9P8kPSXpTyPi6QL6BlAs8hVo\nH+QrukLDxWhE9EmaXMBYADQZ+Qq0D/IV3YLrjAIAAKA0FKMAAAAoTbNW0yO3efPmitiTTz5ZVx/3\n3ntvMr5gwYJk/OWXX66r/yIsWrQoGb/vvvuS8Wr39t1///0LGxMAYGik5jqJ+U5Kz3fMde/GkVEA\nAACUhmIUAAAApaEYBQAAQGkoRgEAAFAailEAAACUhtX0BXn00UeT8WuuuaYitmLFikK2OXXq1GT8\nc5/7XDJ+wQUX1Nz3q6++moxPmTKl5j4k6fOf/3wyvvfee9fVDwCgfPXMdVI58109c53EfNcKODIK\nAACA0lCMAgAAoDQUowAAACgNxSgAAABKQzEKAACA0rCavopqq+tuvPHGZHzhwoXJ+BtvvFERGzdu\nXLLtZZddloyffPLJyfgJJ5yQjNtOxqvZtm1bRez++++vq49Ro0Yl41dffXUyPnw4bz0AaAX1zHf1\nzHVSa813qblOYr5rBRwZBQAAQGkoRgEAAFAailEAAACUhmIUAAAApampGLV9qe1e22/bXjjguY/b\nfsr2W7Yfsj2+KSMFUBPyFWgf5CtQ+2r6lyTNk3SapBE7grZHS1oi6UJJ90m6UdJ3JZ1Y7DCHXrXV\ncvWuVD/yyCMrYo899liy7ciRI+vquyhr166tiF1++eV19XHTTTcl43vsscegxoSGdF2+orrt27cn\n488//3wyfvjhhzdzOKhUer4WMd+l5jqptea71FwnMd+1gpqK0YhYIkm2eyS9r99Tn5K0KiK+lz8/\nV9IrtidGxFMFjxVADchXoH2Qr0Dj54xOkrRyx4OI2Cjp2TwOoLWQr0D7IF/RNRotRveVNPBKt29K\n2m9gQ9sX5efF9Pb19TW4WQCDUHO+SuQsUDLyFV2j0WJ0g6T9B8QOkLR+YMOIWBARPRHRM2bMmAY3\nC2AQas5XiZwFSka+oms0eo+qVZI+s+OB7X0kfSCPt7VzzjknGb/77rvr6mfz5s0VsaVLl9a1zWZb\ntmxZzW1nzJiRjF944YUFjQZN1LH52i7WrVuXjN9+++3J+Pz585s2lmnTpiXjc+bMScYPO+ywpo0F\nSUOWr0XMd6m5Tmqt+a6euU5ivhtKtV7aabjtvSTtJmk323vZHi7pHknH2J6eP/9FSSs5uRooD/kK\ntA/yFaj9a/rZkjZJukbS+fm/Z0dEn6Tpkr4s6TVJJ0j6iyaME0DtyFegfZCv6Hq1XtpprqS5VZ5b\nJmlicUMC0AjyFWgf5CvA7UABAABQIopRAAAAlKbR1fQdq9qK0mq31lu8eHEyvmbNmorY+eefn2y7\nYsWKZPyUU05JxqdPn56MDxuW/ozxyCOPJOM333xzReyhhx5Ktj355JOTcaBTpFYFP/nkk3X1ce+9\n9ybjCxYsSMZffvnluvovwqJFi5Lx++67LxmvdvvQ/fcfePUhtJt65rt65jqpnPmunrlOYr5rBRwZ\nBQAAQGkoRgEAAFAailEAAACUhmIUAAAApaEYBQAAQGkcEUO+0Z6enujt7R3y7RZh69atyfhXv/rV\nZHzVqsrbCNd7f/tqqq1unTBhQjK+cePGZHz9+vUVsZUrVybbjhgxorbB5S6++OJkfNOmTXX1U89Y\nZs6cmYxfeeWVybjtxyOip+EBdbB2ztlqHn300WT8mmuuqYhVW/lbr6lTpybjJ554YjJ+wQUX1Nz3\nq6++moxPmTKl5j4k6YYbbkjGr7322mR8+PChvShLT0+Pent7PaQbbTNF5WtqvqtnrpPKme/qmeuk\n7prvhlqt+cqRUQAAAJSGYhQAAACloRgFAABAaShGAQAAUBqKUQAAAJSGe9PXaffdd0/Gr7vuumR8\ny5YtFbHrr78+2fb0009PxlevXp2MV1slW23Va7V7Cp9xxhkVsVmzZiXbFuXMM89Mxo877rhk/JJL\nLqmIpe5JLEkHHXTQ4AeGtlVtNfmNN96YjC9cuDAZf+ONNypi48aNS7a97LLLkvFq97Q+4YQTknG7\nvsXh27Ztq4jdf//9dfUxatSoZPzqq69Oxod61TzKl5rv6pnrpHLmu3rmOqmc+a6euU7q/PmOI6MA\nAAAoDcUoAAAASkMxCgAAgNJQjAIAAKA0NRWjti+13Wv7bdsL+8Un2A7bG/r9zGnaaAHsEvkKtA/y\nFah9Nf1LkuZJOk1S6gapIyOicnkntMcee1TEJk6cmGx77rnnJuNf+cpXkvF169Yl49VW+FZTz323\nx48fn4wfcsghyfitt96ajE+aNCkZr3a1AtSlK/O12urweleqH3nkkRWxxx57LNl25MiRdfVdlLVr\n11bELr/88rr6uOmmm5Lx1N8sNFVH5Gu1900rzXf1zHVSc+c75rp3q6kYjYglkmS7R9L7mjoiAA0h\nX4H2Qb4CxZ0zusb2i7a/YXt0QX0CaA7yFWgf5Cs6XqPF6CuSJksaL+l4SftJ+laqoe2L8vNievv6\n+hrcLIBBqDlfJXIWKBn5iq7RUDEaERsiojcitkXEOkmXSvovtvdLtF0QET0R0TNmzJhGNgtgEOrJ\n17w9OQuUhHxFNyn60k7RpH4BFI98BdoH+YqOVdMCJtvD87a7SdrN9l6Stin76uB1Sc9Ieo+kWyUt\nj4jKmzvjd7Zv356Mb9iwYYhHkhk7dmxFrNrK3BkzZiTjo0dzKlOr6NZ8Peecc5Lxu+++u65+Nm/e\nXBFbunRpXdtstmXLltXctlrOXnjhhQWNBo3o9HxtpfkuNddJzHetoNZPWLMlbZJ0jaTz83/PlnSY\npAckrZf0C0lvSyrnrzOAHchXoH2Qr+h6tV7aaa6kuVWe/k5RgwHQOPIVaB/kK8C5JwAAACgRxSgA\nAABKQzEKAACA0tR6b3rsQrUVg6+99lpFrNq9d6vd17bZHnzwwYrY0UcfXcJIgMGbM2dOMl4tNxcv\nXpyMr1mzpiJ2/vnnJ9tWu9f1KaeckoxPnz49GR82LH1c4JFHHknGb7755orYQw89lGx78sknJ+PA\nYNQz10mtNd+l5jqJ+a4VcGQUAAAApaEYBQAAQGkoRgEAAFAailEAAACUhgVMBal2kvb1118/xCMB\nutNRRx2VjN91113J+LHHHpuMr1q1qiJW7Zait912W13x559/PhmfMGFCMn788ccn4w888EBFbOXK\nlcm2jz76aDJezcUXX5yMb9q0qa5+UkaMGJGMz5w5Mxm/8sorG94misVch2bgyCgAAABKQzEKAACA\n0lCMAgAAoDQUowAAACgNxSgAAABKw2r6OvX19SXjCxYsaLjvs846KxmfNWtWMn7SSScl41u3bm14\nLECn2H333ZPx6667LhnfsmVLRazaSuHTTz89GV+9enUyPmXKlGT82muvTcar3bL0jDPOqIhV+ztR\nlDPPPDMZP+6445LxSy65pCJW7banBx100OAHhqZJzXdFzHUS8x3ejSOjAAAAKA3FKAAAAEpDMQoA\nAIDSUIwCAACgNBSjAAAAKM0uV9Pb3lPSfElTJR0o6TlJ10bE0vz5j0u6TdKhkn4iaUZErGnaiEt2\nxx13JOMvvvhizX2cdtppyfi8efOS8UmTJtXc984cfPDByfjIkSML6R/lI18bt8cee1TEJk6cmGx7\n7rnnJuPV7t+9bt26ZPyyyy6rcXSZFStW1Nx2/PjxyfghhxySjN96663JeLW/Q9WuVoDatHLOpua7\neuY6qZz5jrmu/dRyZHS4pF9L+pikAyTNlrTY9gTboyUtkTRHWRL1Svpuk8YKYNfIV6C9kLPoers8\nMhoRGyXN7Re63/YLko6XNErSqoj4niTZnivpFdsTI+Kp4ocLYGfIV6C9kLPAIM4ZtT1W0hGSVkma\nJGnljufypHo2jw983UW2e233VrtwPIBiDTZf89eSs8AQY45FN6qrGLW9u6RvSVqUfyrbV9IbA5q9\nKWm/ga+NiAUR0RMRPWPGjBnseAHUqJF8lchZYKgxx6Jb1VyM2h4m6S5JWyRdmoc3SNp/QNMDJK0v\nZHQABoV8BdoLOYtuVtO96W1b0p2Sxko6PSJ23Ax2laTP9Gu3j6QP5PGOdMstt9TVfvLkyRWxJUuW\nJNuOGDEiGX/99deT8YioaywzZ85MxseNG1dXP2ht5Gvxtm/fnoxv2LBhiEeSGTt2bEXs8ssvT7ad\nMWNGMj569Ogih4QGtGrO1jPfpeY6qZz5jrmu/dR6ZPTrko6SdGZEbOoXv0fSMban295L0hclreTE\naqBU5CvQXshZdLVdFqO2x0u6WNKHJa21vSH/OS8i+iRNl/RlSa9JOkHSXzRzwACqI1+B9kLOArVd\n2mmNJO/k+WWS0leEBjCkyFegvZCzALcDBQAAQIkoRgEAAFCamlbT4x1XXHFFMn7VVVcl488991xF\n7Iknnki2PfTQQ5PxadOmJePbtm1Lxs8+++xkfPbs2ck40I2qrZB/7bXXKmLV7jVf7T7uzfbggw9W\nxI4++ugSRoJOlprv6pnrpHLmO+a69sORUQAAAJSGYhQAAACloRgFAABAaShGAQAAUBqKUQAAAJSG\n1fR1+uxnP5uML1q0KBlftaryFsInnXRSIWNJ3Z9aqr6ScM899yxku0AnqLZC/vrrrx/ikQCtKTXf\n1TPXSeXMd8x17YcjowAAACgNxSgAAABKQzEKAACA0lCMAgAAoDQUowAAACgNq+nrNGrUqGT84Ycf\nTsaXLl3atLGcc845yfiwYXzGAHbo6+tLxhcsWNBw32eddVYyPmvWrGS82srirVu3NjwWoGip+a6M\nuU5ivut0/BYBAABQGopRAAAAlIZiFAAAAKWhGAUAAEBpdrmAyfaekuZLmirpQEnPSbo2IpbaniDp\nBUkb+73kryLixuKH2toOPPDAZPy8884b4pGgm5Gvle64445k/MUXX6y5j9NOOy0ZnzdvXjI+adKk\nmvvemYMPPjgZHzlyZCH9o3ztlrPMdWiGWlbTD5f0a0kfk/QrSadLWmz72H5tRkbEtiaMD0B9yFeg\nvZCz6Hq7/Jo+IjZGxNyIWB0R2yPifmWf1I5v/vAA1IN8BdoLOQsM4pxR22MlHSFpVb/wGtsv2v6G\n7dFVXneR7V7bvdWu+wegWIPN1/y15CwwxJhj0Y3qKkZt7y7pW5IWRcRTkl6RNFnSeGWf4vbLn68Q\nEQsioiciesaMGdPYqAHsUiP5KpGzwFBjjkW3qvkOTLaHSbpL0hZJl0pSRGyQ1Js3WWf7Ukkv294v\nItYXPVgAtSFfgfZCzqKb1VSM2rakOyWNlXR6RFS7d13k/+WSUUBJyNd3u+WWW+pqP3ny5IrYkiVL\nkm1HjBiRjL/++uvJeEQk49XMnDkzGR83blxd/aC1kbPodrUeGf26pKMkTY2ITTuCtv9A0uuSnpH0\nHkm3SloeEW8UPVAANSNfgfZCzqKr7fLTle3xki6W9GFJa21vyH/Ok3SYpAckrZf0C0lvSzqnieMF\nsBPkK9BeyFmghiOjEbFGknfS5DvFDQdAI8hXoL2QswDnnQAAAKBEFKMAAAAoTc2XdgKAdnTFFVck\n41dddVUy/txzz1XEnnjiiWTbQw89NBmfNm1aMr5tW/qOjmeffXYyPnv27GQcADoJR0YBAABQGopR\nAAAAlIZiFAAAAKWhGAUAAEBpKEYBAABQGtd7r+RCNmr3SVqTPxwt6ZUhH0Q5umVf220/x0fEmLIH\n0cq6NGfZz9ZEvu4C+drx2mlfa8rXUorRdw3A7o2InlIHMUS6ZV+7ZT+7Vbf8ftlPdIJu+f12y35K\nnbmvfE0PAACA0lCMAgAAoDStUIwuKHsAQ6hb9rVb9rNbdcvvl/1EJ+iW32+37KfUgfta+jmjAAAA\n6F6tcGQUAAAAXYpiFAAAAKUprRi1faDte2xvtL3G9rlljaVIti+13Wv7bdsLBzz3cdtP2X7L9kO2\nx5c0zIbZ3tP2nfnvbr3tn9me1u/5jtlXZMjZ9n0fk6/dh3xt7/dxt+VsmUdGb5O0RdJYSedJ+rrt\nSSWOpygvSZon6e/7B22PlrRE0hxJB0rqlfTdIR9dcYZL+rWkj0k6QNJsSYttT+jAfUWGnG3f9zH5\n2n3I1/Z+H3dVzpZ1B6Z9JL0m6ZiIeDqPfVPSSxFxzZAPqAlsz5P0voiYkT++SNKMiPho/ngfZXdQ\n+EhEPFXaQAtk++eSbpA0Sh2+r92GnO289zH52rnI1858H3dyzpZ1ZPQISdt2JElupaRO+NRWzSRl\n+yhJioiNkp5Vh+yz7bHKfq+r1OH72qXI2Q56H5OvHY987bD3cafnbFnF6L6S3hwQe1PSfiWMZajs\nK+mNAbGO2Gfbu0v6lqRF+aeyjt3XLkbOZtp+n8nXrkC+Zjpin7shZ8sqRjdI2n9A7ABJ60sYy1Dp\nyH22PUzSXcrOTbo0D3fkvna5bvyddtw+k69doxt/px25z92Ss2UVo09LGm77g/1iH1J2+LlTrVK2\nj5J+d47HB9TG+2zbku5UdoL89IjYmj/VcfsKcrbd38fka1chXzvgfdxNOVtKMZqf37BE0pds72P7\nJEmfVFb9tzXbw23vJWk3SbvZ3sv2cEn3SDrG9vT8+S9KWtmuJxvnvi7pKElnRsSmfvFO3NeuRs52\nxPuYfO0S5GvHvI+7J2cjopQfZZcj+L+SNkr6laRzyxpLwfs1V1IM+JmbPzdV0lOSNklaLmlC2eNt\nYD/H5/u2WdlXBjt+zuu0feXnd79zcrZN38fka/f9kK/t/T7utpzl3vQAAAAoDbcDBQAAQGkoRgEA\nAFAailEAAACUhmIUAAAApaEYBQAAQGkoRgEAAFAailEAAACUhmIUAAAApaEYBQAAQGn+P4y6FxPo\n0Jo3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18ec3f49e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "image = X_train[1000]\n",
    "\n",
    "shifted_image_down = shift_image(image, 0, 5)\n",
    "shifted_image_left = shift_image(image, -5, 0)\n",
    "\n",
    "plt.figure(figsize=(12,3))\n",
    "plt.subplot(131)\n",
    "plt.title('Original', fontsize=14)\n",
    "plt.imshow(image.reshape(28,28), interpolation='nearest', cmap=\"Greys\")\n",
    "plt.subplot(132)\n",
    "plt.title('Shift down', fontsize=14)\n",
    "plt.imshow(shifted_image_down.reshape(28, 28), interpolation='nearest', cmap=\"Greys\")\n",
    "plt.subplot(133)\n",
    "plt.title('Shift left', fontsize=14)\n",
    "plt.imshow(shifted_image_left.reshape(28, 28), interpolation='nearest', cmap='Greys')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_augmented = [image for image in X_train]\n",
    "y_train_augmented = [y for y in y_train]\n",
    "\n",
    "for dx, dy in ((1,0), (0, 1), (-1, 0), (0, -1)):\n",
    "    for image, label in zip(X_train, y_train):\n",
    "        X_train_augmented.append(shift_image(image, dx, dy))\n",
    "        y_train_augmented.append(label)\n",
    "        \n",
    "X_train_augmented = np.array(X_train_augmented)\n",
    "y_train_augmented = np.array(y_train_augmented)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shuffle_idx = np.random.permutation(len(X_train_augmented))\n",
    "X_train_augmented = X_train_augmented[shuffle_idx]\n",
    "y_train_augmented = y_train_augmented[shuffle_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn_clf = KNeighborsClassifier(n_neighbors=4, weights='distance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=4, p=2,\n",
       "           weights='distance')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_augmented, y_train_augmented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97629999999999995"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = knn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tackle the Titanic dataset  \n",
    "\n",
    "The goal is to predict whether or not a passenger survived based on attributes such as their age, sex, passenger class, where they embarked and so on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "TITANIC_PATH = os.path.join(\"datasets\", \"titanic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def load_titanic_data(filename, titanic_path=TITANIC_PATHNIC_PATH):\n",
    "    csv_path = os.path.join(titanic_path, filename)\n",
    "    return pd.read_csv(csv_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = load_titanic_data(\"train.csv\")\n",
    "test_data = load_titanic_data(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The attributes have the following meaning:\n",
    "\n",
    "- Survived: that's the target, 0 means the passenger did not survive, while 1 means he/she survived.\n",
    "- Pclass: passenger class.\n",
    "- Name, Sex, Age: self-explanatory\n",
    "- SibSp: how many siblings & spouses of the passenger aboard the Titanic.\n",
    "- Parch: how many children & parents of the passenger aboard the Titanic.\n",
    "- Ticket: ticket id\n",
    "- Fare: price paid (in pounds)\n",
    "- Cabin: passenger's cabin number\n",
    "- Embarked: where the passenger embarked the Titanic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      "PassengerId    891 non-null int64\n",
      "Survived       891 non-null int64\n",
      "Pclass         891 non-null int64\n",
      "Name           891 non-null object\n",
      "Sex            891 non-null object\n",
      "Age            714 non-null float64\n",
      "SibSp          891 non-null int64\n",
      "Parch          891 non-null int64\n",
      "Ticket         891 non-null object\n",
      "Fare           891 non-null float64\n",
      "Cabin          204 non-null object\n",
      "Embarked       889 non-null object\n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.6+ KB\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Okay, the <b>Age, Cabin</b> and <b>Embarked</b> attributes are sometimes null (less than 891 non-null), especially the Cabin (77% are null). We will ignore the <b>Cabin</b> for now and focus on the rest. The <b>Age</b> attribute has about 19% null values, so we will need to decide what to do with them. Replacing null values with the median age seems reasonable.\n",
    "\n",
    "The <b>Name</b> and <b>Ticket</b> attributes may have some value, but they will be a bit tricky to convert into useful numbers that a model can consume. So for now, we will ignore them.\n",
    "\n",
    "Let's take a look at the numerical attributes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>714.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>257.353842</td>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>14.526497</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>223.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>668.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
       "count   891.000000  891.000000  891.000000  714.000000  891.000000   \n",
       "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
       "std     257.353842    0.486592    0.836071   14.526497    1.102743   \n",
       "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
       "25%     223.500000    0.000000    2.000000   20.125000    0.000000   \n",
       "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
       "75%     668.500000    1.000000    3.000000   38.000000    1.000000   \n",
       "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
       "\n",
       "            Parch        Fare  \n",
       "count  891.000000  891.000000  \n",
       "mean     0.381594   32.204208  \n",
       "std      0.806057   49.693429  \n",
       "min      0.000000    0.000000  \n",
       "25%      0.000000    7.910400  \n",
       "50%      0.000000   14.454200  \n",
       "75%      0.000000   31.000000  \n",
       "max      6.000000  512.329200  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    549\n",
       "1    342\n",
       "Name: Survived, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Survived\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    491\n",
       "1    216\n",
       "2    184\n",
       "Name: Pclass, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Pclass\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male      577\n",
       "female    314\n",
       "Name: Sex, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Sex\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "S    644\n",
       "C    168\n",
       "Q     77\n",
       "Name: Embarked, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "train_data[\"Embarked\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The <b>CategoricalEncoder</b> class will allow us to convert categorical attributes to one-hot vectors. It will soon be added to Scikit-Learn, and in the meantime you can use the code below (copied from Pull Request #9151)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Definition of the CategoricalEncoder class, copied from PR #9151.\n",
    "# Just run this cell, or copy it to your code, no need to try to\n",
    "# understand every line.\n",
    "\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.utils import check_array\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from scipy import sparse\n",
    "\n",
    "class CategoricalEncoder(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"Encode categorical features as a numeric array.\n",
    "    The input to this transformer should be a matrix of integers or strings,\n",
    "    denoting the values taken on by categorical (discrete) features.\n",
    "    The features can be encoded using a one-hot aka one-of-K scheme\n",
    "    (``encoding='onehot'``, the default) or converted to ordinal integers\n",
    "    (``encoding='ordinal'``).\n",
    "    This encoding is needed for feeding categorical data to many scikit-learn\n",
    "    estimators, notably linear models and SVMs with the standard kernels.\n",
    "    Read more in the :ref:`User Guide <preprocessing_categorical_features>`.\n",
    "    Parameters\n",
    "    ----------\n",
    "    encoding : str, 'onehot', 'onehot-dense' or 'ordinal'\n",
    "        The type of encoding to use (default is 'onehot'):\n",
    "        - 'onehot': encode the features using a one-hot aka one-of-K scheme\n",
    "          (or also called 'dummy' encoding). This creates a binary column for\n",
    "          each category and returns a sparse matrix.\n",
    "        - 'onehot-dense': the same as 'onehot' but returns a dense array\n",
    "          instead of a sparse matrix.\n",
    "        - 'ordinal': encode the features as ordinal integers. This results in\n",
    "          a single column of integers (0 to n_categories - 1) per feature.\n",
    "    categories : 'auto' or a list of lists/arrays of values.\n",
    "        Categories (unique values) per feature:\n",
    "        - 'auto' : Determine categories automatically from the training data.\n",
    "        - list : ``categories[i]`` holds the categories expected in the ith\n",
    "          column. The passed categories are sorted before encoding the data\n",
    "          (used categories can be found in the ``categories_`` attribute).\n",
    "    dtype : number type, default np.float64\n",
    "        Desired dtype of output.\n",
    "    handle_unknown : 'error' (default) or 'ignore'\n",
    "        Whether to raise an error or ignore if a unknown categorical feature is\n",
    "        present during transform (default is to raise). When this is parameter\n",
    "        is set to 'ignore' and an unknown category is encountered during\n",
    "        transform, the resulting one-hot encoded columns for this feature\n",
    "        will be all zeros.\n",
    "        Ignoring unknown categories is not supported for\n",
    "        ``encoding='ordinal'``.\n",
    "    Attributes\n",
    "    ----------\n",
    "    categories_ : list of arrays\n",
    "        The categories of each feature determined during fitting. When\n",
    "        categories were specified manually, this holds the sorted categories\n",
    "        (in order corresponding with output of `transform`).\n",
    "    Examples\n",
    "    --------\n",
    "    Given a dataset with three features and two samples, we let the encoder\n",
    "    find the maximum value per feature and transform the data to a binary\n",
    "    one-hot encoding.\n",
    "    >>> from sklearn.preprocessing import CategoricalEncoder\n",
    "    >>> enc = CategoricalEncoder(handle_unknown='ignore')\n",
    "    >>> enc.fit([[0, 0, 3], [1, 1, 0], [0, 2, 1], [1, 0, 2]])\n",
    "    ... # doctest: +ELLIPSIS\n",
    "    CategoricalEncoder(categories='auto', dtype=<... 'numpy.float64'>,\n",
    "              encoding='onehot', handle_unknown='ignore')\n",
    "    >>> enc.transform([[0, 1, 1], [1, 0, 4]]).toarray()\n",
    "    array([[ 1.,  0.,  0.,  1.,  0.,  0.,  1.,  0.,  0.],\n",
    "           [ 0.,  1.,  1.,  0.,  0.,  0.,  0.,  0.,  0.]])\n",
    "    See also\n",
    "    --------\n",
    "    sklearn.preprocessing.OneHotEncoder : performs a one-hot encoding of\n",
    "      integer ordinal features. The ``OneHotEncoder assumes`` that input\n",
    "      features take on values in the range ``[0, max(feature)]`` instead of\n",
    "      using the unique values.\n",
    "    sklearn.feature_extraction.DictVectorizer : performs a one-hot encoding of\n",
    "      dictionary items (also handles string-valued features).\n",
    "    sklearn.feature_extraction.FeatureHasher : performs an approximate one-hot\n",
    "      encoding of dictionary items or strings.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, encoding='onehot', categories='auto', dtype=np.float64,\n",
    "                 handle_unknown='error'):\n",
    "        self.encoding = encoding\n",
    "        self.categories = categories\n",
    "        self.dtype = dtype\n",
    "        self.handle_unknown = handle_unknown\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        \"\"\"Fit the CategoricalEncoder to X.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like, shape [n_samples, n_feature]\n",
    "            The data to determine the categories of each feature.\n",
    "        Returns\n",
    "        -------\n",
    "        self\n",
    "        \"\"\"\n",
    "\n",
    "        if self.encoding not in ['onehot', 'onehot-dense', 'ordinal']:\n",
    "            template = (\"encoding should be either 'onehot', 'onehot-dense' \"\n",
    "                        \"or 'ordinal', got %s\")\n",
    "            raise ValueError(template % self.handle_unknown)\n",
    "\n",
    "        if self.handle_unknown not in ['error', 'ignore']:\n",
    "            template = (\"handle_unknown should be either 'error' or \"\n",
    "                        \"'ignore', got %s\")\n",
    "            raise ValueError(template % self.handle_unknown)\n",
    "\n",
    "        if self.encoding == 'ordinal' and self.handle_unknown == 'ignore':\n",
    "            raise ValueError(\"handle_unknown='ignore' is not supported for\"\n",
    "                             \" encoding='ordinal'\")\n",
    "\n",
    "        X = check_array(X, dtype=np.object, accept_sparse='csc', copy=True)\n",
    "        n_samples, n_features = X.shape\n",
    "\n",
    "        self._label_encoders_ = [LabelEncoder() for _ in range(n_features)]\n",
    "\n",
    "        for i in range(n_features):\n",
    "            le = self._label_encoders_[i]\n",
    "            Xi = X[:, i]\n",
    "            if self.categories == 'auto':\n",
    "                le.fit(Xi)\n",
    "            else:\n",
    "                valid_mask = np.in1d(Xi, self.categories[i])\n",
    "                if not np.all(valid_mask):\n",
    "                    if self.handle_unknown == 'error':\n",
    "                        diff = np.unique(Xi[~valid_mask])\n",
    "                        msg = (\"Found unknown categories {0} in column {1}\"\n",
    "                               \" during fit\".format(diff, i))\n",
    "                        raise ValueError(msg)\n",
    "                le.classes_ = np.array(np.sort(self.categories[i]))\n",
    "\n",
    "        self.categories_ = [le.classes_ for le in self._label_encoders_]\n",
    "\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        \"\"\"Transform X using one-hot encoding.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like, shape [n_samples, n_features]\n",
    "            The data to encode.\n",
    "        Returns\n",
    "        -------\n",
    "        X_out : sparse matrix or a 2-d array\n",
    "            Transformed input.\n",
    "        \"\"\"\n",
    "        X = check_array(X, accept_sparse='csc', dtype=np.object, copy=True)\n",
    "        n_samples, n_features = X.shape\n",
    "        X_int = np.zeros_like(X, dtype=np.int)\n",
    "        X_mask = np.ones_like(X, dtype=np.bool)\n",
    "\n",
    "        for i in range(n_features):\n",
    "            valid_mask = np.in1d(X[:, i], self.categories_[i])\n",
    "\n",
    "            if not np.all(valid_mask):\n",
    "                if self.handle_unknown == 'error':\n",
    "                    diff = np.unique(X[~valid_mask, i])\n",
    "                    msg = (\"Found unknown categories {0} in column {1}\"\n",
    "                           \" during transform\".format(diff, i))\n",
    "                    raise ValueError(msg)\n",
    "                else:\n",
    "                    # Set the problematic rows to an acceptable value and\n",
    "                    # continue `The rows are marked `X_mask` and will be\n",
    "                    # removed later.\n",
    "                    X_mask[:, i] = valid_mask\n",
    "                    X[:, i][~valid_mask] = self.categories_[i][0]\n",
    "            X_int[:, i] = self._label_encoders_[i].transform(X[:, i])\n",
    "\n",
    "        if self.encoding == 'ordinal':\n",
    "            return X_int.astype(self.dtype, copy=False)\n",
    "\n",
    "        mask = X_mask.ravel()\n",
    "        n_values = [cats.shape[0] for cats in self.categories_]\n",
    "        n_values = np.array([0] + n_values)\n",
    "        indices = np.cumsum(n_values)\n",
    "\n",
    "        column_indices = (X_int + indices[:-1]).ravel()[mask]\n",
    "        row_indices = np.repeat(np.arange(n_samples, dtype=np.int32),\n",
    "                                n_features)[mask]\n",
    "        data = np.ones(n_samples * n_features)[mask]\n",
    "\n",
    "        out = sparse.csc_matrix((data, (row_indices, column_indices)),\n",
    "                                shape=(n_samples, indices[-1]),\n",
    "                                dtype=self.dtype).tocsr()\n",
    "        if self.encoding == 'onehot-dense':\n",
    "            return out.toarray()\n",
    "        else:\n",
    "            return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's build our preprocessing pipelines. We will reuse the <b>DataframeSelector</b> we built in the previous chapter to select specific attributes from the DataFrame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "# A class to select numerical or categorical columns \n",
    "# since Scikit-Learn doesn't handle DataFrames yet\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's build the pipeline for the numerical attributes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import Imputer\n",
    "\n",
    "imputer = Imputer(strategy=\"median\")\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "        (\"select_numeric\", DataFrameSelector([\"Age\", \"SibSp\", \"Parch\", \"Fare\"])),\n",
    "        (\"imputer\", Imputer(strategy=\"median\")),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 22.    ,   1.    ,   0.    ,   7.25  ],\n",
       "       [ 38.    ,   1.    ,   0.    ,  71.2833],\n",
       "       [ 26.    ,   0.    ,   0.    ,   7.925 ],\n",
       "       ..., \n",
       "       [ 28.    ,   1.    ,   2.    ,  23.45  ],\n",
       "       [ 26.    ,   0.    ,   0.    ,  30.    ],\n",
       "       [ 32.    ,   0.    ,   0.    ,   7.75  ]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_pipeline.fit_transform(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Inspired from stackoverflow.com/questions/25239958\n",
    "class MostFrequentImputer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        self.most_frequent = pd.Series([X[c].value_counts().index[0] for c in X],\n",
    "                                       index=X.columns)\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        return X.fillna(self.most_frequent)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can build the pipeline for the categorical attributes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cat_pipeline = Pipeline([\n",
    "        (\"select_cat\", DataFrameSelector([\"Pclass\", \"Sex\", \"Embarked\"])),\n",
    "        (\"imputer\", MostFrequentImputer()),\n",
    "        (\"cat_encoder\", CategoricalEncoder(encoding='onehot-dense')),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  1., ...,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0., ...,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  1., ...,  0.,  0.,  1.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  1., ...,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0., ...,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  1., ...,  0.,  1.,  0.]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_pipeline.fit_transform(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's join the numerical and categorical pipelines:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import FeatureUnion\n",
    "preprocess_pipeline = FeatureUnion(transformer_list=[\n",
    "        (\"num_pipeline\", num_pipeline),\n",
    "        (\"cat_pipeline\", cat_pipeline),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 22.,   1.,   0., ...,   0.,   0.,   1.],\n",
       "       [ 38.,   1.,   0., ...,   1.,   0.,   0.],\n",
       "       [ 26.,   0.,   0., ...,   0.,   0.,   1.],\n",
       "       ..., \n",
       "       [ 28.,   1.,   2., ...,   0.,   0.,   1.],\n",
       "       [ 26.,   0.,   0., ...,   1.,   0.,   0.],\n",
       "       [ 32.,   0.,   0., ...,   0.,   1.,   0.]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = preprocess_pipeline.fit_transform(train_data)\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train_data[\"Survived\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svm_clf = SVC()\n",
    "svm_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = preprocess_pipeline.transform(test_data)\n",
    "y_pred = svm_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.73652508228350921"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "scores = cross_val_score(svm_clf, X_train, y_train, cv=10)\n",
    "scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8115690614005221"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = RandomForestClassifier(random_state=42)\n",
    "scores = cross_val_score(forest_clf, X_train, y_train, cv=10)\n",
    "scores.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's much better!\n",
    "\n",
    "To improve this result further, you could:\n",
    "\n",
    "- Compare many more models and tune hyperparameters using cross validation and grid search,\n",
    "- Do more feature engineering, for example:\n",
    "    - replace <b>SibSp</b> and Parch with their sum,\n",
    "    - try to identify parts of names that correlate well with the <b>Survived</b> attribute (e.g. if the name contains \"Countess\", then survival seems more likely),\n",
    "    - try to convert numerical attributes to categorical attributes: for example, different age groups had very different survival rates (see below), so it may help to create an age bucket category and use it instead of the age. Similarly, it may be useful to have a special category for people traveling alone since only 30% of them survived (see below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AgeBucket</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.576923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15.0</th>\n",
       "      <td>0.362745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30.0</th>\n",
       "      <td>0.423256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45.0</th>\n",
       "      <td>0.404494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60.0</th>\n",
       "      <td>0.240000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75.0</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Survived\n",
       "AgeBucket          \n",
       "0.0        0.576923\n",
       "15.0       0.362745\n",
       "30.0       0.423256\n",
       "45.0       0.404494\n",
       "60.0       0.240000\n",
       "75.0       1.000000"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"AgeBucket\"] = train_data[\"Age\"] // 15 * 15\n",
    "train_data[[\"AgeBucket\", \"Survived\"]].groupby(['AgeBucket']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RelativesOnboard</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.303538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.552795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.578431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.724138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.136364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Survived\n",
       "RelativesOnboard          \n",
       "0                 0.303538\n",
       "1                 0.552795\n",
       "2                 0.578431\n",
       "3                 0.724138\n",
       "4                 0.200000\n",
       "5                 0.136364\n",
       "6                 0.333333\n",
       "7                 0.000000\n",
       "10                0.000000"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"RelativesOnboard\"] = train_data[\"SibSp\"] + train_data[\"Parch\"]\n",
    "train_data[[\"RelativesOnboard\", \"Survived\"]].groupby(['RelativesOnboard']).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spam classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "from six.moves import urllib\n",
    "\n",
    "DOWNLOAD_ROOT = \"http://spamassassin.apache.org/old/publiccorpus/\"\n",
    "HAM_URL = DOWNLOAD_ROOT + \"20030228_easy_ham.tar.bz2\"\n",
    "SPAM_URL = DOWNLOAD_ROOT + \"20030228_spam.tar.bz2\"\n",
    "SPAM_PATH = os.path.join(\"datasets\", \"spam\")\n",
    "\n",
    "def fetch_spam_data(spam_url=SPAM_URL, spam_path=SPAM_PATH):\n",
    "    if not os.path.isdir(spam_path):\n",
    "        os.makedirs(spam_path)\n",
    "    for filename, url in ((\"ham.tar.bz2\", HAM_URL), (\"spam.tar.bz2\", SPAM_URL)):\n",
    "        path = os.path.join(spam_path, filename)\n",
    "        if not os.path.isfile(path):\n",
    "            urllib.request.urlretrieve(url, path)\n",
    "        tar_bz2_file = tarfile.open(path)\n",
    "        tar_bz2_file.extractall(path=SPAM_PATH)\n",
    "        tar_bz2_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fetch_spam_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "HAM_DIR = os.path.join(SPAM_PATH, \"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH, \"spam\")\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name) > 20]\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name) > 20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2500"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ham_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(spam_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email(is_spam, filename, spam_path=SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(spam_path, directory, filename), \"rb\") as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ham_emails = [load_email(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam=True, filename=name) for name in spam_filenames]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Martin A posted:\n",
      "Tassos Papadopoulos, the Greek sculptor behind the plan, judged that the\n",
      " limestone of Mount Kerdylio, 70 miles east of Salonika and not far from the\n",
      " Mount Athos monastic community, was ideal for the patriotic sculpture. \n",
      " \n",
      " As well as Alexander's granite features, 240 ft high and 170 ft wide, a\n",
      " museum, a restored amphitheatre and car park for admiring crowds are\n",
      "planned\n",
      "---------------------\n",
      "So is this mountain limestone or granite?\n",
      "If it's limestone, it'll weather pretty fast.\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "4 DVDs Free +s&p Join Now\n",
      "http://us.click.yahoo.com/pt6YBB/NXiEAA/mG3HAA/7gSolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "forteana-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "print(ham_emails[1].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help wanted.  We are a 14 year old fortune 500 company, that is\n",
      "growing at a tremendous rate.  We are looking for individuals who\n",
      "want to work from home.\n",
      "\n",
      "This is an opportunity to make an excellent income.  No experience\n",
      "is required.  We will train you.\n",
      "\n",
      "So if you are looking to be employed from home with a career that has\n",
      "vast opportunities, then go:\n",
      "\n",
      "http://www.basetel.com/wealthnow\n",
      "\n",
      "We are looking for energetic and self motivated people.  If that is you\n",
      "than click on the link and fill out the form, and one of our\n",
      "employement specialist will contact you.\n",
      "\n",
      "To be removed from our link simple go to:\n",
      "\n",
      "http://www.basetel.com/remove.html\n",
      "\n",
      "\n",
      "4139vOLW7-758DoDY1425FRhM1-764SMFc8513fCsLl40\n"
     ]
    }
   ],
   "source": [
    "print(spam_emails[6].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_email_structure(email):\n",
    "    if isinstance(email, str):\n",
    "        return email\n",
    "    payload = email.get_payload()\n",
    "    if isinstance(payload, list):\n",
    "        return \"multipart({})\".format(\", \".join([\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload\n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 2408),\n",
       " ('multipart(text/plain, application/pgp-signature)', 66),\n",
       " ('multipart(text/plain, text/html)', 8),\n",
       " ('multipart(text/plain, text/plain)', 4),\n",
       " ('multipart(text/plain)', 3),\n",
       " ('multipart(text/plain, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, text/enriched)', 1),\n",
       " ('multipart(text/plain, application/ms-tnef, text/plain)', 1),\n",
       " ('multipart(multipart(text/plain, text/plain, text/plain), application/pgp-signature)',\n",
       "  1),\n",
       " ('multipart(text/plain, video/mng)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain))', 1),\n",
       " ('multipart(text/plain, application/x-pkcs7-signature)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), text/rfc822-headers)',\n",
       "  1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), multipart(multipart(text/plain, application/x-pkcs7-signature)))',\n",
       "  1),\n",
       " ('multipart(text/plain, application/x-java-applet)', 1)]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(ham_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 218),\n",
       " ('text/html', 183),\n",
       " ('multipart(text/plain, text/html)', 45),\n",
       " ('multipart(text/html)', 20),\n",
       " ('multipart(text/plain)', 19),\n",
       " ('multipart(multipart(text/html))', 5),\n",
       " ('multipart(text/plain, image/jpeg)', 3),\n",
       " ('multipart(text/html, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, application/octet-stream)', 1),\n",
       " ('multipart(text/html, text/plain)', 1),\n",
       " ('multipart(multipart(text/html), application/octet-stream, image/jpeg)', 1),\n",
       " ('multipart(multipart(text/plain, text/html), image/gif)', 1),\n",
       " ('multipart/alternative', 1)]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return-Path : <12a1mailbot1@web.de>\n",
      "Delivered-To : zzzz@localhost.spamassassin.taint.org\n",
      "Received : from localhost (localhost [127.0.0.1])\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\n",
      "Received : from mail.webnote.net [193.120.211.219]\tby localhost with POP3 (fetchmail-5.9.0)\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\n",
      "Received : from dd_it7 ([210.97.77.167])\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\n",
      "From : 12a1mailbot1@web.de\n",
      "Received : from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\t Sat, 24 Aug 2002 09:42:10 +0900\n",
      "To : dcek1a1@netsgo.com\n",
      "Subject : Life Insurance - Why Pay More?\n",
      "Date : Wed, 21 Aug 2002 20:31:57 -1600\n",
      "MIME-Version : 1.0\n",
      "Message-ID : <0103c1042001882DD_IT7@dd_it7>\n",
      "Content-Type : text/html; charset=\"iso-8859-1\"\n",
      "Content-Transfer-Encoding : quoted-printable\n"
     ]
    }
   ],
   "source": [
    "for header, value in spam_emails[0].items():\n",
    "    print(header,\":\",value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Life Insurance - Why Pay More?'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_emails[0][\"Subject\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that the ham emails are more often plain text, while spam has quite a lot of HTML. Moreover, quite a few ham emails are signed using PGP, while no spam is. In short, it seems that the email structure is a usual information to have."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Okay, before we learn too much about the data, let's not forget to split it into a training set and a test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Okay, let's start writing the preprocessing functions. First, we will need a function to convert HTML to plain text. Arguably the best way to do this would be to use the great BeautifulSoup library, but I would like to avoid adding another dependency to this project, so let's hack a quick & dirty solution using regular expressions (at the risk of un̨ho͞ly radiańcé destro҉ying all enli̍̈́̂̈́ghtenment). The following function first drops the <head> section, then converts all <a> tags to the word HYPERLINK, then it gets rid of all HTML tags, leaving only the plain text. For readability, it also replaces multiple newlines with single newlines, and finally it unescapes html entities (such as &gt; or &nbsp;):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a\\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<.*?>', '', text, flags=re.M | re.S)\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', text, flags=re.M | re.S)\n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HTML><HEAD><TITLE></TITLE><META http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\"><STYLE>A:link {TEX-DECORATION: none}A:active {TEXT-DECORATION: none}A:visited {TEXT-DECORATION: none}A:hover {COLOR: #0033ff; TEXT-DECORATION: underline}</STYLE><META content=\"MSHTML 6.00.2713.1100\" name=\"GENERATOR\"></HEAD>\n",
      "<BODY text=\"#000000\" vLink=\"#0033ff\" link=\"#0033ff\" bgColor=\"#CCCC99\"><TABLE borderColor=\"#660000\" cellSpacing=\"0\" cellPadding=\"0\" border=\"0\" width=\"100%\"><TR><TD bgColor=\"#CCCC99\" valign=\"top\" colspan=\"2\" height=\"27\">\n",
      "<font size=\"6\" face=\"Arial, Helvetica, sans-serif\" color=\"#660000\">\n",
      "<b>OTC</b></font></TD></TR><TR><TD height=\"2\" bgcolor=\"#6a694f\">\n",
      "<font size=\"5\" face=\"Times New Roman, Times, serif\" color=\"#FFFFFF\">\n",
      "<b>&nbsp;Newsletter</b></font></TD><TD height=\"2\" bgcolor=\"#6a694f\"><div align=\"right\"><font color=\"#FFFFFF\">\n",
      "<b>Discover Tomorrow's Winners&nbsp;</b></font></div></TD></TR><TR><TD height=\"25\" colspan=\"2\" bgcolor=\"#CCCC99\"><table width=\"100%\" border=\"0\"  ...\n"
     ]
    }
   ],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train==1]\n",
    "                    if get_email_structure(email) == \"text/html\"]\n",
    "sample_html_spam = html_spam_emails[7]\n",
    "print(sample_html_spam.get_content().strip()[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Watch for analyst \"Strong Buy Recommendations\" and several advisory newsletters picking CBYI.  CBYI has filed to be traded on the OTCBB, share prices historically INCREASE when companies get listed on this larger trading exchange. CBYI is trading around 25 cents and should skyrocket to $2.66 - $3.25 a share in the near future.\n",
      "Put CBYI on your watch list, acquire a position TODAY.\n",
      "REASONS TO INVEST IN CBYI\n",
      "A profitable company and is on track to beat ALL earnings estimates!\n",
      "One of the FASTEST growing distributors in environmental & safety equipment instruments.\n",
      "Excellent management team, several EXCLUSIVE contracts.  IMPRESSIVE client list including the U.S. Air Force, Anheuser-Busch, Chevron Refining and Mitsubishi Heavy Industries, GE-Energy & Environmental Research.\n",
      "RAPIDLY GROWING INDUSTRY\n",
      "Industry revenues exceed $900 million, estimates indicate that there could be as much as $25 billi ...\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content())[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\", \"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except: # in case of encoding issues\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Wat ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:100], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computations => comput\n",
      "Computation => comput\n",
      "Computing => comput\n",
      "Computed => comput\n",
      "Compute => comput\n",
      "Compulsive => compuls\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import nltk\n",
    "\n",
    "    stemmer = nltk.PorterStemmer()\n",
    "    for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "        print(word, \"=>\", stemmer.stem(word))\n",
    "except ImportError:\n",
    "    print(\"Error: stemming requires the NLTK module.\")\n",
    "    stemmer = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will also need a way to replace URLs with the word \"URL\". For this, we could use hard core regular expressions but we will just use the urlextract library. You can install it with the following command (don't forget to activate your virtualenv first; if you don't have one, you will likely need administrator rights, or use the --user option):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['github.com', 'https://youtu.be/7Pq-S557XQU?t=3m32s']\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import urlextract # may require an Internet connection to download root domain names\n",
    "    \n",
    "    url_extractor = urlextract.URLExtract()\n",
    "    print(url_extractor.find_urls(\"Will it detect github.com and https://youtu.be/7Pq-S557XQU?t=3m32s\"))\n",
    "except ImportError:\n",
    "    print(\"Error: replacing URLs requires the urlextract module.\")\n",
    "    url_extractor = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are ready to put all this together into a transformer that we will use to convert emails to word counters. Note that we split sentences into words using Python's split() method, which uses whitespaces for word boundaries. This works for many written languages, but not all. For example, Chinese and Japanese scripts generally don't use spaces between words, and Vietnamese often uses spaces even between syllables. It's okay in this exercise, because the dataset is (mostly) in English."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n",
    "                 replace_urls=True, replace_numbers=True, stemming=True):\n",
    "        self.strip_headers = strip_headers\n",
    "        self.lower_case = lower_case\n",
    "        self.remove_punctuation = remove_punctuation\n",
    "        self.replace_urls = replace_urls\n",
    "        self.replace_numbers = replace_numbers\n",
    "        self.stemming = stemming\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"\n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls and url_extractor is not None:\n",
    "                urls = list(set(url_extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url: len(url), reverse=True)\n",
    "                for url in urls:\n",
    "                    text = text.replace(url, \" URL \")\n",
    "            if self.replace_numbers:\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', 'NUMBER', text)\n",
    "            if self.remove_punctuation:\n",
    "                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n",
    "            word_counts = Counter(text.split())\n",
    "            if self.stemming and stemmer is not None:\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word, count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)\n",
    "                    stemmed_word_counts[stemmed_word] += count\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ Counter({'chuck': 1, 'murcko': 1, 'wrote': 1, 'stuff': 1, 'yawn': 1, 'r': 1}),\n",
       "       Counter({'the': 11, 'of': 9, 'and': 8, 'all': 3, 'christian': 3, 'to': 3, 'by': 3, 'jefferson': 2, 'i': 2, 'have': 2, 'superstit': 2, 'one': 2, 'on': 2, 'been': 2, 'ha': 2, 'half': 2, 'rogueri': 2, 'teach': 2, 'jesu': 2, 'some': 1, 'interest': 1, 'quot': 1, 'url': 1, 'thoma': 1, 'examin': 1, 'known': 1, 'word': 1, 'do': 1, 'not': 1, 'find': 1, 'in': 1, 'our': 1, 'particular': 1, 'redeem': 1, 'featur': 1, 'they': 1, 'are': 1, 'alik': 1, 'found': 1, 'fabl': 1, 'mytholog': 1, 'million': 1, 'innoc': 1, 'men': 1, 'women': 1, 'children': 1, 'sinc': 1, 'introduct': 1, 'burnt': 1, 'tortur': 1, 'fine': 1, 'imprison': 1, 'what': 1, 'effect': 1, 'thi': 1, 'coercion': 1, 'make': 1, 'world': 1, 'fool': 1, 'other': 1, 'hypocrit': 1, 'support': 1, 'error': 1, 'over': 1, 'earth': 1, 'six': 1, 'histor': 1, 'american': 1, 'john': 1, 'e': 1, 'remsburg': 1, 'letter': 1, 'william': 1, 'short': 1, 'again': 1, 'becom': 1, 'most': 1, 'pervert': 1, 'system': 1, 'that': 1, 'ever': 1, 'shone': 1, 'man': 1, 'absurd': 1, 'untruth': 1, 'were': 1, 'perpetr': 1, 'upon': 1, 'a': 1, 'larg': 1, 'band': 1, 'dupe': 1, 'import': 1, 'led': 1, 'paul': 1, 'first': 1, 'great': 1, 'corrupt': 1}),\n",
       "       Counter({'url': 5, 's': 3, 'group': 3, 'to': 3, 'in': 2, 'forteana': 2, 'martin': 2, 'an': 2, 'and': 2, 'we': 2, 'is': 2, 'yahoo': 2, 'unsubscrib': 2, 'y': 1, 'adamson': 1, 'wrote': 1, 'for': 1, 'altern': 1, 'rather': 1, 'more': 1, 'factual': 1, 'base': 1, 'rundown': 1, 'on': 1, 'hamza': 1, 'career': 1, 'includ': 1, 'hi': 1, 'belief': 1, 'that': 1, 'all': 1, 'non': 1, 'muslim': 1, 'yemen': 1, 'should': 1, 'be': 1, 'murder': 1, 'outright': 1, 'know': 1, 'how': 1, 'unbias': 1, 'memri': 1, 'don': 1, 't': 1, 'html': 1, 'rob': 1, 'sponsor': 1, 'number': 1, 'dvd': 1, 'free': 1, 'p': 1, 'join': 1, 'now': 1, 'from': 1, 'thi': 1, 'send': 1, 'email': 1, 'your': 1, 'use': 1, 'of': 1, 'subject': 1})], dtype=object)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have the word counts, and we need to convert them to vectors. For this, we will build another transformer whose fit() method will build the vocabulary (an ordered list of the most common words) and whose transform() method will use the vocabulary to convert word counts to vectors. The output is a sparse matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size=1000):\n",
    "        self.vocabulary_size = vocabulary_size\n",
    "    def fit(self, X, y=None):\n",
    "        total_count = Counter()\n",
    "        for word_count in X:\n",
    "            for word, count in word_count.items():\n",
    "                total_count[word] += min(count, 10)\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):\n",
    "            for word, count in word_count.items():\n",
    "                rows.append(row)\n",
    "                cols.append(self.vocabulary_.get(word, 0))\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x11 sparse matrix of type '<class 'numpy.int32'>'\n",
       "\twith 20 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [99, 11,  9,  8,  1,  3,  3,  1,  3,  2,  3],\n",
       "       [65,  0,  1,  2,  5,  3,  1,  2,  0,  1,  0]], dtype=int32)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few_vectors.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What does this matrix mean? Well, the 65 in the third row, first column, means that the third email contains 65 words that are not part of the vocabulary. The 0 next to it means that the first word in the vocabulary is not present in this email. The 1 next to it means that the second word is present once, and so on. You can look at the vocabulary to know which words we are talking about. The first word is \"the\", the second word is \"of\", etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'all': 6,\n",
       " 'and': 3,\n",
       " 'by': 10,\n",
       " 'christian': 8,\n",
       " 'in': 7,\n",
       " 'of': 2,\n",
       " 'on': 9,\n",
       " 'the': 1,\n",
       " 'to': 5,\n",
       " 'url': 4}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer.vocabulary_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are now ready to train our first spam classifier! Let's transform the whole dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_vector\", WordCounterToVectorTransformer()),\n",
    "])\n",
    "\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV] ................................. , score=0.985000, total=   0.1s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................. , score=0.985000, total=   0.1s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................. , score=0.992500, total=   0.7s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    1.2s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.98749999999999993"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression()\n",
    "score = cross_val_score(log_clf, X_train_transformed, y_train, cv=3, verbose=3)\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Over 98.7%, not bad for a first try! :) However, remember that we are using the \"easy\" dataset. You can try with the harder datasets, the results won't be so amazing. You would have to try multiple models, select the best ones and fine-tune them using cross-validation, and so on.\n",
    "\n",
    "But you get the picture, so let's stop now, and just print out the precision/recall we get on the test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.95%\n",
      "Recall: 0.98%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "\n",
    "log_clf = LogisticRegression()\n",
    "log_clf.fit(X_train_transformed, y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(\"Precision: {:.2f}%\".format(precision_score(y_test, y_pred)))\n",
    "print(\"Recall: {:.2f}%\".format(recall_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}