SUSY Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SUSY Dataset\n",
    "See https://archive.ics.uci.edu/ml/datasets/SUSY for dataset information and feature descriptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import pickle\n",
    "import operator\n",
    "import glob\n",
    "from scipy.io.arff import loadarff \n",
    "from scipy.stats import ttest_rel\n",
    "\n",
    "import seaborn as sns; sns.set_style('white')\n",
    "\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import accuracy_score, plot_confusion_matrix, f1_score, plot_roc_curve, roc_auc_score, make_scorer\n",
    "from sklearn.model_selection import KFold, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>lepton_1_pT</th>\n",
       "      <th>lepton_1_eta</th>\n",
       "      <th>lepton_1_phi</th>\n",
       "      <th>lepton_2_pT</th>\n",
       "      <th>lepton_2_eta</th>\n",
       "      <th>lepton_2_phi</th>\n",
       "      <th>missing_energy_magnitude</th>\n",
       "      <th>missing_energy_phi</th>\n",
       "      <th>MET_rel</th>\n",
       "      <th>axial_MET</th>\n",
       "      <th>M_R</th>\n",
       "      <th>M_TR_2</th>\n",
       "      <th>R</th>\n",
       "      <th>MT2</th>\n",
       "      <th>S_R</th>\n",
       "      <th>M_Delta_R</th>\n",
       "      <th>dPhi_r_b</th>\n",
       "      <th>cos(theta_r1)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.972861</td>\n",
       "      <td>0.653855</td>\n",
       "      <td>1.176225</td>\n",
       "      <td>1.157156</td>\n",
       "      <td>-1.739873</td>\n",
       "      <td>-0.874309</td>\n",
       "      <td>0.567765</td>\n",
       "      <td>-0.175000</td>\n",
       "      <td>0.810061</td>\n",
       "      <td>-0.252552</td>\n",
       "      <td>1.921887</td>\n",
       "      <td>0.889637</td>\n",
       "      <td>0.410772</td>\n",
       "      <td>1.145621</td>\n",
       "      <td>1.932632</td>\n",
       "      <td>0.994464</td>\n",
       "      <td>1.367815</td>\n",
       "      <td>0.040714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.667973</td>\n",
       "      <td>0.064191</td>\n",
       "      <td>-1.225171</td>\n",
       "      <td>0.506102</td>\n",
       "      <td>-0.338939</td>\n",
       "      <td>1.672543</td>\n",
       "      <td>3.475464</td>\n",
       "      <td>-1.219136</td>\n",
       "      <td>0.012955</td>\n",
       "      <td>3.775174</td>\n",
       "      <td>1.045977</td>\n",
       "      <td>0.568051</td>\n",
       "      <td>0.481928</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.448410</td>\n",
       "      <td>0.205356</td>\n",
       "      <td>1.321893</td>\n",
       "      <td>0.377584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.444840</td>\n",
       "      <td>-0.134298</td>\n",
       "      <td>-0.709972</td>\n",
       "      <td>0.451719</td>\n",
       "      <td>-1.613871</td>\n",
       "      <td>-0.768661</td>\n",
       "      <td>1.219918</td>\n",
       "      <td>0.504026</td>\n",
       "      <td>1.831248</td>\n",
       "      <td>-0.431385</td>\n",
       "      <td>0.526283</td>\n",
       "      <td>0.941514</td>\n",
       "      <td>1.587535</td>\n",
       "      <td>2.024308</td>\n",
       "      <td>0.603498</td>\n",
       "      <td>1.562374</td>\n",
       "      <td>1.135454</td>\n",
       "      <td>0.180910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.381256</td>\n",
       "      <td>-0.976145</td>\n",
       "      <td>0.693152</td>\n",
       "      <td>0.448959</td>\n",
       "      <td>0.891753</td>\n",
       "      <td>-0.677328</td>\n",
       "      <td>2.033060</td>\n",
       "      <td>1.533041</td>\n",
       "      <td>3.046260</td>\n",
       "      <td>-1.005285</td>\n",
       "      <td>0.569386</td>\n",
       "      <td>1.015211</td>\n",
       "      <td>1.582217</td>\n",
       "      <td>1.551914</td>\n",
       "      <td>0.761215</td>\n",
       "      <td>1.715464</td>\n",
       "      <td>1.492257</td>\n",
       "      <td>0.090719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.309996</td>\n",
       "      <td>-0.690089</td>\n",
       "      <td>-0.676259</td>\n",
       "      <td>1.589283</td>\n",
       "      <td>-0.693326</td>\n",
       "      <td>0.622907</td>\n",
       "      <td>1.087562</td>\n",
       "      <td>-0.381742</td>\n",
       "      <td>0.589204</td>\n",
       "      <td>1.365479</td>\n",
       "      <td>1.179295</td>\n",
       "      <td>0.968218</td>\n",
       "      <td>0.728563</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.083158</td>\n",
       "      <td>0.043429</td>\n",
       "      <td>1.154854</td>\n",
       "      <td>0.094859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999995</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.853325</td>\n",
       "      <td>-0.961783</td>\n",
       "      <td>-1.487277</td>\n",
       "      <td>0.678190</td>\n",
       "      <td>0.493580</td>\n",
       "      <td>1.647969</td>\n",
       "      <td>1.843867</td>\n",
       "      <td>0.276954</td>\n",
       "      <td>1.025105</td>\n",
       "      <td>-1.486535</td>\n",
       "      <td>0.892879</td>\n",
       "      <td>1.684429</td>\n",
       "      <td>1.674084</td>\n",
       "      <td>3.366298</td>\n",
       "      <td>1.046707</td>\n",
       "      <td>2.646649</td>\n",
       "      <td>1.389226</td>\n",
       "      <td>0.364599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.951581</td>\n",
       "      <td>0.139370</td>\n",
       "      <td>1.436884</td>\n",
       "      <td>0.880440</td>\n",
       "      <td>-0.351948</td>\n",
       "      <td>-0.740852</td>\n",
       "      <td>0.290863</td>\n",
       "      <td>-0.732360</td>\n",
       "      <td>0.001360</td>\n",
       "      <td>0.257738</td>\n",
       "      <td>0.802871</td>\n",
       "      <td>0.545319</td>\n",
       "      <td>0.602730</td>\n",
       "      <td>0.002998</td>\n",
       "      <td>0.748959</td>\n",
       "      <td>0.401166</td>\n",
       "      <td>0.443471</td>\n",
       "      <td>0.239953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.840389</td>\n",
       "      <td>1.419162</td>\n",
       "      <td>-1.218766</td>\n",
       "      <td>1.195631</td>\n",
       "      <td>1.695645</td>\n",
       "      <td>0.663756</td>\n",
       "      <td>0.490888</td>\n",
       "      <td>-0.509186</td>\n",
       "      <td>0.704289</td>\n",
       "      <td>0.045744</td>\n",
       "      <td>0.825015</td>\n",
       "      <td>0.723530</td>\n",
       "      <td>0.778236</td>\n",
       "      <td>0.752942</td>\n",
       "      <td>0.838953</td>\n",
       "      <td>0.614048</td>\n",
       "      <td>1.210595</td>\n",
       "      <td>0.026692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999998</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.784218</td>\n",
       "      <td>-0.833565</td>\n",
       "      <td>-0.560091</td>\n",
       "      <td>0.953342</td>\n",
       "      <td>-0.688969</td>\n",
       "      <td>-1.428233</td>\n",
       "      <td>2.660703</td>\n",
       "      <td>-0.861344</td>\n",
       "      <td>2.116892</td>\n",
       "      <td>2.906151</td>\n",
       "      <td>1.232334</td>\n",
       "      <td>0.952444</td>\n",
       "      <td>0.685846</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.781874</td>\n",
       "      <td>0.676003</td>\n",
       "      <td>1.197807</td>\n",
       "      <td>0.093689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.761500</td>\n",
       "      <td>0.680454</td>\n",
       "      <td>-1.186213</td>\n",
       "      <td>1.043521</td>\n",
       "      <td>-0.316755</td>\n",
       "      <td>0.246879</td>\n",
       "      <td>1.120280</td>\n",
       "      <td>0.998479</td>\n",
       "      <td>1.640881</td>\n",
       "      <td>-0.797688</td>\n",
       "      <td>0.854212</td>\n",
       "      <td>1.121858</td>\n",
       "      <td>1.165438</td>\n",
       "      <td>1.498351</td>\n",
       "      <td>0.931580</td>\n",
       "      <td>1.293524</td>\n",
       "      <td>1.539167</td>\n",
       "      <td>0.187496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5000000 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         class  lepton_1_pT  lepton_1_eta  lepton_1_phi  lepton_2_pT  \\\n",
       "0          0.0     0.972861      0.653855      1.176225     1.157156   \n",
       "1          1.0     1.667973      0.064191     -1.225171     0.506102   \n",
       "2          1.0     0.444840     -0.134298     -0.709972     0.451719   \n",
       "3          1.0     0.381256     -0.976145      0.693152     0.448959   \n",
       "4          1.0     1.309996     -0.690089     -0.676259     1.589283   \n",
       "...        ...          ...           ...           ...          ...   \n",
       "4999995    1.0     0.853325     -0.961783     -1.487277     0.678190   \n",
       "4999996    0.0     0.951581      0.139370      1.436884     0.880440   \n",
       "4999997    0.0     0.840389      1.419162     -1.218766     1.195631   \n",
       "4999998    1.0     1.784218     -0.833565     -0.560091     0.953342   \n",
       "4999999    0.0     0.761500      0.680454     -1.186213     1.043521   \n",
       "\n",
       "         lepton_2_eta  lepton_2_phi  missing_energy_magnitude  \\\n",
       "0           -1.739873     -0.874309                  0.567765   \n",
       "1           -0.338939      1.672543                  3.475464   \n",
       "2           -1.613871     -0.768661                  1.219918   \n",
       "3            0.891753     -0.677328                  2.033060   \n",
       "4           -0.693326      0.622907                  1.087562   \n",
       "...               ...           ...                       ...   \n",
       "4999995      0.493580      1.647969                  1.843867   \n",
       "4999996     -0.351948     -0.740852                  0.290863   \n",
       "4999997      1.695645      0.663756                  0.490888   \n",
       "4999998     -0.688969     -1.428233                  2.660703   \n",
       "4999999     -0.316755      0.246879                  1.120280   \n",
       "\n",
       "         missing_energy_phi   MET_rel  axial_MET       M_R    M_TR_2  \\\n",
       "0                 -0.175000  0.810061  -0.252552  1.921887  0.889637   \n",
       "1                 -1.219136  0.012955   3.775174  1.045977  0.568051   \n",
       "2                  0.504026  1.831248  -0.431385  0.526283  0.941514   \n",
       "3                  1.533041  3.046260  -1.005285  0.569386  1.015211   \n",
       "4                 -0.381742  0.589204   1.365479  1.179295  0.968218   \n",
       "...                     ...       ...        ...       ...       ...   \n",
       "4999995            0.276954  1.025105  -1.486535  0.892879  1.684429   \n",
       "4999996           -0.732360  0.001360   0.257738  0.802871  0.545319   \n",
       "4999997           -0.509186  0.704289   0.045744  0.825015  0.723530   \n",
       "4999998           -0.861344  2.116892   2.906151  1.232334  0.952444   \n",
       "4999999            0.998479  1.640881  -0.797688  0.854212  1.121858   \n",
       "\n",
       "                R       MT2       S_R  M_Delta_R  dPhi_r_b  cos(theta_r1)  \n",
       "0        0.410772  1.145621  1.932632   0.994464  1.367815       0.040714  \n",
       "1        0.481928  0.000000  0.448410   0.205356  1.321893       0.377584  \n",
       "2        1.587535  2.024308  0.603498   1.562374  1.135454       0.180910  \n",
       "3        1.582217  1.551914  0.761215   1.715464  1.492257       0.090719  \n",
       "4        0.728563  0.000000  1.083158   0.043429  1.154854       0.094859  \n",
       "...           ...       ...       ...        ...       ...            ...  \n",
       "4999995  1.674084  3.366298  1.046707   2.646649  1.389226       0.364599  \n",
       "4999996  0.602730  0.002998  0.748959   0.401166  0.443471       0.239953  \n",
       "4999997  0.778236  0.752942  0.838953   0.614048  1.210595       0.026692  \n",
       "4999998  0.685846  0.000000  0.781874   0.676003  1.197807       0.093689  \n",
       "4999999  1.165438  1.498351  0.931580   1.293524  1.539167       0.187496  \n",
       "\n",
       "[5000000 rows x 19 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('SUSY.csv', header=None)\n",
    "df.columns = ['class', 'lepton_1_pT', 'lepton_1_eta', 'lepton_1_phi', 'lepton_2_pT', 'lepton_2_eta', \n",
    "              'lepton_2_phi', 'missing_energy_magnitude', 'missing_energy_phi', 'MET_rel', 'axial_MET', \n",
    "              'M_R', 'M_TR_2', 'R', 'MT2', 'S_R', 'M_Delta_R', 'dPhi_r_b', 'cos(theta_r1)']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    2712173\n",
       "1.0    2287827\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['class'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    6000\n",
       "0.0    6000\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separate majority and minority classes\n",
    "df_majority = df[df['class']==0.0]\n",
    "df_minority = df[df['class']==1.0]\n",
    "\n",
    "# Downsample majority and minority class\n",
    "df_majority_downsampled = resample(df_majority, \n",
    "                                 replace=False,    # sample without replacement\n",
    "                                 n_samples=6000,     # to match minority class\n",
    "                                 random_state=123) # reproducible results\n",
    "\n",
    "df_minority_downsampled = resample(df_minority, \n",
    "                                 replace=False,    # sample without replacement\n",
    "                                 n_samples=6000,     # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    " \n",
    "# Combine minority class with downsampled majority class\n",
    "df_downsampled = pd.concat([df_majority_downsampled, df_minority_downsampled])\n",
    " \n",
    "# Display new class counts\n",
    "df_downsampled['class'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_downsampled.copy()\n",
    "scaler = StandardScaler()\n",
    "X, y = df.iloc[:,1:].to_numpy(), df.iloc[:,0].to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Search & Experimentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def experiment():\n",
    "    pipeline1 = Pipeline((\n",
    "    ('clf', RandomForestClassifier()),\n",
    "    ))\n",
    "\n",
    "    pipeline2 = Pipeline((\n",
    "    ('clf', KNeighborsClassifier()),\n",
    "    ))\n",
    "\n",
    "    pipeline3 = Pipeline((\n",
    "    ('clf', AdaBoostClassifier()),\n",
    "    ))\n",
    "    \n",
    "    pipeline4 = Pipeline((\n",
    "    ('clf', LogisticRegression()),\n",
    "    ))\n",
    "    \n",
    "    pipeline5 = Pipeline((\n",
    "    ('clf', MLPClassifier()),\n",
    "    ))\n",
    "    \n",
    "    # Random Forest\n",
    "    parameters1 = {\n",
    "    'clf__n_estimators': [1024],\n",
    "    'clf__max_features': [1, 2, 4, 6, 8, 12, 16, 20]\n",
    "    }\n",
    "\n",
    "    # KNN\n",
    "    parameters2 = {\n",
    "    'clf__n_neighbors': [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,65,69,73,77,81,85,89,93,97,101,105],\n",
    "    'clf__weights': ['uniform', 'distance']\n",
    "    }\n",
    "    \n",
    "    # AdaBoost (Boosted Decision Tree)\n",
    "    parameters3 = {\n",
    "        'clf__algorithm': ['SAMME.R'],\n",
    "        'clf__n_estimators': [2,4,8,16,32,64,128,256,512,1024,2048],\n",
    "        'clf__learning_rate': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 2e1, 5e1]\n",
    "    }\n",
    "    \n",
    "    # Logistic\n",
    "    parameters4 = {\n",
    "    'clf__penalty':['l1', 'l2', None],\n",
    "    'clf__C':[1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0, 1e0, 1e1, 1e2, 1e3, 1e4],\n",
    "    'clf__max_iter':[5000]\n",
    "    }\n",
    "    \n",
    "    # Multi-layer Perceptron\n",
    "    parameters5 = {\n",
    "        'clf__hidden_layer_sizes':[(1,), (2,), (4,), (8,), (32,), (128,)],\n",
    "        'clf__solver':['sgd'],\n",
    "        'clf__activation':['relu'], \n",
    "        'clf__learning_rate':['constant', 'invscaling'], \n",
    "        'clf__learning_rate_init': [1e-3, 1e-2, 1e-1, 1e0],\n",
    "        'clf__max_iter': [2, 4, 8, 16, 32, 64, 128, 256, 512]\n",
    "    }\n",
    "    \n",
    "    pars = [parameters1, parameters2, parameters3, parameters4, parameters5]\n",
    "    pips = [pipeline1, pipeline2, pipeline3, pipeline4, pipeline5]\n",
    "    \n",
    "    # List of dictionaries to hold the scores of the various metrics for each type of classifier\n",
    "    best_clf_list = []\n",
    "    trial_storage = {}\n",
    "    training_storage = {}\n",
    "    \n",
    "    print(\"starting Gridsearch\")\n",
    "    for i in range(len(pars)):\n",
    "        trial_averages = []\n",
    "        train_performance = []\n",
    "        for t in range(5):\n",
    "            # split and scale data\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/6, random_state=t)\n",
    "            X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.5, random_state=t)\n",
    "            X_train = scaler.fit_transform(X_train)\n",
    "            X_val = scaler.transform(X_val)\n",
    "            X_test = scaler.transform(X_test)\n",
    "                \n",
    "            clf = GridSearchCV(pips[i], pars[i], refit=False, n_jobs=8, cv=5, verbose=3, scoring=('accuracy', 'roc_auc', 'f1'))\n",
    "            clf = clf.fit(X_val, y_val)\n",
    "            \n",
    "            print(\"finished Gridsearch trial \" + str(t + 1) + \" classifier \" + str(i + 1))\n",
    "            print(\"\")\n",
    "            print(\"\")\n",
    "            \n",
    "            # find the best params for each metric in a given trial \n",
    "            best_index_acc = np.argmin(clf.cv_results_['rank_test_accuracy'])\n",
    "            best_params_acc = clf.cv_results_['params'][best_index_acc]\n",
    "            best_index_roc = np.argmin(clf.cv_results_['rank_test_roc_auc'])\n",
    "            best_params_roc = clf.cv_results_['params'][best_index_roc]\n",
    "            best_index_f1 = np.argmin(clf.cv_results_['rank_test_f1'])\n",
    "            best_params_f1 = clf.cv_results_['params'][best_index_f1]\n",
    "    \n",
    "            # train and test models for given metric with their corresponding best parameter settings\n",
    "            pipe = pips[i]\n",
    "            clf_acc = pipe.set_params(**best_params_acc)\n",
    "            clf_acc = clf_acc.fit(X_train, y_train)\n",
    "            clf_roc = pipe.set_params(**best_params_roc)\n",
    "            clf_roc = clf_roc.fit(X_train, y_train)\n",
    "            clf_f1 = pipe.set_params(**best_params_f1)\n",
    "            clf_f1 = clf_f1.fit(X_train, y_train)\n",
    "            \n",
    "            # get training set performance\n",
    "            train_acc = accuracy_score(y_train, clf_acc.predict(X_train))\n",
    "            train_roc = roc_auc_score(y_train, clf_roc.predict_proba(X_train)[:, 1])\n",
    "            train_f1 = f1_score(y_train, clf_f1.predict(X_train))\n",
    "            \n",
    "            train_performance.append({\n",
    "                'Model #': i + 1,\n",
    "                'average': (train_f1 + train_acc + train_roc)/3,\n",
    "                'accuracy': train_acc,\n",
    "                'roc_auc_score': train_roc,\n",
    "                'f1 score': train_f1\n",
    "            })\n",
    "            \n",
    "            # get test set performances \n",
    "            trial_acc = clf_acc.score(X_test, y_test)\n",
    "            trial_roc = roc_auc_score(y_test, clf_roc.predict_proba(X_test)[:, 1])\n",
    "            trial_f1 = f1_score(y_test, clf_f1.predict(X_test))\n",
    "            \n",
    "            # store scores and their averages in list containing averages for each trial\n",
    "            trial_averages.append({\n",
    "                'Model #': i + 1, # model number corresponds to the numbers used in pipeline above (i.e. 1 = Random Forest)\n",
    "                'average':(trial_acc + trial_roc + trial_f1) / 3,\n",
    "                'accuracy': trial_acc,\n",
    "                'roc_auc_score': trial_roc,\n",
    "                'f1_score': trial_f1\n",
    "            })\n",
    "            \n",
    "            train_performance.append({\n",
    "                'Model #': i + 1, # model number corresponds to the numbers used in pipeline above (i.e. 1 = Random Forest)\n",
    "                'average':(train_acc + train_roc + train_f1) / 3,\n",
    "                'accuracy': train_acc,\n",
    "                'roc_auc_score': train_roc,\n",
    "                'f1_score': train_f1\n",
    "            })\n",
    "            \n",
    "        # find the trial with the best average metric scores and append those scores as a dict to best clf list\n",
    "        max_average = 0\n",
    "        for trial in trial_averages:\n",
    "            if trial['average'] > max_average:\n",
    "                max_average = trial['average']\n",
    "                best_trial = trial\n",
    "\n",
    "        best_clf_list.append(best_trial)\n",
    "        training_storage[str(i + 1)]=train_performance\n",
    "        trial_storage[str(i + 1)]=trial_averages\n",
    "    \n",
    "    return best_clf_list, trial_storage, training_storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting Gridsearch\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 1 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 2 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 3 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 4 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 5 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 1 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 2 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 3 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 4 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 5 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 1 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 2 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 3 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 4 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 5 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 1 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 2 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 3 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 4 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 5 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 1 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 2 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 3 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 4 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 5 classifier 5\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%capture --no-stdout --no-display\n",
    "best_clf_list, trial_storage, training_perf = experiment()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating and Organizing Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Models On Average For Test Set:\n",
      "{'Model #': 1, 'average': 0.8155095263452585, 'accuracy': 0.7955, 'roc_auc_score': 0.8675452545254526, 'f1_score': 0.783483324510323}\n",
      "{'Model #': 2, 'average': 0.7731779562405686, 'accuracy': 0.7585, 'roc_auc_score': 0.8328402840284028, 'f1_score': 0.7281935846933033}\n",
      "{'Model #': 3, 'average': 0.8143371160478606, 'accuracy': 0.7915, 'roc_auc_score': 0.8665706570657066, 'f1_score': 0.7849406910778751}\n",
      "{'Model #': 4, 'average': 0.8116257306890109, 'accuracy': 0.791, 'roc_auc_score': 0.871051105110511, 'f1_score': 0.7728260869565217}\n",
      "{'Model #': 5, 'average': 0.8188073529165999, 'accuracy': 0.7965, 'roc_auc_score': 0.874019901990199, 'f1_score': 0.7859021567596003}\n",
      "\n",
      "Train Set Data\n",
      "[{'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}]\n",
      "[{'Model #': 2, 'average': 0.8119888062465478, 'accuracy': 0.7908, 'roc_auc_score': 0.8757660660059574, 'f1 score': 0.769400352733686}, {'Model #': 2, 'average': 0.8119888062465478, 'accuracy': 0.7908, 'roc_auc_score': 0.8757660660059574, 'f1_score': 0.769400352733686}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 2, 'average': 0.7869677586833174, 'accuracy': 0.7692, 'roc_auc_score': 0.8525531856340389, 'f1 score': 0.7391500904159133}, {'Model #': 2, 'average': 0.7869677586833174, 'accuracy': 0.7692, 'roc_auc_score': 0.8525531856340389, 'f1_score': 0.7391500904159133}]\n",
      "[{'Model #': 3, 'average': 0.8309274077230716, 'accuracy': 0.8074, 'roc_auc_score': 0.8863002285937922, 'f1 score': 0.7990819945754226}, {'Model #': 3, 'average': 0.8309274077230716, 'accuracy': 0.8074, 'roc_auc_score': 0.8863002285937922, 'f1_score': 0.7990819945754226}, {'Model #': 3, 'average': 0.8248132249523804, 'accuracy': 0.7996, 'roc_auc_score': 0.8830691012910563, 'f1 score': 0.7917705735660848}, {'Model #': 3, 'average': 0.8248132249523804, 'accuracy': 0.7996, 'roc_auc_score': 0.8830691012910563, 'f1_score': 0.7917705735660848}, {'Model #': 3, 'average': 0.8418045411090489, 'accuracy': 0.8174, 'roc_auc_score': 0.8996995355673312, 'f1 score': 0.8083140877598152}, {'Model #': 3, 'average': 0.8418045411090488, 'accuracy': 0.8174, 'roc_auc_score': 0.8996995355673312, 'f1_score': 0.8083140877598152}, {'Model #': 3, 'average': 0.8311218595332234, 'accuracy': 0.808, 'roc_auc_score': 0.8898819825195678, 'f1 score': 0.7954835960801023}, {'Model #': 3, 'average': 0.8311218595332234, 'accuracy': 0.808, 'roc_auc_score': 0.8898819825195678, 'f1_score': 0.7954835960801023}, {'Model #': 3, 'average': 0.8335477262592926, 'accuracy': 0.8086, 'roc_auc_score': 0.8910450502688323, 'f1 score': 0.8009981285090456}, {'Model #': 3, 'average': 0.8335477262592926, 'accuracy': 0.8086, 'roc_auc_score': 0.8910450502688323, 'f1_score': 0.8009981285090456}]\n",
      "[{'Model #': 4, 'average': 0.8067827633249033, 'accuracy': 0.7878, 'roc_auc_score': 0.8614587538366301, 'f1 score': 0.7710895361380797}, {'Model #': 4, 'average': 0.8067827633249033, 'accuracy': 0.7878, 'roc_auc_score': 0.8614587538366301, 'f1_score': 0.7710895361380797}, {'Model #': 4, 'average': 0.8036118350354551, 'accuracy': 0.7832, 'roc_auc_score': 0.860153737624598, 'f1 score': 0.7674817674817674}, {'Model #': 4, 'average': 0.8036118350354552, 'accuracy': 0.7832, 'roc_auc_score': 0.860153737624598, 'f1_score': 0.7674817674817674}, {'Model #': 4, 'average': 0.8142920919095308, 'accuracy': 0.7954, 'roc_auc_score': 0.8677130895068489, 'f1 score': 0.7797631862217437}, {'Model #': 4, 'average': 0.8142920919095308, 'accuracy': 0.7954, 'roc_auc_score': 0.8677130895068489, 'f1_score': 0.7797631862217437}, {'Model #': 4, 'average': 0.8080586788953648, 'accuracy': 0.7906, 'roc_auc_score': 0.8644470399937016, 'f1 score': 0.7691289966923925}, {'Model #': 4, 'average': 0.8080586788953648, 'accuracy': 0.7906, 'roc_auc_score': 0.8644470399937016, 'f1_score': 0.7691289966923925}, {'Model #': 4, 'average': 0.8036892496357865, 'accuracy': 0.7854, 'roc_auc_score': 0.8576677489073593, 'f1 score': 0.7680000000000001}, {'Model #': 4, 'average': 0.8036892496357865, 'accuracy': 0.7854, 'roc_auc_score': 0.8576677489073593, 'f1_score': 0.7680000000000001}]\n",
      "[{'Model #': 5, 'average': 0.8160180985920991, 'accuracy': 0.7932, 'roc_auc_score': 0.8726302014207792, 'f1 score': 0.782224094355518}, {'Model #': 5, 'average': 0.8160180985920991, 'accuracy': 0.7932, 'roc_auc_score': 0.8726302014207792, 'f1_score': 0.782224094355518}, {'Model #': 5, 'average': 0.8222340800148724, 'accuracy': 0.7992, 'roc_auc_score': 0.8771055803368928, 'f1 score': 0.7903966597077244}, {'Model #': 5, 'average': 0.8222340800148724, 'accuracy': 0.7992, 'roc_auc_score': 0.8771055803368928, 'f1_score': 0.7903966597077244}, {'Model #': 5, 'average': 0.8322117669504517, 'accuracy': 0.8104, 'roc_auc_score': 0.8844787136493476, 'f1 score': 0.8017565872020075}, {'Model #': 5, 'average': 0.8322117669504517, 'accuracy': 0.8104, 'roc_auc_score': 0.8844787136493476, 'f1_score': 0.8017565872020075}, {'Model #': 5, 'average': 0.8225718797427476, 'accuracy': 0.8024, 'roc_auc_score': 0.8817924139872262, 'f1 score': 0.7835232252410167}, {'Model #': 5, 'average': 0.8225718797427476, 'accuracy': 0.8024, 'roc_auc_score': 0.8817924139872262, 'f1_score': 0.7835232252410167}, {'Model #': 5, 'average': 0.8216220211479742, 'accuracy': 0.7998, 'roc_auc_score': 0.8758472805422596, 'f1 score': 0.7892187829016635}, {'Model #': 5, 'average': 0.8216220211479742, 'accuracy': 0.7998, 'roc_auc_score': 0.8758472805422596, 'f1_score': 0.7892187829016635}]\n",
      "\n",
      "set of averages of algorithms over 5 trials:\n",
      "{'1': [0.8103054881429115, 0.784287133641621, 0.8024050452130066, 0.8026583075053352, 0.8155095263452585], '2': [0.768930451968841, 0.7580980510560553, 0.7569913040162296, 0.7662832609514352, 0.7731779562405686], '3': [0.8108460373522145, 0.7897186963264669, 0.7995211610388241, 0.8120673513656537, 0.8143371160478606], '4': [0.8020464180679553, 0.7870126883896598, 0.7915167217206504, 0.8016420003454564, 0.8116257306890109], '5': [0.8158317077355797, 0.7900813572056341, 0.8066910825288552, 0.817776543699833, 0.8188073529165999]}\n",
      "\n",
      "set of acc values of algorithms over 5 trials\n",
      "{'1': [0.792, 0.771, 0.7865, 0.7855, 0.7955], '2': [0.7555, 0.7505, 0.7485, 0.7575, 0.7585], '3': [0.788, 0.771, 0.779, 0.7945, 0.7915], '4': [0.782, 0.7735, 0.775, 0.7825, 0.791], '5': [0.795, 0.7695, 0.7875, 0.802, 0.7965]}\n",
      "\n",
      "set of roc values of algorithms over 5 trials\n",
      "{'1': [0.8553576715046967, 0.8360234431002238, 0.8509038148843027, 0.8539913823109329, 0.8675452545254526], '2': [0.8160450104815093, 0.8134052965227804, 0.8217005628517823, 0.8281623198146837, 0.8328402840284028], '3': [0.8613070282120626, 0.8457236565469683, 0.8582243902439024, 0.8625510600077615, 0.8665706570657066], '4': [0.8531308508425213, 0.8401872564684776, 0.8512280175109443, 0.8634232309532668, 0.871051105110511], '5': [0.8644909867020851, 0.8489939369911618, 0.8609450906816759, 0.8677230737224496, 0.874019901990199]}\n",
      "\n",
      "set of f1 values of algorithms over 5 trials\n",
      "{'1': [0.7835587929240375, 0.7458379578246392, 0.769811320754717, 0.7684835402050728, 0.783483324510323], '2': [0.7352463454250134, 0.7103888566453859, 0.7007733491969066, 0.7131874630396215, 0.7281935846933033], '3': [0.7832310838445808, 0.7524324324324324, 0.7613390928725702, 0.7791509940891993, 0.7849406910778751], '4': [0.7710084033613446, 0.7473508087005019, 0.7483221476510067, 0.7590027700831025, 0.7728260869565217], '5': [0.7880041365046535, 0.7517501346257404, 0.7716281569048898, 0.7836065573770491, 0.7859021567596003]}\n"
     ]
    }
   ],
   "source": [
    "print('Best Models On Average For Test Set:')\n",
    "for element in best_clf_list:\n",
    "    print(element)\n",
    "\n",
    "print()\n",
    "\n",
    "print('Train Set Data')\n",
    "for i in range(len(training_perf)):\n",
    "    print(training_perf[str(i + 1)])\n",
    "\n",
    "print()\n",
    "\n",
    "alg_avg = {}\n",
    "alg_acc = {}\n",
    "alg_roc = {}\n",
    "alg_f1 = {}\n",
    "\n",
    "for i in range(len(trial_storage)):\n",
    "    alg_avg[str(i + 1)]=[]\n",
    "    alg_acc[str(i + 1)]=[]\n",
    "    alg_roc[str(i + 1)]=[]\n",
    "    alg_f1[str(i + 1)]=[]\n",
    "    for entry in trial_storage[str(i + 1)]:\n",
    "        alg_avg[str(i + 1)].append(entry['average'])\n",
    "        alg_acc[str(i + 1)].append(entry['accuracy'])\n",
    "        alg_roc[str(i + 1)].append(entry['roc_auc_score'])\n",
    "        alg_f1[str(i + 1)].append(entry['f1_score'])\n",
    "\n",
    "print('set of averages of algorithms over 5 trials:')\n",
    "print(alg_avg)\n",
    "print()\n",
    "print('set of acc values of algorithms over 5 trials')\n",
    "print(alg_acc)\n",
    "print()\n",
    "print('set of roc values of algorithms over 5 trials')\n",
    "print(alg_roc)\n",
    "print()\n",
    "print('set of f1 values of algorithms over 5 trials')\n",
    "print(alg_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7861, '2': 0.7540999999999999, '3': 0.7848, '4': 0.7807999999999999, '5': 0.7901}\n"
     ]
    }
   ],
   "source": [
    "# calculate average acc metric scores per algorithm over 5 trials\n",
    "alg_acc_averages = {}\n",
    "for i in range(len(alg_acc)):\n",
    "    alg_acc_averages[str(i + 1)] = sum(alg_acc[str(i + 1)])/5\n",
    "\n",
    "print(alg_acc_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.8527643132651217, '2': 0.8224306947398317, '3': 0.8588753584152802, '4': 0.8558040921771444, '5': 0.8632345980175143}\n"
     ]
    }
   ],
   "source": [
    "# calculate average roc metric scores per algorithm over 5 trials\n",
    "alg_roc_averages = {}\n",
    "for i in range(len(alg_roc)):\n",
    "    alg_roc_averages[str(i + 1)] = sum(alg_roc[str(i + 1)])/5\n",
    "\n",
    "print(alg_roc_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7702349872437579, '2': 0.717557919800046, '3': 0.7722188588633315, '4': 0.7597020433504955, '5': 0.7761782284343866}\n"
     ]
    }
   ],
   "source": [
    "# calculate average f1 metric scores per algorithm over 5 trials\n",
    "alg_f1_averages = {}\n",
    "for i in range(len(alg_f1)):\n",
    "    alg_f1_averages[str(i + 1)] = sum(alg_f1[str(i + 1)])/5\n",
    "\n",
    "print(alg_f1_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.8030331001696265, '2': 0.7646962048466258, '3': 0.8052980724262039, '4': 0.7987687118425466, '5': 0.8098376088173002}\n"
     ]
    }
   ],
   "source": [
    "averages = {}\n",
    "for i in range(len(alg_acc_averages)):\n",
    "    averages[str(i + 1)] = (alg_acc_averages[str(i + 1)] + alg_roc_averages[str(i + 1)] + alg_f1_averages[str(i + 1)])/3\n",
    "\n",
    "print(averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ttest_relResult(statistic=4.5653650438780184, pvalue=0.0004405925267108117)\n",
      "Ttest_relResult(statistic=13.035113141942922, pvalue=3.2114263925488175e-09)\n",
      "Ttest_relResult(statistic=5.266886492081862, pvalue=0.00011912737913248329)\n",
      "Ttest_relResult(statistic=5.438233464673745, pvalue=8.738872162370697e-05)\n"
     ]
    }
   ],
   "source": [
    "# t-test best against rest mean of metrics (ANN against rest)\n",
    "combined_metrics_1 = []\n",
    "combined_metrics_2 = []\n",
    "combined_metrics_3 = []\n",
    "combined_metrics_4 = []\n",
    "combined_metrics_5 = []\n",
    "\n",
    "for item in alg_acc['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "for item in alg_roc['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "for item in alg_f1['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "    \n",
    "for item in alg_acc['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "for item in alg_roc['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "for item in alg_f1['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "\n",
    "for item in alg_acc['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "for item in alg_roc['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "for item in alg_f1['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "    \n",
    "for item in alg_acc['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "for item in alg_roc['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "for item in alg_f1['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "\n",
    "for item in alg_acc['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "for item in alg_roc['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "for item in alg_f1['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "    \n",
    "print(ttest_rel(combined_metrics_5, combined_metrics_1))\n",
    "print(ttest_rel(combined_metrics_5, combined_metrics_2))\n",
    "print(ttest_rel(combined_metrics_5, combined_metrics_3))\n",
    "print(ttest_rel(combined_metrics_5, combined_metrics_4))"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "1deebc6d9ba74352abfb02ead14c1b16": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_3ea7d3db89954e99bb215a875514ee99",
       "style": "IPY_MODEL_f55bb251989547589929fdebd6c45fc7",
       "value": " 1/1 [00:11&lt;00:00, 11.93s/it]"
      }
     },
     "24d5d736274848cb8fa7c6e2ed0d1fa5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "25a4c03767a54fe4b3b626627e0bb0a9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_99a93c0319ed41dbb2791fee5178e525",
        "IPY_MODEL_c5a96159d5494c5fbf9c4d825fa48d39",
        "IPY_MODEL_29630c250d174aa8a12b71355ab846a4"
       ],
       "layout": "IPY_MODEL_52e23afe81e2429887e81148e5f9d48d"
      }
     },
     "29630c250d174aa8a12b71355ab846a4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_d4c19facb84243f5a1d882c6f9c3820d",
       "style": "IPY_MODEL_fdd5c8d34ecf47bcbddb09e0dc26ab2f",
       "value": " 1/1 [00:07&lt;00:00,  7.17s/it]"
      }
     },
     "31af6c3a8fce477992df34b25847797d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_f03daa07fcd248c48e355de88152f217",
       "style": "IPY_MODEL_dede0baf0bac4b6dbd81cfceb488b3c8",
       "value": "Summarize dataset: 100%"
      }
     },
     "3ea7d3db89954e99bb215a875514ee99": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "40e6faada7e84bb991798451660aee2d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_5439f52c9767434a8e00e1026dc04d77",
       "style": "IPY_MODEL_9831c3f4a515454ab5f91cae164a026d",
       "value": "Generate report structure: 100%"
      }
     },
     "465eee19a7794f97ad44f4de01abef0f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_da1f7eba5bc84939a542e4957ef4ab6b",
       "max": 33,
       "style": "IPY_MODEL_24d5d736274848cb8fa7c6e2ed0d1fa5",
       "value": 33
      }
     },
     "52e23afe81e2429887e81148e5f9d48d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "5439f52c9767434a8e00e1026dc04d77": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "688737c7fc674024bab57024cc83d41c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "71b218bd754b4ffcb535c35a93459d91": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7750c059f56b450f9e33f0b35bcce2f7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "81f4c9d86fdb4f97a269105fd5f014b7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_71b218bd754b4ffcb535c35a93459d91",
       "style": "IPY_MODEL_a130ba019ad24126b9a944d6fdfd9964",
       "value": " 33/33 [11:13&lt;00:00, 20.42s/it, Completed]"
      }
     },
     "909a35500d8e4252979b52e23d41cec5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "90e2ab5f945a40a5919eb36b7e359e99": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "94cb7f83c6c54a25867540f63592b005": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_31af6c3a8fce477992df34b25847797d",
        "IPY_MODEL_465eee19a7794f97ad44f4de01abef0f",
        "IPY_MODEL_81f4c9d86fdb4f97a269105fd5f014b7"
       ],
       "layout": "IPY_MODEL_909a35500d8e4252979b52e23d41cec5"
      }
     },
     "9831c3f4a515454ab5f91cae164a026d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "99a93c0319ed41dbb2791fee5178e525": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_90e2ab5f945a40a5919eb36b7e359e99",
       "style": "IPY_MODEL_688737c7fc674024bab57024cc83d41c",
       "value": "Render HTML: 100%"
      }
     },
     "9ea3a492fda341deb6e1f83449a79400": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_e4a69972312a482d892b2344c4d2a3ed",
       "max": 1,
       "style": "IPY_MODEL_b0007059c4014f5ba73059ef597bbb83",
       "value": 1
      }
     },
     "a130ba019ad24126b9a944d6fdfd9964": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b0007059c4014f5ba73059ef597bbb83": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b2dd916b85cd4eadae245916b3a67dd9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_40e6faada7e84bb991798451660aee2d",
        "IPY_MODEL_9ea3a492fda341deb6e1f83449a79400",
        "IPY_MODEL_1deebc6d9ba74352abfb02ead14c1b16"
       ],
       "layout": "IPY_MODEL_7750c059f56b450f9e33f0b35bcce2f7"
      }
     },
     "c5a96159d5494c5fbf9c4d825fa48d39": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_db0988599718437c95510ebf5d223cc7",
       "max": 1,
       "style": "IPY_MODEL_c9547f7c9aed40e0a0ef0318fdbe941c",
       "value": 1
      }
     },
     "c9547f7c9aed40e0a0ef0318fdbe941c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "d4c19facb84243f5a1d882c6f9c3820d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "da1f7eba5bc84939a542e4957ef4ab6b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "db0988599718437c95510ebf5d223cc7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "dede0baf0bac4b6dbd81cfceb488b3c8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e4a69972312a482d892b2344c4d2a3ed": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f03daa07fcd248c48e355de88152f217": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f55bb251989547589929fdebd6c45fc7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "fdd5c8d34ecf47bcbddb09e0dc26ab2f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}