lazypredict_winedataset-Copy2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "#from collections import Counter\n",
    "%matplotlib inline\n",
    "\n",
    "import pyforest\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['import sklearn',\n",
       " 'import pickle',\n",
       " 'from sklearn.ensemble import RandomForestClassifier',\n",
       " 'from sklearn.ensemble import RandomForestRegressor',\n",
       " 'import datetime as dt',\n",
       " 'import matplotlib as mpl',\n",
       " 'from sklearn.model_selection import train_test_split',\n",
       " 'import numpy as np',\n",
       " 'import plotly.graph_objs as go',\n",
       " 'from sklearn.preprocessing import OneHotEncoder',\n",
       " 'import statistics',\n",
       " 'import plotly.express as px',\n",
       " 'import glob',\n",
       " 'import gensim',\n",
       " 'import os',\n",
       " 'import pydot',\n",
       " 'import xgboost as xgb',\n",
       " 'from pathlib import Path',\n",
       " 'from sklearn import svm',\n",
       " 'from sklearn.ensemble import GradientBoostingClassifier',\n",
       " 'import nltk',\n",
       " 'from pyspark import SparkContext',\n",
       " 'import sys',\n",
       " 'import dash',\n",
       " 'import tensorflow as tf',\n",
       " 'import matplotlib.pyplot as plt',\n",
       " 'import seaborn as sns',\n",
       " 'import bokeh',\n",
       " 'from sklearn.ensemble import GradientBoostingRegressor',\n",
       " 'import awswrangler as wr',\n",
       " 'from dask import dataframe as dd',\n",
       " 'import lightgbm as lgb',\n",
       " 'import pandas as pd',\n",
       " 'import plotly as py',\n",
       " 'from sklearn.feature_extraction.text import TfidfVectorizer',\n",
       " 'from openpyxl import load_workbook',\n",
       " 'import re',\n",
       " 'import altair as alt',\n",
       " 'import tqdm',\n",
       " 'import spacy',\n",
       " 'from sklearn.manifold import TSNE',\n",
       " 'import keras']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lazy_imports()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.read_excel('Uneven Resampling.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>p</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.4</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.145</td>\n",
       "      <td>34.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.99832</td>\n",
       "      <td>3.38</td>\n",
       "      <td>0.86</td>\n",
       "      <td>9.9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.4</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.145</td>\n",
       "      <td>34.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.99832</td>\n",
       "      <td>3.38</td>\n",
       "      <td>0.86</td>\n",
       "      <td>9.9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.6</td>\n",
       "      <td>0.580</td>\n",
       "      <td>0.66</td>\n",
       "      <td>2.2</td>\n",
       "      <td>0.074</td>\n",
       "      <td>10.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>1.00080</td>\n",
       "      <td>3.25</td>\n",
       "      <td>0.57</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.3</td>\n",
       "      <td>1.020</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.4</td>\n",
       "      <td>0.084</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.99892</td>\n",
       "      <td>3.48</td>\n",
       "      <td>0.49</td>\n",
       "      <td>11.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.1</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.05</td>\n",
       "      <td>5.7</td>\n",
       "      <td>0.082</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.99808</td>\n",
       "      <td>3.40</td>\n",
       "      <td>0.52</td>\n",
       "      <td>10.2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0           10.4             0.440         0.42             1.5      0.145   \n",
       "1           10.4             0.440         0.42             1.5      0.145   \n",
       "2           11.6             0.580         0.66             2.2      0.074   \n",
       "3            8.3             1.020         0.02             3.4      0.084   \n",
       "4            7.1             0.875         0.05             5.7      0.082   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density     p  sulphates  \\\n",
       "0                 34.0                  48.0  0.99832  3.38       0.86   \n",
       "1                 34.0                  48.0  0.99832  3.38       0.86   \n",
       "2                 10.0                  47.0  1.00080  3.25       0.57   \n",
       "3                  6.0                  11.0  0.99892  3.48       0.49   \n",
       "4                  3.0                  14.0  0.99808  3.40       0.52   \n",
       "\n",
       "   alcohol  quality  \n",
       "0      9.9        3  \n",
       "1      9.9        3  \n",
       "2      9.0        3  \n",
       "3     11.0        3  \n",
       "4     10.2        3  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del X['Unnamed: 0'] \n",
    "X.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y= pd.read_csv('ValidationDataset.csv', sep=';', skiprows=1, \n",
    "                names=['fixed acidity', 'volatile acidity','citric acid', 'residual sugar', 'chlorides','free sulfur dioxide' , 'total sulfur dioxide','density','pH', 'sulphates','alcohol','quality'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11</td>\n",
       "      <td>34</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.098</td>\n",
       "      <td>25</td>\n",
       "      <td>67</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.68</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0.092</td>\n",
       "      <td>15</td>\n",
       "      <td>54</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.65</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.2</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.075</td>\n",
       "      <td>17</td>\n",
       "      <td>60</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.58</td>\n",
       "      <td>9.8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11</td>\n",
       "      <td>34</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            7.4              0.70         0.00             1.9      0.076   \n",
       "1            7.8              0.88         0.00             2.6      0.098   \n",
       "2            7.8              0.76         0.04             2.3      0.092   \n",
       "3           11.2              0.28         0.56             1.9      0.075   \n",
       "4            7.4              0.70         0.00             1.9      0.076   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                   11                    34   0.9978  3.51       0.56   \n",
       "1                   25                    67   0.9968  3.20       0.68   \n",
       "2                   15                    54   0.9970  3.26       0.65   \n",
       "3                   17                    60   0.9980  3.16       0.58   \n",
       "4                   11                    34   0.9978  3.51       0.56   \n",
       "\n",
       "   alcohol  quality  \n",
       "0      9.4        5  \n",
       "1      9.8        5  \n",
       "2      9.8        5  \n",
       "3      9.8        6  \n",
       "4      9.4        5  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X.iloc[:,0:11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>p</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.4</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.145</td>\n",
       "      <td>34.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.99832</td>\n",
       "      <td>3.38</td>\n",
       "      <td>0.86</td>\n",
       "      <td>9.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.4</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.145</td>\n",
       "      <td>34.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.99832</td>\n",
       "      <td>3.38</td>\n",
       "      <td>0.86</td>\n",
       "      <td>9.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.6</td>\n",
       "      <td>0.580</td>\n",
       "      <td>0.66</td>\n",
       "      <td>2.2</td>\n",
       "      <td>0.074</td>\n",
       "      <td>10.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>1.00080</td>\n",
       "      <td>3.25</td>\n",
       "      <td>0.57</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.3</td>\n",
       "      <td>1.020</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.4</td>\n",
       "      <td>0.084</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.99892</td>\n",
       "      <td>3.48</td>\n",
       "      <td>0.49</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.1</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.05</td>\n",
       "      <td>5.7</td>\n",
       "      <td>0.082</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.99808</td>\n",
       "      <td>3.40</td>\n",
       "      <td>0.52</td>\n",
       "      <td>10.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0           10.4             0.440         0.42             1.5      0.145   \n",
       "1           10.4             0.440         0.42             1.5      0.145   \n",
       "2           11.6             0.580         0.66             2.2      0.074   \n",
       "3            8.3             1.020         0.02             3.4      0.084   \n",
       "4            7.1             0.875         0.05             5.7      0.082   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density     p  sulphates  \\\n",
       "0                 34.0                  48.0  0.99832  3.38       0.86   \n",
       "1                 34.0                  48.0  0.99832  3.38       0.86   \n",
       "2                 10.0                  47.0  1.00080  3.25       0.57   \n",
       "3                  6.0                  11.0  0.99892  3.48       0.49   \n",
       "4                  3.0                  14.0  0.99808  3.40       0.52   \n",
       "\n",
       "   alcohol  \n",
       "0      9.9  \n",
       "1      9.9  \n",
       "2      9.0  \n",
       "3     11.0  \n",
       "4     10.2  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = X.iloc[:,11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3\n",
       "1    3\n",
       "2    3\n",
       "3    3\n",
       "4    3\n",
       "Name: quality, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = pd.DataFrame(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = Y.iloc[:,0:11]\n",
    "y_test = Y.iloc[:,11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lazypredict\n",
    "from lazypredict.Supervised import LazyClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 29/29 [00:03<00:00,  8.82it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>ROC AUC</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Time Taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LogisticRegression</th>\n",
       "      <td>0.52</td>\n",
       "      <td>0.52</td>\n",
       "      <td>None</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SGDClassifier</th>\n",
       "      <td>0.51</td>\n",
       "      <td>0.42</td>\n",
       "      <td>None</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassiveAggressiveClassifier</th>\n",
       "      <td>0.31</td>\n",
       "      <td>0.39</td>\n",
       "      <td>None</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BaggingClassifier</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.36</td>\n",
       "      <td>None</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CalibratedClassifierCV</th>\n",
       "      <td>0.54</td>\n",
       "      <td>0.35</td>\n",
       "      <td>None</td>\n",
       "      <td>0.53</td>\n",
       "      <td>1.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NearestCentroid</th>\n",
       "      <td>0.31</td>\n",
       "      <td>0.33</td>\n",
       "      <td>None</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExtraTreeClassifier</th>\n",
       "      <td>0.49</td>\n",
       "      <td>0.33</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.53</td>\n",
       "      <td>0.32</td>\n",
       "      <td>None</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExtraTreesClassifier</th>\n",
       "      <td>0.54</td>\n",
       "      <td>0.32</td>\n",
       "      <td>None</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianNB</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.32</td>\n",
       "      <td>None</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Perceptron</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.31</td>\n",
       "      <td>None</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BernoulliNB</th>\n",
       "      <td>0.51</td>\n",
       "      <td>0.28</td>\n",
       "      <td>None</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LabelSpreading</th>\n",
       "      <td>0.49</td>\n",
       "      <td>0.27</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LabelPropagation</th>\n",
       "      <td>0.49</td>\n",
       "      <td>0.27</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearDiscriminantAnalysis</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.27</td>\n",
       "      <td>None</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBClassifier</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.27</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.54</td>\n",
       "      <td>0.26</td>\n",
       "      <td>None</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>QuadraticDiscriminantAnalysis</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.25</td>\n",
       "      <td>None</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LGBMClassifier</th>\n",
       "      <td>0.52</td>\n",
       "      <td>0.23</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighborsClassifier</th>\n",
       "      <td>0.48</td>\n",
       "      <td>0.23</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RidgeClassifier</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.22</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RidgeClassifierCV</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.22</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVC</th>\n",
       "      <td>0.54</td>\n",
       "      <td>0.22</td>\n",
       "      <td>None</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.20</td>\n",
       "      <td>None</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AdaBoostClassifier</th>\n",
       "      <td>0.49</td>\n",
       "      <td>0.20</td>\n",
       "      <td>None</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DummyClassifier</th>\n",
       "      <td>0.26</td>\n",
       "      <td>0.11</td>\n",
       "      <td>None</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Accuracy  Balanced Accuracy ROC AUC  F1 Score  \\\n",
       "Model                                                                          \n",
       "LogisticRegression                 0.52               0.52    None      0.52   \n",
       "SGDClassifier                      0.51               0.42    None      0.51   \n",
       "PassiveAggressiveClassifier        0.31               0.39    None      0.34   \n",
       "BaggingClassifier                  0.55               0.36    None      0.55   \n",
       "CalibratedClassifierCV             0.54               0.35    None      0.53   \n",
       "NearestCentroid                    0.31               0.33    None      0.34   \n",
       "ExtraTreeClassifier                0.49               0.33    None      0.50   \n",
       "RandomForestClassifier             0.53               0.32    None      0.52   \n",
       "ExtraTreesClassifier               0.54               0.32    None      0.51   \n",
       "GaussianNB                         0.44               0.32    None      0.49   \n",
       "Perceptron                         0.44               0.31    None      0.46   \n",
       "BernoulliNB                        0.51               0.28    None      0.51   \n",
       "LabelSpreading                     0.49               0.27    None      0.50   \n",
       "LabelPropagation                   0.49               0.27    None      0.50   \n",
       "LinearDiscriminantAnalysis         0.50               0.27    None      0.51   \n",
       "XGBClassifier                      0.50               0.27    None      0.50   \n",
       "LinearSVC                          0.54               0.26    None      0.51   \n",
       "QuadraticDiscriminantAnalysis      0.44               0.25    None      0.46   \n",
       "LGBMClassifier                     0.52               0.23    None      0.50   \n",
       "KNeighborsClassifier               0.48               0.23    None      0.50   \n",
       "RidgeClassifier                    0.55               0.22    None      0.50   \n",
       "RidgeClassifierCV                  0.55               0.22    None      0.50   \n",
       "SVC                                0.54               0.22    None      0.50   \n",
       "DecisionTreeClassifier             0.44               0.20    None      0.45   \n",
       "AdaBoostClassifier                 0.49               0.20    None      0.43   \n",
       "DummyClassifier                    0.26               0.11    None      0.28   \n",
       "\n",
       "                               Time Taken  \n",
       "Model                                      \n",
       "LogisticRegression                   0.06  \n",
       "SGDClassifier                        0.03  \n",
       "PassiveAggressiveClassifier          0.02  \n",
       "BaggingClassifier                    0.07  \n",
       "CalibratedClassifierCV               1.32  \n",
       "NearestCentroid                      0.01  \n",
       "ExtraTreeClassifier                  0.01  \n",
       "RandomForestClassifier               0.27  \n",
       "ExtraTreesClassifier                 0.18  \n",
       "GaussianNB                           0.01  \n",
       "Perceptron                           0.01  \n",
       "BernoulliNB                          0.01  \n",
       "LabelSpreading                       0.08  \n",
       "LabelPropagation                     0.06  \n",
       "LinearDiscriminantAnalysis           0.01  \n",
       "XGBClassifier                        0.23  \n",
       "LinearSVC                            0.30  \n",
       "QuadraticDiscriminantAnalysis        0.01  \n",
       "LGBMClassifier                       0.30  \n",
       "KNeighborsClassifier                 0.02  \n",
       "RidgeClassifier                      0.01  \n",
       "RidgeClassifierCV                    0.02  \n",
       "SVC                                  0.07  \n",
       "DecisionTreeClassifier               0.02  \n",
       "AdaBoostClassifier                   0.12  \n",
       "DummyClassifier                      0.01  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LazyClassifier(verbose=0,ignore_warnings=True)\n",
    "models, predictions = clf.fit(X_train, X_test, y_train, y_test)\n",
    "models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train, y_train)\n",
    "y_pred = rf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Evaluation Metrics – Random Forest:')\n",
    "print('Accuracy: ' + str(metrics.accuracy_score(y_test, y_pred)))\n",
    "print('F1 Score: ' + str(metrics.f1_score(y_test, y_pred, average='macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(max_features='auto', oob_score=True, random_state=1, n_jobs=-1)\n",
    "\n",
    "param_grid = { \"criterion\" : [\"gini\", \"entropy\"], \"min_samples_leaf\" : [1, 5, 10], \"min_samples_split\" : [2, 4, 10, 12, 16], \"n_estimators\": [50, 100, 400, 700, 1000]}\n",
    "\n",
    "gs = GridSearchCV(estimator=rf, param_grid=param_grid, scoring='accuracy', cv=3, n_jobs=-1)\n",
    "\n",
    "gs = gs.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(gs.best_score_)\n",
    "print(gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(criterion='gini', \n",
    "                             n_estimators=100,\n",
    "                             min_samples_split=2,\n",
    "                             min_samples_leaf=1,\n",
    "                             max_features='auto',\n",
    "                             oob_score=True,\n",
    "                             random_state=1,\n",
    "                             n_jobs=None)\n",
    "rf.fit(X_train, y_train)\n",
    "print(\"%.4f\" % rf.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "\n",
    "y_pred = rf.predict(X_test)\n",
    "\n",
    "print(confusion_matrix(y_test,y_pred))\n",
    "print(classification_report(y_test,y_pred))\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression()\n",
    "model_res = clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Evaluation Metrics – Random Forest:')\n",
    "print('Accuracy: ' + str(metrics.accuracy_score(y_test, y_pred)))\n",
    "print('F1 Score: ' + str(metrics.f1_score(y_test, y_pred, average='macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create first pipeline for base without reducing features.\n",
    "\n",
    "#pipe = Pipeline([('classifier' , RandomForestClassifier())])\n",
    "# pipe = Pipeline([('classifier', RandomForestClassifier())])\n",
    "\n",
    "# Create param grid.\n",
    "clf = LogisticRegression()\n",
    "\n",
    "param_grid = {\n",
    "     'penalty' : ['l1', 'l2'],\n",
    "    'C' : np.logspace(-4, 4, 20),\n",
    "    'solver' : ['liblinear', 'newton-cg']\n",
    "}\n",
    "\n",
    "# Create grid search object\n",
    "\n",
    "clf = GridSearchCV(estimator=clf, param_grid = param_grid, cv = 5, verbose=True, n_jobs=-1)\n",
    "\n",
    "# Fit on data\n",
    "\n",
    "best_clf = clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(clf.best_score_)\n",
    "print(clf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier( bootstrap=True,\n",
    "                             ccp_alpha= 0.0,\n",
    "                             class_weight= None,\n",
    "                             criterion= 'gini',\n",
    "                             max_depth= None,\n",
    "                             max_features= 'auto',\n",
    "                             max_leaf_nodes= None,\n",
    "                             max_samples= None,\n",
    "                             min_impurity_decrease= 0.0,\n",
    "                             min_impurity_split= None,\n",
    "                             min_samples_leaf= 1,\n",
    "                             min_samples_split= 2,\n",
    "                             min_weight_fraction_leaf= 0.0,\n",
    "                             n_estimators= 100,\n",
    "                             n_jobs= None,\n",
    "                             oob_score=True,\n",
    "                             random_state= None,\n",
    "                             verbose= 0,\n",
    "                             warm_start = False)\n",
    "rf.fit(X_train, y_train)\n",
    "print(\"%.4f\" % rf.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "\n",
    "y_pred = rf.predict(X_test)\n",
    "\n",
    "print(confusion_matrix(y_test,y_pred))\n",
    "print(classification_report(y_test,y_pred))\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import f1_score, make_scorer\n",
    "from sklearn.model_selection import cross_val_score\n",
    "#from sklearn.preprocessing import Imputer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# Custom scorer for cross validation\n",
    "scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_score = cross_val_score(rf, X_train, y_train, cv = 4, scoring = scorer)\n",
    "\n",
    "print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean(), 4)} with std = {round(cv_score.std(), 4)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_score = cross_val_score(rf, X_train, y_train, cv = 4, scoring = scorer)\n",
    "\n",
    "print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean(), 4)} with std = {round(cv_score.std(), 4)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}