02-Improving-linear-model-baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "data_paths = {}\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        data_paths[filename] = os.path.join(dirname, filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Best Model so far is the Logistic Regression (Baseline) with public LB score of 0.97519. The data is used for this model is not preprocessed/stemmed. I directly used tfidf vectorizer with `lower = True` and `stopwords = english`.\n",
    "\n",
    "- I expected this model (Logistic - Baseline) performance to improve by preprocessing the comment text data by removing punctuations, stopwords, urls, htmls & converting emoticons/emojis...but results didn't improve & models under performed the above baseline by around 1-2%. This is probably because we are loosing some information by excessive preprocessing. The results for this models are present in this [Notebook](https://www.kaggle.com/asrsaiteja/toxic-comments-preprocessing-baselines)\n",
    "\n",
    "- Now In this notebook,I am trying the Logistic model to > 0.98. I am not going to use any fancy preprocessing as they are helping us, instead I am sticking to tfidf preprocessing parameters `lower = True` and `stopwords = english` for now.\n",
    "\n",
    "- Further I am planning to try kfold cross validation to see if it can improve the LB score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape: (159571, 8)\n",
      "Columns in Train: Index(['id', 'comment_text', 'toxic', 'severe_toxic', 'obscene', 'threat',\n",
      "       'insult', 'identity_hate'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(data_paths['train.csv'])\n",
    "test_df = pd.read_csv(data_paths['test.csv'])\n",
    "sub_df = pd.read_csv(data_paths['sample_submission.csv'])\n",
    "print('Train data shape:', train_df.shape)\n",
    "print('Columns in Train:', train_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_col = ['id', 'is_clean']  # columns not neccessary - can be dropped\n",
    "text_col = ['comment_text']  # text feature\n",
    "label_col = [col for col in train_df.columns if col not in text_col + drop_col] # target variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data points in train data after splitting: 127656\n",
      "Data points in valiadtion data: 31915\n",
      "Data points in test data: 153164\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# For SUBMISSION\n",
    "X_train_val = train_df['comment_text']\n",
    "y_train_val = train_df[label_col]\n",
    "\n",
    "X_test = test_df['comment_text']\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, \n",
    "                                                  test_size=0.2, random_state=2019)\n",
    "print('Data points in train data after splitting:', len(X_train))\n",
    "print('Data points in valiadtion data:', len(X_val))\n",
    "print('Data points in test data:', len(X_test))\n",
    "\n",
    "del train_df, test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wrapper for mean roc auc metric\n",
    "from sklearn.metrics import roc_auc_score\n",
    "def mean_roc_auc(y_true, y_pred):\n",
    "    num_class = 6\n",
    "    return np.mean([roc_auc_score(y_true[:,i], y_pred[:,i]) for i in range(num_class)]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### feature transformation (tfidf - text feature extraction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(127656, 70000) (31915, 70000)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from nltk.corpus import stopwords\n",
    "from scipy.sparse import hstack\n",
    "from nltk import pos_tag\n",
    "\n",
    "# word features\n",
    "tfidf_words = TfidfVectorizer(ngram_range = (1,1), strip_accents='unicode', analyzer='word',\n",
    "                              smooth_idf=1, sublinear_tf=1,stop_words = 'english', max_features = 20000)\n",
    "X_train_words = tfidf_words.fit_transform(X_train)\n",
    "X_val_words = tfidf_words.transform(X_val)\n",
    "\n",
    "# char features\n",
    "tfidf_char = TfidfVectorizer(ngram_range = (2,6), strip_accents='unicode', analyzer='char', \n",
    "                              smooth_idf=1, sublinear_tf=1,stop_words = 'english', max_features = 50000)\n",
    "X_train_char = tfidf_char.fit_transform(X_train)\n",
    "X_val_char = tfidf_char.transform(X_val)\n",
    "\n",
    "# stack in\n",
    "train_features = hstack([X_train_words, X_train_char]).tocsr()\n",
    "val_features = hstack([X_val_words, X_val_char]).tocsr()\n",
    "\n",
    "print(train_features.shape, val_features.shape)\n",
    "del X_train_words, X_val_words, X_train_char, X_val_char"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train + Val features & Test features (for making submissions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159571, 70000) (153164, 70000)\n"
     ]
    }
   ],
   "source": [
    "# transform into features\n",
    "X_train_val_words = tfidf_words.fit_transform(X_train_val)\n",
    "X_test_words = tfidf_words.transform(X_test)\n",
    "\n",
    "X_train_val_char = tfidf_char.fit_transform(X_train_val)\n",
    "X_test_char = tfidf_char.transform(X_test)\n",
    "\n",
    "train_val_features = hstack([X_train_val_words, X_train_val_char]).tocsr()\n",
    "test_features = hstack([X_test_words, X_test_char]).tocsr()\n",
    "\n",
    "print(train_val_features.shape, test_features.shape)\n",
    "del X_train_val_words, X_test_words, X_train_val_char, X_test_char"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logisitc Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model: Logistic Regression\n",
      "mean ROC-AUC on train set: 0.9955191025106922\n",
      "mean ROC-AUC on validation set: 0.984386798576792\n"
     ]
    }
   ],
   "source": [
    "model = OneVsRestClassifier(LogisticRegression(solver='sag'))  # default C = 1.0\n",
    "model.fit(train_features, y_train)\n",
    "print('model: Logistic Regression')\n",
    "print('mean ROC-AUC on train set:', roc_auc_score(y_train, model.predict_proba(train_features)))\n",
    "y_pred_log = model.predict_proba(val_features)\n",
    "print('mean ROC-AUC on validation set:', roc_auc_score(y_val, y_pred_log))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The above logisitc model with tfidf words + tfidf char features scored 0.98003 on the private Leaderboard and 0.97897 on public leaderboard\n",
    "- To my models built on minimal preprocessed data (only lowering & stopwords removal) has performed better than the models built on preprocessed comment data.\n",
    "- I observed wide variations of ROC-AUC scores with change in tfidf parameters - especially min_df, max_df and max_features. I tuned these a bit manually which helped to reduce the model complexity using `max_features` parameter. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning Logisitc Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_validate_model(model):\n",
    "    \n",
    "    model.fit(train_features, y_train)\n",
    "    train_mean_roc = roc_auc_score(y_train, model.predict_proba(train_features))\n",
    "    print('mean ROC-AUC on train set:', train_mean_roc)\n",
    "    y_pred_log = model.predict_proba(val_features)\n",
    "    val_mean_roc = roc_auc_score(y_val, y_pred_log)\n",
    "    print('mean ROC-AUC on validation set:', val_mean_roc)\n",
    "    \n",
    "    return train_mean_roc, val_mean_roc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression - Fine Tuning\n",
      "\n",
      "C:= 0.001\n",
      "mean ROC-AUC on train set: 0.963532670494819\n",
      "mean ROC-AUC on validation set: 0.9592854059846184\n",
      "\n",
      "C:= 0.01\n",
      "mean ROC-AUC on train set: 0.9719240033748707\n",
      "mean ROC-AUC on validation set: 0.9668758008463559\n",
      "\n",
      "C:= 0.1\n",
      "mean ROC-AUC on train set: 0.9853337365045296\n",
      "mean ROC-AUC on validation set: 0.9786513010413289\n",
      "\n",
      "C:= 1\n",
      "mean ROC-AUC on train set: 0.9955191082732129\n",
      "mean ROC-AUC on validation set: 0.9843868196166347\n",
      "\n",
      "C:= 2\n",
      "mean ROC-AUC on train set: 0.997412244665699\n",
      "mean ROC-AUC on validation set: 0.9841449112138427\n",
      "\n",
      "C:= 3\n",
      "mean ROC-AUC on train set: 0.99824458668684\n",
      "mean ROC-AUC on validation set: 0.9835529293358425\n",
      "\n",
      "C:= 5\n",
      "mean ROC-AUC on train set: 0.9990087933155457\n",
      "mean ROC-AUC on validation set: 0.9823872595317367\n",
      "\n",
      "C:= 7\n",
      "mean ROC-AUC on train set: 0.9993576262765776\n",
      "mean ROC-AUC on validation set: 0.9813902833154792\n",
      "\n",
      "C:= 10\n",
      "mean ROC-AUC on train set: 0.9996161280259827\n",
      "mean ROC-AUC on validation set: 0.9801972388424326\n",
      "\n",
      "C:= 25\n",
      "mean ROC-AUC on train set: 0.9999246540213038\n",
      "mean ROC-AUC on validation set: 0.9767562779411905\n",
      "\n",
      "C:= 60\n",
      "mean ROC-AUC on train set: 0.9999889873992854\n",
      "mean ROC-AUC on validation set: 0.9734174865453481\n",
      "\n",
      "C:= 100\n",
      "mean ROC-AUC on train set: 0.9999954358419628\n",
      "mean ROC-AUC on validation set: 0.9715861634988509\n",
      "\n",
      "C:= 1000\n",
      "mean ROC-AUC on train set: 0.9999992507675595\n",
      "mean ROC-AUC on validation set: 0.9663779107431195\n"
     ]
    }
   ],
   "source": [
    "print('Logistic Regression - Fine Tuning')\n",
    "scores = []\n",
    "for c_value in [0.001, 0.01, 0.1, 1, 2, 3, 5, 7, 10, 25, 60, 100, 1000]:\n",
    "    print('\\nC:=', c_value)\n",
    "    model = OneVsRestClassifier(LogisticRegression(C = c_value, solver='sag'))\n",
    "    _, val_roc = train_validate_model(model)\n",
    "    scores.append((c_value, val_roc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C value from finetuning: 1.0\n"
     ]
    }
   ],
   "source": [
    "scores_df = pd.DataFrame(scores, columns = ['c_value', 'val_roc'])\n",
    "best_c_value = scores_df.loc[scores_df['val_roc'].idxmax(), 'c_value']\n",
    "print('Best C value from finetuning:', best_c_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model: Tuned Logistic Regression\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='logistic_submission.csv' target='_blank'>logistic_submission.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/logistic_submission.csv"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define & fit the best logistic model\n",
    "model = OneVsRestClassifier(LogisticRegression(C = best_c_value,solver='sag'))\n",
    "model.fit(train_val_features, y_train_val)\n",
    "print('model: Tuned Logistic Regression')\n",
    "\n",
    "# make predictions on Test data\n",
    "y_test_pred = model.predict_proba(test_features)\n",
    "## making a submission file\n",
    "sub_df.iloc[:,1:] = y_test_pred\n",
    "sub_df.head()\n",
    "from IPython.display import FileLink\n",
    "sub_df.to_csv('logistic_submission.csv', index = None)\n",
    "FileLink('logistic_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The best C value from finetuning and default C value are same (C = 1.0). So the Tuned Logistic model will also have same test scores of 0.98003 on the private Leaderboard and 0.97897 on public leaderboard.\n",
    "\n",
    "### using a Regressor: Following this method known to improve the LB score slightly (from kaggle discussions)\n",
    "\n",
    "- As the evaluation is based on the class 'probability scores'...Let's try some thing different: Using a Regressor to predict the score (Like Logistic, Lasso or Ridge). Later if needed (if values are out of bounds) we can just apply simple `sigmoid` function over the output to squash the values b/w 0 & 1.\n",
    "\n",
    "- In my experients, Ridge worked better... It is nothing but Logistic Regression with L2 regularization. (where as Lasso is with L1 regularization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean ROC-AUC on validation set: 0.9846564408459272\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "y_val_pred = np.zeros(y_val.shape)\n",
    "for i, class_name in enumerate(label_col):\n",
    "    train_target = y_train[class_name]\n",
    "    classifier = Ridge(alpha=20, copy_X=True, solver='auto',tol=0.0025)\n",
    "    classifier.fit(train_features, train_target)\n",
    "    y_val_pred[:,i] = classifier.predict(val_features)\n",
    "print('mean ROC-AUC on validation set:', roc_auc_score(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model: Ridge Regression\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='ridge_submission.csv' target='_blank'>ridge_submission.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/ridge_submission.csv"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test submission\n",
    "print('model: Ridge Regression')\n",
    "y_test_pred = np.zeros((len(X_test), y_val.shape[1]))\n",
    "for i, class_name in enumerate(label_col):\n",
    "    train_val_target = y_train_val[class_name]\n",
    "    clf = Ridge(alpha=20, copy_X=True, fit_intercept=True, solver='auto',\n",
    "                        max_iter=100,   normalize=False, random_state=0,  tol=0.0025)\n",
    "    clf.fit(train_val_features, train_val_target)\n",
    "    y_test_pred[:,i] = clf.predict(test_features)\n",
    "\n",
    "## making a submission file\n",
    "sub_df.iloc[:,1:] = y_test_pred\n",
    "sub_df.head()\n",
    "from IPython.display import FileLink\n",
    "sub_df.to_csv('ridge_submission.csv', index = None)\n",
    "FileLink('ridge_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The above submission gave me a public LB score of 0.9809 on test data.\n",
    "\n",
    "### NB Logisitc:\n",
    "- There is this interesting paper which combines Naive Bayes & SVM - (NBSVM model)\n",
    "- Baselines and Bigrams: Simple, Good Sentiment and Topic Classiﬁcation.\n",
    "- The paper used the log probability ratio of features multiplied with original features as input to the liblinear version of SVM (=~ sklearn LinearSVM) and obtained better results on text classification task.\n",
    "- Let us implement this paper by replacing SVM with Logistic (As logistic is performing best in this case I am swaping them.)\n",
    "\n",
    "reference:\n",
    "- [Baselines and Bigrams: Simple, Good Sentiment and Topic Classiﬁcation.](https://nlp.stanford.edu/pubs/sidaw12_simple_sentiment.pdf)\n",
    "- [Jeremy Howard's Strong linear baseline notebook](https://www.kaggle.com/jhoward/nb-svm-strong-linear-baseline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, ClassifierMixin\n",
    "from sklearn.utils.validation import check_X_y, check_is_fitted\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from scipy import sparse\n",
    "\n",
    "class NbLogisticClassifier(BaseEstimator, ClassifierMixin):\n",
    "    def __init__(self, C=1.0, dual=False, n_jobs=1):\n",
    "        self.C = C\n",
    "        self.dual = dual\n",
    "        self.n_jobs = n_jobs\n",
    "        \n",
    "    def fit(self, x, y):\n",
    "        # Check that X and y have correct shape\n",
    "        y = y.values\n",
    "        x, y = check_X_y(x, y, accept_sparse=True)\n",
    "\n",
    "        # calculate prior\n",
    "        def pr(x, y_i,y):\n",
    "            p = x[y==y_i].sum(0)\n",
    "            return (p+1) / ((y==y_i).sum()+1)\n",
    "\n",
    "        # calculate ratio & matrix multiply X with ratio\n",
    "        self._r = sparse.csr_matrix(np.log(pr(x,1,y) / pr(x,0,y)))\n",
    "        x_nb = x.multiply(self._r)\n",
    "        \n",
    "        # fit the logisitic with NB features\n",
    "        self._clf = LogisticRegression(C=self.C, dual=self.dual, \n",
    "                                       n_jobs=self.n_jobs).fit(x_nb, y)\n",
    "        return self\n",
    "\n",
    "    def predict(self, x):\n",
    "        # Verify that model has been fit\n",
    "        check_is_fitted(self, ['_r', '_clf'])\n",
    "        return self._clf.predict(x.multiply(self._r))\n",
    "\n",
    "    def predict_proba(self, x):\n",
    "        # Verify that model has been fit\n",
    "        check_is_fitted(self, ['_r', '_clf'])\n",
    "        return self._clf.predict_proba(x.multiply(self._r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean ROC-AUC on validation set: 0.9845241792468226\n"
     ]
    }
   ],
   "source": [
    "nblog_clf = NbLogisticClassifier(C=1.0, dual=True)\n",
    "\n",
    "y_val_pred = np.zeros(y_val.shape)\n",
    "for i, class_name in enumerate(label_col):\n",
    "    train_target = y_train[class_name]\n",
    "    nblog_clf.fit(train_features, train_target)\n",
    "    y_val_pred[:,i] = nblog_clf.predict_proba(val_features)[:,1]\n",
    "print('mean ROC-AUC on validation set:', roc_auc_score(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model: NB-Logisitc\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='nblog_submission.csv' target='_blank'>nblog_submission.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/nblog_submission.csv"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test submission\n",
    "print('model: NB-Logisitc')\n",
    "\n",
    "y_test_pred = np.zeros((len(X_test), y_val.shape[1]))\n",
    "for i, class_name in enumerate(label_col):\n",
    "    # define target\n",
    "    train_val_target = y_train_val[class_name]\n",
    "    # define clf\n",
    "    nblog_clf = NbLogisticClassifier(C=1.0, dual=True)\n",
    "    # fit the clf\n",
    "    nblog_clf.fit(train_val_features, train_val_target)\n",
    "    # make pred on test\n",
    "    y_test_pred[:,i] = nblog_clf.predict_proba(test_features)[:,1]\n",
    "\n",
    "## making a submission file\n",
    "sub_df.iloc[:,1:] = y_test_pred\n",
    "sub_df.head()\n",
    "from IPython.display import FileLink\n",
    "sub_df.to_csv('nblog_submission.csv', index = None)\n",
    "FileLink('nblog_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# # number of folds\n",
    "# num_folds = 2 \n",
    "\n",
    "# predict = np.zeros((test_df.shape[0],6))\n",
    "\n",
    "# # Uncomment for out-of-fold predictions\n",
    "# #scores = []\n",
    "# #oof_predict = np.zeros((train.shape[0],6))\n",
    "\n",
    "# kf = KFold(n_splits = num_folds, shuffle=True, random_state=2019)\n",
    "\n",
    "# for train_index, test_index in kf.split(X_train_val):\n",
    "    \n",
    "#     kfold_y_train, kfold_y_test = y_train[train_index], y_train[test_index]\n",
    "#     kfold_X_train = x_train[train_index]\n",
    "#     kfold_X_features = features[train_index]\n",
    "#     kfold_X_valid = x_train[test_index]\n",
    "#     kfold_X_valid_features = features[test_index]\n",
    "    \n",
    "#     gc.collect()\n",
    "#     K.clear_session()\n",
    "    \n",
    "#     model = get_model(features)\n",
    "    \n",
    "#     ra_val = RocAucEvaluation(validation_data=([kfold_X_valid,kfold_X_valid_features], kfold_y_test), interval = 1)\n",
    "    \n",
    "#     model.fit([kfold_X_train,kfold_X_features], kfold_y_train, batch_size=batch_size, epochs=epochs, verbose=1,\n",
    "#              callbacks = [ra_val])\n",
    "#     gc.collect()\n",
    "    \n",
    "#     #model.load_weights(bst_model_path)\n",
    "#     model.load_weights(\"best_weights.h5\")\n",
    "    \n",
    "#     predict += model.predict([x_test,test_features], batch_size=batch_size,verbose=1) / num_folds\n",
    "    \n",
    "#     #gc.collect()\n",
    "#     # uncomment for out of fold predictions\n",
    "#     #oof_predict[test_index] = model.predict([kfold_X_valid, kfold_X_valid_features],batch_size=batch_size, verbose=1)\n",
    "#     #cv_score = roc_auc_score(kfold_y_test, oof_predict[test_index])\n",
    "    \n",
    "#     #scores.append(cv_score)\n",
    "#     #print('score: ',cv_score)\n",
    "\n",
    "# print(\"Done\")\n",
    "# #print('Total CV score is {}'.format(np.mean(scores)))    \n",
    "\n",
    "\n",
    "# sample_submission = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv\")\n",
    "# class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
    "# sample_submission[class_names] = predict\n",
    "# sample_submission.to_csv('model_9872_baseline_submission.csv',index=False)\n",
    "\n",
    "# # uncomment for out of fold predictions\n",
    "# #oof = pd.DataFrame.from_dict({'id': train['id']})\n",
    "# #for c in class_names:\n",
    "# #    oof[c] = np.zeros(len(train))\n",
    "# #    \n",
    "# #oof[class_names] = oof_predict\n",
    "# #for c in class_names:\n",
    "# #    oof['prediction_' +c] = oof[c]\n",
    "# #oof.to_csv('oof-model_9872_baseline_submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}