Logistic_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import roc_auc_score \n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import confusion_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stops = stopwords.words(\"english\")\n",
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")\n",
    "train_df['comment_text'].fillna(\"unknown\", inplace=True)\n",
    "test_df['comment_text'].fillna(\"unknown\", inplace=True)\n",
    "\n",
    "X = train_df['comment_text'].values\n",
    "y = train_df[['toxic','severe_toxic','obscene','threat','insult','identity_hate']].values\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "X_test = test_df.drop(\"id\", axis =1).values.ravel()\n",
    "#X_test.shape\n",
    "\n",
    "def text_prepare(text):\n",
    "    text = [i for i in text.split() if i not in stops]\n",
    "    return \" \".join(text)\n",
    "\n",
    "X_train = [text_prepare(x) for x in X_train]\n",
    "X_val = [text_prepare(x) for x in X_val]\n",
    "X_test = [text_prepare(x) for x in X_test]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<111699x140456 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 5172475 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(min_df=5,max_df = 0.9,ngram_range=(1, 3))\n",
    "tfidf_vectorizer.fit(X_train)\n",
    "X_train_tfid = tfidf_vectorizer.transform(X_train)\n",
    "X_val_tfid = tfidf_vectorizer.transform(X_val) #validation\n",
    "X_test_tfid = tfidf_vectorizer.transform(X_test) #test\n",
    "\n",
    "X_train_tfid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "##taking c= 4 where it produced the best score from below\n",
    "model_tfidf = OneVsRestClassifier(LogisticRegression(C = 4)).fit(X_train_tfid, y_train)\n",
    "predicted_val_tfidf = model_tfidf.predict(X_val_tfid) \n",
    "\n",
    "#predicted_test_tfidf = model_tfidf.predict(X_test_tfid) \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For label toxic:\n",
      "Confusion Matrix:\n",
      " [[42955   335]\n",
      " [ 1600  2982]]\n",
      "Acc: 0.9595797125668449\n",
      "\n",
      "\n",
      "For label severe_toxic:\n",
      "Confusion Matrix:\n",
      " [[47285   101]\n",
      " [  366   120]]\n",
      "Acc: 0.9902448195187166\n",
      "\n",
      "\n",
      "For label obscene:\n",
      "Confusion Matrix:\n",
      " [[45153   163]\n",
      " [  902  1654]]\n",
      "Acc: 0.9777531751336899\n",
      "\n",
      "\n",
      "For label threat:\n",
      "Confusion Matrix:\n",
      " [[47727     9]\n",
      " [  105    31]]\n",
      "Acc: 0.9976186497326203\n",
      "\n",
      "\n",
      "For label insult:\n",
      "Confusion Matrix:\n",
      " [[45180   303]\n",
      " [ 1068  1321]]\n",
      "Acc: 0.9713611296791443\n",
      "\n",
      "\n",
      "For label identity_hate:\n",
      "Confusion Matrix:\n",
      " [[47397    43]\n",
      " [  343    89]]\n",
      "Acc: 0.9919368315508021\n",
      "\n",
      "\n",
      "Metrics in general\n",
      "accuracy:  0.9199740975935828\n",
      "f1:  0.5295317982291321\n",
      "precision:  0.3608543319515538\n",
      "recall 0.4219561099564177\n",
      "ROC AUC: 0.7092094152073677\n"
     ]
    }
   ],
   "source": [
    "#'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.\n",
    "def get_score(y_val, pred):\n",
    "    accuracy = accuracy_score(y_val, pred)\n",
    "    f1 = f1_score(y_val, pred,average='macro')\n",
    "    precision = average_precision_score(y_val, pred,average='macro')\n",
    "    recall = recall_score(y_val, pred, average='macro')\n",
    "    #confu_matrix = confusion_matrix(y_val, pred, labels=['toxic','severe_toxic','obscene','threat','insult','identity_hate'])\n",
    "    roc_auc = roc_auc_score(y_val, pred, average='macro')\n",
    "    print(\"accuracy: \", accuracy)\n",
    "    print(\"f1: \",f1)\n",
    "    print(\"precision: \",precision)\n",
    "    print(\"recall\", recall)\n",
    "    #print(\"confusion matrix:\", confu_matrix)\n",
    "    print(\"ROC AUC:\", roc_auc)\n",
    "    \n",
    "###2*2 confusion matrix for each class\n",
    "labels = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']\n",
    "conf_mat_dict={}\n",
    "acc_dict = {}\n",
    "for label_col in range(len(labels)):\n",
    "    y_val_label = y_val[:, label_col]\n",
    "    y_pred_label = predicted_tfidf[:, label_col]\n",
    "    acc_dict[labels[label_col]] = accuracy_score(y_val_label,y_pred_label)\n",
    "    conf_mat_dict[labels[label_col]] = confusion_matrix(y_pred=y_pred_label, y_true=y_val_label)\n",
    "    \n",
    "for label, matrix in conf_mat_dict.items():\n",
    "    print(\"For label {}:\".format(label))\n",
    "    print(\"Confusion Matrix:\\n\", matrix)\n",
    "    print(\"Acc:\", acc_dict[label])\n",
    "    print(\"\\n\")\n",
    "\n",
    "    \n",
    "### General scores\n",
    "print(\"Metrics in general\")\n",
    "get_score(y_val, predicted_tfidf)\n",
    "#cf = confusion_matrix(y_val.argmax(axis=1),predicted_tfidf.argmax(axis=1))\n",
    "#print(\"\\n Confusion matrix - y is actual, x is predict, labels are 'toxic','severe_toxic','obscene','threat','insult','identity_hate'\")\n",
    "#print(cf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lambda:  1e-05\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.8850058489304813\n",
      "f1:  0.48295042223947804\n",
      "precision:  0.2875945405211972\n",
      "recall 0.4668274513372188\n",
      "ROC AUC: 0.7267703081295919\n",
      "\n",
      "\n",
      "Lambda:  0.001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9088611296791443\n",
      "f1:  0.5295034824809032\n",
      "precision:  0.33946516210398975\n",
      "recall 0.4565927771486305\n",
      "ROC AUC: 0.7247690142216304\n",
      "\n",
      "\n",
      "Lambda:  0.01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9161096256684492\n",
      "f1:  0.5428371624109026\n",
      "precision:  0.35784428362725706\n",
      "recall 0.45664671163089415\n",
      "ROC AUC: 0.725582912326629\n",
      "\n",
      "\n",
      "Lambda:  0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9194727606951871\n",
      "f1:  0.5391209364943597\n",
      "precision:  0.3646493574753645\n",
      "recall 0.43838140361143624\n",
      "ROC AUC: 0.7171229970042269\n",
      "\n",
      "\n",
      "Lambda:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9163602941176471\n",
      "f1:  0.44332867671987813\n",
      "precision:  0.3037272489789623\n",
      "recall 0.3351844217713172\n",
      "ROC AUC: 0.6663836087124374\n",
      "\n",
      "\n",
      "Lambda:  10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9049339906417112\n",
      "f1:  0.20649623693823527\n",
      "precision:  0.15055917207699201\n",
      "recall 0.12957133728043302\n",
      "ROC AUC: 0.5645940975773734\n",
      "\n",
      "\n",
      "Lambda:  100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.8983748328877005\n",
      "f1:  0.0013191496117279918\n",
      "precision:  0.03744653552176972\n",
      "recall 0.0006609406019836301\n",
      "ROC AUC: 0.5003304703009918\n",
      "\n",
      "\n",
      "Lambda:  1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.8983330548128342\n",
      "f1:  0.0\n",
      "precision:  0.03683781751336898\n",
      "recall 0.0\n",
      "ROC AUC: 0.5\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "#Changing the regularization parameter lambda in the Logit Regression function\n",
    "\n",
    "lambdas = [10**-5, 10**-3, 10**-2, 10**-1, 1, 10, 100, 1000]\n",
    "\n",
    "for lambda_val in lambdas:\n",
    "    \n",
    "    print(\"Lambda: \",lambda_val)\n",
    "    model_lambda = OneVsRestClassifier(LogisticRegression(C = 1/lambda_val)).fit(X_train_tfid, y_train)\n",
    "    predicted_tfidf_lambda = model_lambda.predict(X_val_tfid)\n",
    "    get_score(y_val, predicted_tfidf_lambda)\n",
    "    print(\"\\n\")\n",
    "    \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lambda:  0.55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9178434157754011\n",
      "f1:  0.48850229733571066\n",
      "precision:  0.33284801146967463\n",
      "\n",
      "\n",
      "Lambda:  0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9194727606951871\n",
      "f1:  0.5391209364943597\n",
      "precision:  0.3646493574753645\n",
      "\n",
      "\n",
      "Lambda:  0.25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9199740975935828\n",
      "f1:  0.5295317982291321\n",
      "precision:  0.3608543319515538\n",
      "\n",
      "\n",
      "Lambda:  0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9183447526737968\n",
      "f1:  0.49606877726376936\n",
      "precision:  0.33783332153204376\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lambda_vals = [0.55,0.1,0.25,0.5]    \n",
    "\n",
    "for lambda_val in lambda_vals:\n",
    "    print(\"Lambda: \",lambda_val)\n",
    "    model_lambda = OneVsRestClassifier(LogisticRegression(C = 1/lambda_val)).fit(X_train_tfid, y_train)\n",
    "    predicted_tfidf_lambda = model_lambda.predict(X_val_tfid)\n",
    "    get_score(y_val, predicted_tfidf_lambda)\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9183447526737968\n",
      "f1:  0.49606877726376936\n",
      "precision:  0.33783332153204376\n"
     ]
    }
   ],
   "source": [
    "#l1 regulation\n",
    "model_l1 = OneVsRestClassifier(LogisticRegression(C= 1/0.25,penalty=\"l1\")).fit(X_train_tfid, y_train)\n",
    "predicted_tfidf_l1 = model_lambda.predict(X_val_tfid)\n",
    "get_score(y_val, predicted_tfidf_l1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}