190101103.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:43:17.096285Z",
     "iopub.status.busy": "2022-02-13T06:43:17.095152Z",
     "iopub.status.idle": "2022-02-13T06:43:18.537276Z",
     "shell.execute_reply": "2022-02-13T06:43:18.536807Z",
     "shell.execute_reply.started": "2022-02-13T06:43:17.09621Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk # natural language toolkit\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the dataset name carefully - only one of the lines should be uncommented amongst the 3\n",
    "\n",
    "dataset_name = 'FNC_multiclass'\n",
    "# dataset_name = 'FNC_binary'\n",
    "# dataset_name = 'NELA_binary'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(name):\n",
    "    if name == 'FNC_multiclass':\n",
    "        train_bodies = pd.read_csv('./Data-Set/FNC-1/train_bodies.csv')\n",
    "        train_stances = pd.read_csv('./Data-Set/FNC-1/train_stances.csv')\n",
    "        test_bodies = pd.read_csv('./Data-Set/FNC-1/competition_test_bodies.csv')\n",
    "        test_stances = pd.read_csv('./Data-Set/FNC-1/competition_test_stances.csv')\n",
    "        labels = ['agree', 'disagree', 'discuss', 'unrelated']\n",
    "        labels_dict = dict(zip(labels, range(len(labels))))\n",
    "        train_dataset = pd.merge(train_stances, train_bodies, how='left', on='Body ID')\n",
    "        train_dataset['Label'] = train_dataset['Stance'].map(lambda x: labels_dict[x])\n",
    "        train_dataset.drop(columns=['Stance', 'Body ID'], axis=1, inplace=True)\n",
    "        test_dataset = pd.merge(test_stances, test_bodies, how='left', on='Body ID')\n",
    "        test_dataset['Label'] = test_dataset['Stance'].map(lambda x: labels_dict[x])\n",
    "        test_dataset.drop(columns=['Stance', 'Body ID'], axis=1, inplace=True)\n",
    "        return train_dataset, test_dataset\n",
    "    \n",
    "    elif name == 'FNC_binary':\n",
    "        train_dataset = pd.read_csv('./Data-Set/FNC_Bin_Train.csv')\n",
    "        train_dataset.drop(['Body ID'], axis=1, inplace=True)\n",
    "        train_dataset.rename(columns={'Stance':'Label'}, inplace=True)\n",
    "        test_dataset = pd.read_csv('./Data-Set/FNC_Bin_Test.csv')\n",
    "        test_dataset.drop(['Body ID'], axis=1, inplace=True)\n",
    "        test_dataset.rename(columns={'Stance':'Label'}, inplace=True)\n",
    "        return train_dataset, test_dataset\n",
    "    \n",
    "    elif name == 'NELA_binary':\n",
    "        train_dataset = pd.read_csv('./Data-Set/NELA_Train.csv')\n",
    "        train_dataset.drop(['ID'], axis=1, inplace=True)\n",
    "        train_dataset.rename(columns={'Body':'articleBody'}, inplace=True)\n",
    "        test_dataset = pd.read_csv('./Data-Set/NELA_Test.csv')\n",
    "        test_dataset.drop(['ID'], axis=1, inplace=True)\n",
    "        test_dataset.rename(columns={'Body':'articleBody'}, inplace=True)\n",
    "        return train_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>This week , explained : spies , special counse...</td>\n",
       "      <td>The Russia investigation got real this week . ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Jolyon Palmer to leave Renault after Japanese ...</td>\n",
       "      <td>British driver Jolyon Palmer is to leave the R...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Former FBi Director James Comey Admits That He...</td>\n",
       "      <td>Liberals have been frothing at the mouth , wai...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Trump : ' I have n't changed my stance ' on China</td>\n",
       "      <td>President Trump said Tuesday that he has n't s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Trump Announces New Cabinet Member With Choice...</td>\n",
       "      <td>Newsmax reported Trump ’ s next move in the st...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Headline  \\\n",
       "0  This week , explained : spies , special counse...   \n",
       "1  Jolyon Palmer to leave Renault after Japanese ...   \n",
       "2  Former FBi Director James Comey Admits That He...   \n",
       "3  Trump : ' I have n't changed my stance ' on China   \n",
       "4  Trump Announces New Cabinet Member With Choice...   \n",
       "\n",
       "                                         articleBody  Label  \n",
       "0  The Russia investigation got real this week . ...      0  \n",
       "1  British driver Jolyon Palmer is to leave the R...      1  \n",
       "2  Liberals have been frothing at the mouth , wai...      0  \n",
       "3  President Trump said Tuesday that he has n't s...      1  \n",
       "4  Newsmax reported Trump ’ s next move in the st...      0  "
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset, test_dataset = load_dataset(dataset_name)\n",
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:43:30.02925Z",
     "iopub.status.busy": "2022-02-13T06:43:30.029084Z",
     "iopub.status.idle": "2022-02-13T06:43:30.038286Z",
     "shell.execute_reply": "2022-02-13T06:43:30.037611Z",
     "shell.execute_reply.started": "2022-02-13T06:43:30.029227Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((71420, 3), (6302, 3))"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.shape, test_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:54:38.750621Z",
     "iopub.status.busy": "2022-02-13T06:54:38.750166Z",
     "iopub.status.idle": "2022-02-13T06:54:38.766159Z",
     "shell.execute_reply": "2022-02-13T06:54:38.765626Z",
     "shell.execute_reply.started": "2022-02-13T06:54:38.750588Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "    Lemmatization is the process of grouping together the different inflected \n",
    "    forms of a word so they can be analyzed as a single item. Lemmatization is \n",
    "    similar to stemming but it brings context to the words. So it links words \n",
    "    with similar meanings to one word. \n",
    "    References - \n",
    "    https://www.geeksforgeeks.org/introduction-to-stemming/\n",
    "    https://www.geeksforgeeks.org/python-lemmatization-with-nltk/\n",
    "    https://www.geeksforgeeks.org/nlp-how-tokenizing-text-sentence-words-works/\n",
    "\"\"\"\n",
    "\n",
    "from sklearn import feature_extraction\n",
    "lemmatizer = nltk.WordNetLemmatizer()\n",
    "\n",
    "\"\"\"\n",
    "    Preprocesses a string: Lowercasing, trimming, \n",
    "    removing non-alphanumeric\n",
    "\"\"\"\n",
    "def preprocess(s):\n",
    "    return \" \".join(re.findall(r'\\w+', s, flags=re.UNICODE)).lower()\n",
    "\n",
    "def lemmatize_lowercase(w):\n",
    "    return lemmatizer.lemmatize(w).lower()\n",
    "\n",
    "\"\"\"\n",
    "    split the sentence into words using word tokenizer and \n",
    "    then produce a lemmatized lowercase version of each word\n",
    "\"\"\"\n",
    "def word_tokenize_lemmatize_lowercase(s):\n",
    "    return [lemmatize_lowercase(t) for t in nltk.word_tokenize(s)]\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "    The stopwords are a list of words that are very very common but don’t \n",
    "    provide useful information for most text analysis procedures. That is, \n",
    "    these words are ignored during most natural language processing tasks, \n",
    "    such as part-of-speech tagging, tokenization and parsing.\n",
    "    \n",
    "    While they are helpful for understanding the structure of sentences, they do \n",
    "    not help you understand the semantics of the sentences themselves.\n",
    "\"\"\"\n",
    "def remove_stopwords(l):\n",
    "    \"\"\"\n",
    "        Remove stopwords from the lemmatized list of words l\n",
    "    \"\"\"\n",
    "    return [w for w in l if w not in feature_extraction.text.ENGLISH_STOP_WORDS]\n",
    "\n",
    "\n",
    "def unigrams(line):\n",
    "    \"\"\"\n",
    "        Generate unigrams from a given string.\n",
    "        returns a list of unigrams\n",
    "    \"\"\"\n",
    "    line = preprocess(line)\n",
    "    lemmatized_line = word_tokenize_lemmatize_lowercase(line)\n",
    "    lemmatized_line_wo_stopwords = remove_stopwords(lemmatized_line)\n",
    "    return lemmatized_line_wo_stopwords\n",
    "\n",
    "def bigrams(line):     \n",
    "    \"\"\"\n",
    "        Generate bigrams from a given string\n",
    "        returns a list of bigrams where\n",
    "        each bigram is represented as a list of 2 words\n",
    "    \"\"\"\n",
    "    line = preprocess(line)\n",
    "    lemmatized_line = word_tokenize_lemmatize_lowercase(line)\n",
    "    lemmatized_line_wo_stopwords = remove_stopwords(lemmatized_line)\n",
    "    output = []\n",
    "    for i in range(len(lemmatized_line_wo_stopwords) - 1):\n",
    "        output.append(lemmatized_line_wo_stopwords[i:i + 2])\n",
    "        \n",
    "    return output\n",
    "\n",
    "\n",
    "def trigrams(line):\n",
    "    \"\"\"\n",
    "        Generate trigrams from a given string\n",
    "        returns a list of trigrams where\n",
    "        each trigram is represented as a list of 3 words\n",
    "    \"\"\"\n",
    "    line = preprocess(line)\n",
    "    lemmatized_line = word_tokenize_lemmatize_lowercase(line)\n",
    "    lemmatized_line_wo_stopwords = remove_stopwords(lemmatized_line)\n",
    "    output = []\n",
    "    for i in range(len(lemmatized_line_wo_stopwords) - 2):\n",
    "        output.append(lemmatized_line_wo_stopwords[i:i + 3])\n",
    "        \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:49:32.263Z",
     "iopub.status.busy": "2022-02-13T06:49:32.262751Z",
     "iopub.status.idle": "2022-02-13T06:49:32.279319Z",
     "shell.execute_reply": "2022-02-13T06:49:32.278523Z",
     "shell.execute_reply.started": "2022-02-13T06:49:32.262973Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.sample(n=5000, random_state=1)\n",
    "test_dataset = test_dataset.sample(n=1000, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:49:35.367116Z",
     "iopub.status.busy": "2022-02-13T06:49:35.366202Z",
     "iopub.status.idle": "2022-02-13T06:49:35.372279Z",
     "shell.execute_reply": "2022-02-13T06:49:35.371569Z",
     "shell.execute_reply.started": "2022-02-13T06:49:35.367091Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 3), (1000, 3))"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.shape, test_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:49:37.947341Z",
     "iopub.status.busy": "2022-02-13T06:49:37.9465Z",
     "iopub.status.idle": "2022-02-13T06:49:38.219198Z",
     "shell.execute_reply": "2022-02-13T06:49:38.218216Z",
     "shell.execute_reply.started": "2022-02-13T06:49:37.947286Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/pradnesh/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/pradnesh/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T06:54:42.639123Z",
     "iopub.status.busy": "2022-02-13T06:54:42.638901Z",
     "iopub.status.idle": "2022-02-13T06:56:30.231764Z",
     "shell.execute_reply": "2022-02-13T06:56:30.230923Z",
     "shell.execute_reply.started": "2022-02-13T06:54:42.639095Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "headline unigrams train generated\n",
      "body unigrams train generated\n",
      "headline unigrams test generated\n",
      "body unigrams test generated\n"
     ]
    }
   ],
   "source": [
    "# generate unigrams\n",
    "train_dataset[\"Headline_unigrams\"] = train_dataset[\"Headline\"].map(lambda x: unigrams(x))\n",
    "print(\"headline unigrams train generated\")\n",
    "train_dataset[\"articleBody_unigrams\"] = train_dataset[\"articleBody\"].map(lambda x: unigrams(x))\n",
    "print(\"body unigrams train generated\")\n",
    "test_dataset[\"Headline_unigrams\"] = test_dataset[\"Headline\"].map(lambda x: unigrams(x))\n",
    "print(\"headline unigrams test generated\")\n",
    "test_dataset[\"articleBody_unigrams\"] = test_dataset[\"articleBody\"].map(lambda x: unigrams(x))\n",
    "print(\"body unigrams test generated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-02-12T07:40:50.105651Z",
     "iopub.status.idle": "2022-02-12T07:40:50.10632Z",
     "shell.execute_reply": "2022-02-12T07:40:50.106135Z",
     "shell.execute_reply.started": "2022-02-12T07:40:50.106113Z"
    }
   },
   "outputs": [],
   "source": [
    "# # generate bigrams\n",
    "# train_dataset[\"Headline_bigrams\"] = train_dataset[\"Headline\"].map(lambda x: bigrams(x))\n",
    "# print(\"headline bigrams train generated\")\n",
    "# train_dataset[\"articleBody_bigrams\"] = train_dataset[\"articleBody\"].map(lambda x: bigrams(x))\n",
    "# print(\"body bigrams train generated\")\n",
    "# test_dataset[\"Headline_bigrams\"] = test_dataset[\"Headline\"].map(lambda x: bigrams(x))\n",
    "# print(\"headline bigrams test generated\")\n",
    "# test_dataset[\"articleBody_bigrams\"] = test_dataset[\"articleBody\"].map(lambda x: bigrams(x))\n",
    "# print(\"body bigrams test generated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # generate trigrams\n",
    "# train_dataset[\"Headline_trigrams\"] = train_dataset[\"Headline\"].map(lambda x: trigrams(x))\n",
    "# print(\"headline trigrams train generated\")\n",
    "# train_dataset[\"articleBody_trigrams\"] = train_dataset[\"articleBody\"].map(lambda x: trigrams(x))\n",
    "# print(\"body trigrams train generated\")\n",
    "# test_dataset[\"Headline_trigrams\"] = test_dataset[\"Headline\"].map(lambda x: trigrams(x))\n",
    "# print(\"headline trigrams test generated\")\n",
    "# test_dataset[\"articleBody_trigrams\"] = test_dataset[\"articleBody\"].map(lambda x: trigrams(x))\n",
    "# print(\"body trigrams test generated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T08:05:27.35442Z",
     "iopub.status.busy": "2022-02-12T08:05:27.354094Z",
     "iopub.status.idle": "2022-02-12T08:05:27.37605Z",
     "shell.execute_reply": "2022-02-12T08:05:27.375093Z",
     "shell.execute_reply.started": "2022-02-12T08:05:27.354383Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Label</th>\n",
       "      <th>Headline_unigrams</th>\n",
       "      <th>articleBody_unigrams</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49255</th>\n",
       "      <td>21K Illegal Aliens Arrested Under Trump So Far</td>\n",
       "      <td>According to newly released statistics by Immi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[21k, illegal, alien, arrested, trump, far]</td>\n",
       "      <td>[according, newly, released, statistic, immigr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55535</th>\n",
       "      <td>University of Wisconsin Passes Dangerous New P...</td>\n",
       "      <td>It ’ s good to know that the First Amendment i...</td>\n",
       "      <td>1</td>\n",
       "      <td>[university, wisconsin, pass, dangerous, new, ...</td>\n",
       "      <td>[s, good, know, amendment, protected, college,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6633</th>\n",
       "      <td>No Diversity Here : Officially Recognized 'Fac...</td>\n",
       "      <td>Posts over the next several days will show tha...</td>\n",
       "      <td>1</td>\n",
       "      <td>[diversity, officially, recognized, fact, chec...</td>\n",
       "      <td>[post, day, certain, left, leaning, website, e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68666</th>\n",
       "      <td>CONFIRMED : Syrian Arab Army takes control of ...</td>\n",
       "      <td>Fierce fighting with ISIS and US backed jihadi...</td>\n",
       "      <td>0</td>\n",
       "      <td>[confirmed, syrian, arab, army, control, jorda...</td>\n",
       "      <td>[fierce, fighting, isi, u, backed, jihadist, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9359</th>\n",
       "      <td>Jets are seeing a whole new side of Muhammad W...</td>\n",
       "      <td>According to David Z. Morris at Fortune magazi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[jet, seeing, new, muhammad, wilkerson]</td>\n",
       "      <td>[according, david, z, morris, fortune, magazin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Headline  \\\n",
       "49255     21K Illegal Aliens Arrested Under Trump So Far   \n",
       "55535  University of Wisconsin Passes Dangerous New P...   \n",
       "6633   No Diversity Here : Officially Recognized 'Fac...   \n",
       "68666  CONFIRMED : Syrian Arab Army takes control of ...   \n",
       "9359   Jets are seeing a whole new side of Muhammad W...   \n",
       "\n",
       "                                             articleBody  Label  \\\n",
       "49255  According to newly released statistics by Immi...      1   \n",
       "55535  It ’ s good to know that the First Amendment i...      1   \n",
       "6633   Posts over the next several days will show tha...      1   \n",
       "68666  Fierce fighting with ISIS and US backed jihadi...      0   \n",
       "9359   According to David Z. Morris at Fortune magazi...      1   \n",
       "\n",
       "                                       Headline_unigrams  \\\n",
       "49255        [21k, illegal, alien, arrested, trump, far]   \n",
       "55535  [university, wisconsin, pass, dangerous, new, ...   \n",
       "6633   [diversity, officially, recognized, fact, chec...   \n",
       "68666  [confirmed, syrian, arab, army, control, jorda...   \n",
       "9359             [jet, seeing, new, muhammad, wilkerson]   \n",
       "\n",
       "                                    articleBody_unigrams  \n",
       "49255  [according, newly, released, statistic, immigr...  \n",
       "55535  [s, good, know, amendment, protected, college,...  \n",
       "6633   [post, day, certain, left, leaning, website, e...  \n",
       "68666  [fierce, fighting, isi, u, backed, jihadist, s...  \n",
       "9359   [according, david, z, morris, fortune, magazin...  "
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:14:09.109953Z",
     "iopub.status.busy": "2022-02-13T07:14:09.109494Z",
     "iopub.status.idle": "2022-02-13T07:14:09.114833Z",
     "shell.execute_reply": "2022-02-13T07:14:09.114131Z",
     "shell.execute_reply.started": "2022-02-13T07:14:09.109914Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Join the unigrams of headline and body as a single string\n",
    "\"\"\"\n",
    "def combine_headline_body_unigrams(x):\n",
    "    res = '%s %s' % (' '.join(x['Headline_unigrams']), ' '.join(x['articleBody_unigrams']))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:17:57.816253Z",
     "iopub.status.busy": "2022-02-13T07:17:57.815116Z",
     "iopub.status.idle": "2022-02-13T07:17:58.049222Z",
     "shell.execute_reply": "2022-02-13T07:17:58.048435Z",
     "shell.execute_reply.started": "2022-02-13T07:17:57.816215Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset[\"headline_plus_body\"] = train_dataset.apply(combine_headline_body_unigrams, axis=1)\n",
    "test_dataset[\"headline_plus_body\"] = test_dataset.apply(combine_headline_body_unigrams, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:18:04.230118Z",
     "iopub.status.busy": "2022-02-13T07:18:04.229867Z",
     "iopub.status.idle": "2022-02-13T07:18:04.251967Z",
     "shell.execute_reply": "2022-02-13T07:18:04.251464Z",
     "shell.execute_reply.started": "2022-02-13T07:18:04.23009Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Label</th>\n",
       "      <th>Headline_unigrams</th>\n",
       "      <th>articleBody_unigrams</th>\n",
       "      <th>headline_plus_body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49255</th>\n",
       "      <td>21K Illegal Aliens Arrested Under Trump So Far</td>\n",
       "      <td>According to newly released statistics by Immi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[21k, illegal, alien, arrested, trump, far]</td>\n",
       "      <td>[according, newly, released, statistic, immigr...</td>\n",
       "      <td>21k illegal alien arrested trump far according...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55535</th>\n",
       "      <td>University of Wisconsin Passes Dangerous New P...</td>\n",
       "      <td>It ’ s good to know that the First Amendment i...</td>\n",
       "      <td>1</td>\n",
       "      <td>[university, wisconsin, pass, dangerous, new, ...</td>\n",
       "      <td>[s, good, know, amendment, protected, college,...</td>\n",
       "      <td>university wisconsin pass dangerous new policy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6633</th>\n",
       "      <td>No Diversity Here : Officially Recognized 'Fac...</td>\n",
       "      <td>Posts over the next several days will show tha...</td>\n",
       "      <td>1</td>\n",
       "      <td>[diversity, officially, recognized, fact, chec...</td>\n",
       "      <td>[post, day, certain, left, leaning, website, e...</td>\n",
       "      <td>diversity officially recognized fact checker l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68666</th>\n",
       "      <td>CONFIRMED : Syrian Arab Army takes control of ...</td>\n",
       "      <td>Fierce fighting with ISIS and US backed jihadi...</td>\n",
       "      <td>0</td>\n",
       "      <td>[confirmed, syrian, arab, army, control, jorda...</td>\n",
       "      <td>[fierce, fighting, isi, u, backed, jihadist, s...</td>\n",
       "      <td>confirmed syrian arab army control jordanian b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9359</th>\n",
       "      <td>Jets are seeing a whole new side of Muhammad W...</td>\n",
       "      <td>According to David Z. Morris at Fortune magazi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[jet, seeing, new, muhammad, wilkerson]</td>\n",
       "      <td>[according, david, z, morris, fortune, magazin...</td>\n",
       "      <td>jet seeing new muhammad wilkerson according da...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Headline  \\\n",
       "49255     21K Illegal Aliens Arrested Under Trump So Far   \n",
       "55535  University of Wisconsin Passes Dangerous New P...   \n",
       "6633   No Diversity Here : Officially Recognized 'Fac...   \n",
       "68666  CONFIRMED : Syrian Arab Army takes control of ...   \n",
       "9359   Jets are seeing a whole new side of Muhammad W...   \n",
       "\n",
       "                                             articleBody  Label  \\\n",
       "49255  According to newly released statistics by Immi...      1   \n",
       "55535  It ’ s good to know that the First Amendment i...      1   \n",
       "6633   Posts over the next several days will show tha...      1   \n",
       "68666  Fierce fighting with ISIS and US backed jihadi...      0   \n",
       "9359   According to David Z. Morris at Fortune magazi...      1   \n",
       "\n",
       "                                       Headline_unigrams  \\\n",
       "49255        [21k, illegal, alien, arrested, trump, far]   \n",
       "55535  [university, wisconsin, pass, dangerous, new, ...   \n",
       "6633   [diversity, officially, recognized, fact, chec...   \n",
       "68666  [confirmed, syrian, arab, army, control, jorda...   \n",
       "9359             [jet, seeing, new, muhammad, wilkerson]   \n",
       "\n",
       "                                    articleBody_unigrams  \\\n",
       "49255  [according, newly, released, statistic, immigr...   \n",
       "55535  [s, good, know, amendment, protected, college,...   \n",
       "6633   [post, day, certain, left, leaning, website, e...   \n",
       "68666  [fierce, fighting, isi, u, backed, jihadist, s...   \n",
       "9359   [according, david, z, morris, fortune, magazin...   \n",
       "\n",
       "                                      headline_plus_body  \n",
       "49255  21k illegal alien arrested trump far according...  \n",
       "55535  university wisconsin pass dangerous new policy...  \n",
       "6633   diversity officially recognized fact checker l...  \n",
       "68666  confirmed syrian arab army control jordanian b...  \n",
       "9359   jet seeing new muhammad wilkerson according da...  "
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:32:41.07512Z",
     "iopub.status.busy": "2022-02-13T07:32:41.074791Z",
     "iopub.status.idle": "2022-02-13T07:33:07.608168Z",
     "shell.execute_reply": "2022-02-13T07:33:07.607456Z",
     "shell.execute_reply.started": "2022-02-13T07:32:41.075089Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(max_features=2500, ngram_range=(1, 3))"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit the tfidf vectorizer on the headline+body strings and\n",
    "# calculate tfidf on all the text from training dataset\n",
    "tfidf_vec = TfidfVectorizer(ngram_range=(1, 3), max_features=2500)\n",
    "tfidf_vec.fit(train_dataset['headline_plus_body'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:33:41.861554Z",
     "iopub.status.busy": "2022-02-13T07:33:41.861271Z",
     "iopub.status.idle": "2022-02-13T07:33:41.877363Z",
     "shell.execute_reply": "2022-02-13T07:33:41.876947Z",
     "shell.execute_reply.started": "2022-02-13T07:33:41.861531Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2500\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('illegal', 1153),\n",
       " ('alien', 126),\n",
       " ('arrested', 189),\n",
       " ('trump', 2299),\n",
       " ('far', 917)]"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vocab = tfidf_vec.vocabulary_\n",
    "print(len(list(tfidf_vocab.items())))\n",
    "list(tfidf_vocab.items())[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:35:44.168021Z",
     "iopub.status.busy": "2022-02-13T07:35:44.167792Z",
     "iopub.status.idle": "2022-02-13T07:35:44.432145Z",
     "shell.execute_reply": "2022-02-13T07:35:44.431169Z",
     "shell.execute_reply.started": "2022-02-13T07:35:44.167998Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 2500), (1000, 2500))"
      ]
     },
     "execution_count": 267,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headline_vectorizer_tfidf = TfidfVectorizer(ngram_range=(1, 3), vocabulary=tfidf_vocab)\n",
    "Features_Train_Headline_Tfidf = headline_vectorizer_tfidf.fit_transform(train_dataset['Headline_unigrams'].map(lambda x: ' '.join(x)))\n",
    "Features_Test_Headline_Tfidf = headline_vectorizer_tfidf.transform(test_dataset['Headline_unigrams'].map(lambda x: ' '.join(x)))\n",
    "Features_Train_Headline_Tfidf.shape, Features_Test_Headline_Tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:45:20.743977Z",
     "iopub.status.busy": "2022-02-13T07:45:20.743686Z",
     "iopub.status.idle": "2022-02-13T07:45:26.853386Z",
     "shell.execute_reply": "2022-02-13T07:45:26.852558Z",
     "shell.execute_reply.started": "2022-02-13T07:45:20.743948Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 2500), (1000, 2500))"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articleBody_vectorizer_tfidf = TfidfVectorizer(ngram_range=(1, 3), vocabulary=tfidf_vocab)\n",
    "Features_Train_Body_Tfidf = articleBody_vectorizer_tfidf.fit_transform(train_dataset['articleBody_unigrams'].map(lambda x: ' '.join(x)))\n",
    "Features_Test_Body_Tfidf = articleBody_vectorizer_tfidf.transform(test_dataset['articleBody_unigrams'].map(lambda x: ' '.join(x)))\n",
    "Features_Train_Body_Tfidf.shape, Features_Test_Body_Tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:45:51.208692Z",
     "iopub.status.busy": "2022-02-13T07:45:51.208475Z",
     "iopub.status.idle": "2022-02-13T07:45:51.214347Z",
     "shell.execute_reply": "2022-02-13T07:45:51.213613Z",
     "shell.execute_reply.started": "2022-02-13T07:45:51.20867Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 2500), (5000, 2500), (1000, 2500), (1000, 2500))"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Features_Train_Headline_Tfidf.shape, Features_Train_Body_Tfidf.shape, Features_Test_Headline_Tfidf.shape, Features_Test_Body_Tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:46:35.868894Z",
     "iopub.status.busy": "2022-02-13T07:46:35.868627Z",
     "iopub.status.idle": "2022-02-13T07:46:35.875391Z",
     "shell.execute_reply": "2022-02-13T07:46:35.874308Z",
     "shell.execute_reply.started": "2022-02-13T07:46:35.868865Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(Features_Train_Headline_Tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:47:51.310351Z",
     "iopub.status.busy": "2022-02-13T07:47:51.310066Z",
     "iopub.status.idle": "2022-02-13T07:47:51.404288Z",
     "shell.execute_reply": "2022-02-13T07:47:51.403478Z",
     "shell.execute_reply.started": "2022-02-13T07:47:51.310321Z"
    }
   },
   "outputs": [],
   "source": [
    "train_Features_Tfidf = [Features_Train_Headline_Tfidf.toarray(), Features_Train_Body_Tfidf.toarray()]\n",
    "test_Features_Tfidf = [Features_Test_Headline_Tfidf.toarray(), Features_Test_Body_Tfidf.toarray()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:47:53.985664Z",
     "iopub.status.busy": "2022-02-13T07:47:53.985445Z",
     "iopub.status.idle": "2022-02-13T07:47:53.990944Z",
     "shell.execute_reply": "2022-02-13T07:47:53.990082Z",
     "shell.execute_reply.started": "2022-02-13T07:47:53.985642Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_Features_Tfidf[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:48:07.340388Z",
     "iopub.status.busy": "2022-02-13T07:48:07.33974Z",
     "iopub.status.idle": "2022-02-13T07:48:07.390967Z",
     "shell.execute_reply": "2022-02-13T07:48:07.389696Z",
     "shell.execute_reply.started": "2022-02-13T07:48:07.340356Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x = np.hstack(train_Features_Tfidf)\n",
    "test_x = np.hstack(test_Features_Tfidf)\n",
    "type(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:48:10.056629Z",
     "iopub.status.busy": "2022-02-13T07:48:10.05636Z",
     "iopub.status.idle": "2022-02-13T07:48:10.062382Z",
     "shell.execute_reply": "2022-02-13T07:48:10.061636Z",
     "shell.execute_reply.started": "2022-02-13T07:48:10.056602Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 5000)"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:48:15.062923Z",
     "iopub.status.busy": "2022-02-13T07:48:15.062603Z",
     "iopub.status.idle": "2022-02-13T07:48:15.066959Z",
     "shell.execute_reply": "2022-02-13T07:48:15.066197Z",
     "shell.execute_reply.started": "2022-02-13T07:48:15.0629Z"
    }
   },
   "outputs": [],
   "source": [
    "train_y = train_dataset[\"Label\"].values\n",
    "test_y = test_dataset[\"Label\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:48:31.719998Z",
     "iopub.status.busy": "2022-02-13T07:48:31.719775Z",
     "iopub.status.idle": "2022-02-13T07:48:32.537927Z",
     "shell.execute_reply": "2022-02-13T07:48:32.537097Z",
     "shell.execute_reply.started": "2022-02-13T07:48:31.719977Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# saving RAM\n",
    "import gc #garbage collector\n",
    "del train_dataset, test_dataset\n",
    "del Features_Test_Headline_Tfidf, Features_Test_Body_Tfidf, Features_Train_Body_Tfidf, Features_Train_Headline_Tfidf\n",
    "del train_Features_Tfidf, test_Features_Tfidf\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:48:50.032611Z",
     "iopub.status.busy": "2022-02-13T07:48:50.032379Z",
     "iopub.status.idle": "2022-02-13T07:48:50.036935Z",
     "shell.execute_reply": "2022-02-13T07:48:50.0361Z",
     "shell.execute_reply.started": "2022-02-13T07:48:50.032588Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:49:03.951563Z",
     "iopub.status.busy": "2022-02-13T07:49:03.951071Z",
     "iopub.status.idle": "2022-02-13T07:49:04.185033Z",
     "shell.execute_reply": "2022-02-13T07:49:04.184473Z",
     "shell.execute_reply.started": "2022-02-13T07:49:03.951523Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC()"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use svms linear svc - one vs all multiclass classification\n",
    "svm_classifier = svm.LinearSVC()\n",
    "svm_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:49:21.43738Z",
     "iopub.status.busy": "2022-02-13T07:49:21.436927Z",
     "iopub.status.idle": "2022-02-13T07:49:21.454698Z",
     "shell.execute_reply": "2022-02-13T07:49:21.453846Z",
     "shell.execute_reply.started": "2022-02-13T07:49:21.437348Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.546"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_predictions = svm_classifier.predict(test_x)\n",
    "accuracy_svm = accuracy_score(test_y, svm_predictions)\n",
    "accuracy_svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:49:37.946591Z",
     "iopub.status.busy": "2022-02-13T07:49:37.94629Z",
     "iopub.status.idle": "2022-02-13T07:49:38.05091Z",
     "shell.execute_reply": "2022-02-13T07:49:38.050043Z",
     "shell.execute_reply.started": "2022-02-13T07:49:37.946562Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB()"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb_classifier = GaussianNB()\n",
    "gnb_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:49:52.650881Z",
     "iopub.status.busy": "2022-02-13T07:49:52.650589Z",
     "iopub.status.idle": "2022-02-13T07:49:52.707585Z",
     "shell.execute_reply": "2022-02-13T07:49:52.706724Z",
     "shell.execute_reply.started": "2022-02-13T07:49:52.650851Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.495"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnb_predictions = gnb_classifier.predict(test_x)\n",
    "accuracy_gnb = accuracy_score(test_y, gnb_predictions)\n",
    "accuracy_gnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:50:32.180842Z",
     "iopub.status.busy": "2022-02-13T07:50:32.180598Z",
     "iopub.status.idle": "2022-02-13T07:50:32.4727Z",
     "shell.execute_reply": "2022-02-13T07:50:32.472027Z",
     "shell.execute_reply.started": "2022-02-13T07:50:32.180804Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(max_depth=2)"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dectree_classifier = DecisionTreeClassifier(max_depth=2)\n",
    "dectree_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:50:38.74759Z",
     "iopub.status.busy": "2022-02-13T07:50:38.747342Z",
     "iopub.status.idle": "2022-02-13T07:50:38.763773Z",
     "shell.execute_reply": "2022-02-13T07:50:38.762835Z",
     "shell.execute_reply.started": "2022-02-13T07:50:38.747563Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.607"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dectree_predictions = dectree_classifier.predict(test_x)\n",
    "accuracy_dtree = accuracy_score(test_y, dectree_predictions)\n",
    "accuracy_dtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:50:44.342778Z",
     "iopub.status.busy": "2022-02-13T07:50:44.342539Z",
     "iopub.status.idle": "2022-02-13T07:50:44.992394Z",
     "shell.execute_reply": "2022-02-13T07:50:44.991557Z",
     "shell.execute_reply.started": "2022-02-13T07:50:44.342756Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(max_depth=2, random_state=0)"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "random_forest_classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "random_forest_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:50:48.643122Z",
     "iopub.status.busy": "2022-02-13T07:50:48.642858Z",
     "iopub.status.idle": "2022-02-13T07:50:48.692919Z",
     "shell.execute_reply": "2022-02-13T07:50:48.692297Z",
     "shell.execute_reply.started": "2022-02-13T07:50:48.643094Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.599"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rndforest_predictions = random_forest_classifier.predict(test_x)\n",
    "accuracy_rndforest = accuracy_score(test_y, rndforest_predictions)\n",
    "accuracy_rndforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:50:52.35812Z",
     "iopub.status.busy": "2022-02-13T07:50:52.357099Z",
     "iopub.status.idle": "2022-02-13T07:50:52.361722Z",
     "shell.execute_reply": "2022-02-13T07:50:52.361076Z",
     "shell.execute_reply.started": "2022-02-13T07:50:52.35805Z"
    }
   },
   "outputs": [],
   "source": [
    "#################################################################\n",
    "######### Experiment 2 begins - Feature Selection ###############\n",
    "#################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:51:10.928713Z",
     "iopub.status.busy": "2022-02-13T07:51:10.92829Z",
     "iopub.status.idle": "2022-02-13T07:51:10.942187Z",
     "shell.execute_reply": "2022-02-13T07:51:10.941233Z",
     "shell.execute_reply.started": "2022-02-13T07:51:10.928673Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:51:12.646954Z",
     "iopub.status.busy": "2022-02-13T07:51:12.646557Z",
     "iopub.status.idle": "2022-02-13T07:51:12.653025Z",
     "shell.execute_reply": "2022-02-13T07:51:12.651976Z",
     "shell.execute_reply.started": "2022-02-13T07:51:12.646912Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 5000), (5000,))"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:51:40.956264Z",
     "iopub.status.busy": "2022-02-13T07:51:40.955465Z",
     "iopub.status.idle": "2022-02-13T07:51:41.033829Z",
     "shell.execute_reply": "2022-02-13T07:51:41.033259Z",
     "shell.execute_reply.started": "2022-02-13T07:51:40.956229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 500), (1000, 500))"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using chi square for feature selection\n",
    "# Note: we wont use test data and dev set data here for fitting chi_square\n",
    "chi_square = SelectKBest(chi2, k=500)\n",
    "chi_square.fit(train_x, train_y)\n",
    "train_x_chi_square = chi_square.transform(train_x)\n",
    "test_x_chi_square = chi_square.transform(test_x)\n",
    "train_x_chi_square.shape, test_x_chi_square.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:52:09.476492Z",
     "iopub.status.busy": "2022-02-13T07:52:09.47621Z",
     "iopub.status.idle": "2022-02-13T07:52:09.559144Z",
     "shell.execute_reply": "2022-02-13T07:52:09.558452Z",
     "shell.execute_reply.started": "2022-02-13T07:52:09.476463Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC()"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_classifier = svm.LinearSVC()\n",
    "svm_classifier.fit(train_x_chi_square, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:52:12.03616Z",
     "iopub.status.busy": "2022-02-13T07:52:12.035479Z",
     "iopub.status.idle": "2022-02-13T07:52:12.045825Z",
     "shell.execute_reply": "2022-02-13T07:52:12.045136Z",
     "shell.execute_reply.started": "2022-02-13T07:52:12.036131Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.569"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the svm classifier increased slightly due to feature selection and it took less time to train\n",
    "svm_predictions = svm_classifier.predict(test_x_chi_square)\n",
    "accuracy_svm = accuracy_score(test_y, svm_predictions)\n",
    "accuracy_svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:53:07.098519Z",
     "iopub.status.busy": "2022-02-13T07:53:07.098254Z",
     "iopub.status.idle": "2022-02-13T07:53:07.139934Z",
     "shell.execute_reply": "2022-02-13T07:53:07.139459Z",
     "shell.execute_reply.started": "2022-02-13T07:53:07.098491Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.487"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the naive bayes classifier increased slightly due to feature selection and it took less time to train\n",
    "gnb_classifier = GaussianNB()\n",
    "gnb_classifier.fit(train_x_chi_square, train_y)\n",
    "gnb_predictions = gnb_classifier.predict(test_x_chi_square)\n",
    "accuracy_gnb = accuracy_score(test_y, gnb_predictions)\n",
    "accuracy_gnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:53:38.314348Z",
     "iopub.status.busy": "2022-02-13T07:53:38.314114Z",
     "iopub.status.idle": "2022-02-13T07:53:38.371112Z",
     "shell.execute_reply": "2022-02-13T07:53:38.370464Z",
     "shell.execute_reply.started": "2022-02-13T07:53:38.314323Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.607"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the decision tree classifier increased due to feature selection and it took less time to train\n",
    "dectree_classifier = DecisionTreeClassifier(max_depth=2).fit(train_x_chi_square, train_y)\n",
    "dectree_predictions = dectree_classifier.predict(test_x_chi_square)\n",
    "accuracy_dtree = accuracy_score(test_y, dectree_predictions)\n",
    "accuracy_dtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:53:56.428599Z",
     "iopub.status.busy": "2022-02-13T07:53:56.428307Z",
     "iopub.status.idle": "2022-02-13T07:53:56.852852Z",
     "shell.execute_reply": "2022-02-13T07:53:56.852286Z",
     "shell.execute_reply.started": "2022-02-13T07:53:56.428569Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.592"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the random forest classifier remained same due to feature selection and it took less time to train\n",
    "random_forest_classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "random_forest_classifier.fit(train_x_chi_square, train_y)\n",
    "rndforest_predictions = random_forest_classifier.predict(test_x_chi_square)\n",
    "accuracy_rndforest = accuracy_score(test_y, rndforest_predictions)\n",
    "accuracy_rndforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:53:59.674127Z",
     "iopub.status.busy": "2022-02-13T07:53:59.673774Z",
     "iopub.status.idle": "2022-02-13T07:53:59.679494Z",
     "shell.execute_reply": "2022-02-13T07:53:59.678328Z",
     "shell.execute_reply.started": "2022-02-13T07:53:59.674104Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 5000), (5000,))"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T07:54:06.91789Z",
     "iopub.status.busy": "2022-02-13T07:54:06.917581Z",
     "iopub.status.idle": "2022-02-13T07:55:17.668098Z",
     "shell.execute_reply": "2022-02-13T07:55:17.667391Z",
     "shell.execute_reply.started": "2022-02-13T07:54:06.91786Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 500), (1000, 500))"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using mutual information for feature selection\n",
    "mutual_information = SelectKBest(mutual_info_classif, k = 500)\n",
    "mutual_information.fit(train_x, train_y)\n",
    "train_x_mutual_info = mutual_information.transform(train_x)\n",
    "test_x_mutual_info = mutual_information.transform(test_x)\n",
    "train_x_mutual_info.shape, test_x_mutual_info.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:02:06.563548Z",
     "iopub.status.busy": "2022-02-13T08:02:06.563247Z",
     "iopub.status.idle": "2022-02-13T08:02:06.674339Z",
     "shell.execute_reply": "2022-02-13T08:02:06.673543Z",
     "shell.execute_reply.started": "2022-02-13T08:02:06.563514Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.573"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the svm classifier increased due to feature selection and it took less time to train\n",
    "svm_classifier = svm.LinearSVC()\n",
    "svm_classifier.fit(train_x_mutual_info, train_y)\n",
    "svm_predictions = svm_classifier.predict(test_x_mutual_info)\n",
    "accuracy_svm = accuracy_score(test_y, svm_predictions)\n",
    "accuracy_svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:02:27.250822Z",
     "iopub.status.busy": "2022-02-13T08:02:27.249833Z",
     "iopub.status.idle": "2022-02-13T08:02:27.290829Z",
     "shell.execute_reply": "2022-02-13T08:02:27.289948Z",
     "shell.execute_reply.started": "2022-02-13T08:02:27.250779Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.486"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the gnb classifier remained same due to feature selection and it took less time to train\n",
    "gnb_classifier = GaussianNB()\n",
    "gnb_classifier.fit(train_x_mutual_info, train_y)\n",
    "gnb_predictions = gnb_classifier.predict(test_x_mutual_info)\n",
    "accuracy_gnb = accuracy_score(test_y, gnb_predictions)\n",
    "accuracy_gnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:04:11.057612Z",
     "iopub.status.busy": "2022-02-13T08:04:11.057318Z",
     "iopub.status.idle": "2022-02-13T08:04:11.142341Z",
     "shell.execute_reply": "2022-02-13T08:04:11.141606Z",
     "shell.execute_reply.started": "2022-02-13T08:04:11.057581Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.607"
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the dectree classifier increased slightly due to feature selection and it took less time to train\n",
    "dectree_classifier = DecisionTreeClassifier(max_depth=2).fit(train_x_mutual_info, train_y)\n",
    "dectree_predictions = dectree_classifier.predict(test_x_mutual_info)\n",
    "accuracy_dtree = accuracy_score(test_y, dectree_predictions)\n",
    "accuracy_dtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:04:22.140711Z",
     "iopub.status.busy": "2022-02-13T08:04:22.140387Z",
     "iopub.status.idle": "2022-02-13T08:04:22.656528Z",
     "shell.execute_reply": "2022-02-13T08:04:22.65561Z",
     "shell.execute_reply.started": "2022-02-13T08:04:22.140674Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.584"
      ]
     },
     "execution_count": 300,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the random forest classifier remained same due to feature selection and it took less time to train\n",
    "random_forest_classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "random_forest_classifier.fit(train_x_mutual_info, train_y)\n",
    "rndforest_predictions = random_forest_classifier.predict(test_x_mutual_info)\n",
    "accuracy_rndforest = accuracy_score(test_y, rndforest_predictions)\n",
    "accuracy_rndforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-12T08:10:48.011925Z",
     "iopub.status.busy": "2022-02-12T08:10:48.011644Z",
     "iopub.status.idle": "2022-02-12T08:10:48.015866Z",
     "shell.execute_reply": "2022-02-12T08:10:48.014807Z",
     "shell.execute_reply.started": "2022-02-12T08:10:48.011895Z"
    }
   },
   "outputs": [],
   "source": [
    "######################################################################\n",
    "############Experiment3 - PCA, SVD, Word embedding ###################\n",
    "######################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:05:00.96328Z",
     "iopub.status.busy": "2022-02-13T08:05:00.962861Z",
     "iopub.status.idle": "2022-02-13T08:05:04.479691Z",
     "shell.execute_reply": "2022-02-13T08:05:04.479088Z",
     "shell.execute_reply.started": "2022-02-13T08:05:00.963251Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA(n_components=500)"
      ]
     },
     "execution_count": 302,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# PCA\n",
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components = 500)\n",
    "pca.fit(train_x) \n",
    "# for dimension reduction, we fit only on the features and not labels unlike feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:05:09.301021Z",
     "iopub.status.busy": "2022-02-13T08:05:09.300569Z",
     "iopub.status.idle": "2022-02-13T08:05:09.589896Z",
     "shell.execute_reply": "2022-02-13T08:05:09.589302Z",
     "shell.execute_reply.started": "2022-02-13T08:05:09.300989Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 500), (1000, 500))"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x_pca = pca.transform(train_x)\n",
    "test_x_pca = pca.transform(test_x)\n",
    "train_x_pca.shape, test_x_pca.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:05:14.143062Z",
     "iopub.status.busy": "2022-02-13T08:05:14.1426Z",
     "iopub.status.idle": "2022-02-13T08:05:15.01394Z",
     "shell.execute_reply": "2022-02-13T08:05:15.013098Z",
     "shell.execute_reply.started": "2022-02-13T08:05:14.14303Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.57"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the svm classifier increased slightly due to pca and it took less time to train\n",
    "svm_classifier = svm.LinearSVC()\n",
    "svm_classifier.fit(train_x_pca, train_y)\n",
    "svm_predictions = svm_classifier.predict(test_x_pca)\n",
    "accuracy_svm = accuracy_score(test_y, svm_predictions)\n",
    "accuracy_svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:05:55.817724Z",
     "iopub.status.busy": "2022-02-13T08:05:55.81747Z",
     "iopub.status.idle": "2022-02-13T08:05:55.859649Z",
     "shell.execute_reply": "2022-02-13T08:05:55.858904Z",
     "shell.execute_reply.started": "2022-02-13T08:05:55.817697Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.573"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the gnb classifier increased slightly due to pca and it took less time to train\n",
    "gnb_classifier = GaussianNB()\n",
    "gnb_classifier.fit(train_x_pca, train_y)\n",
    "gnb_predictions = gnb_classifier.predict(test_x_pca)\n",
    "accuracy_gnb = accuracy_score(test_y, gnb_predictions)\n",
    "accuracy_gnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:06:08.079449Z",
     "iopub.status.busy": "2022-02-13T08:06:08.079186Z",
     "iopub.status.idle": "2022-02-13T08:06:08.908562Z",
     "shell.execute_reply": "2022-02-13T08:06:08.907699Z",
     "shell.execute_reply.started": "2022-02-13T08:06:08.079407Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.581"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the dectree classifier decreased slightly due to pca and it took less time to train\n",
    "dectree_classifier = DecisionTreeClassifier(max_depth=2)\n",
    "dectree_classifier.fit(train_x_pca, train_y)\n",
    "dectree_predictions = dectree_classifier.predict(test_x_pca)\n",
    "accuracy_dtree = accuracy_score(test_y, dectree_predictions)\n",
    "accuracy_dtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:06:32.403089Z",
     "iopub.status.busy": "2022-02-13T08:06:32.402827Z",
     "iopub.status.idle": "2022-02-13T08:06:34.990324Z",
     "shell.execute_reply": "2022-02-13T08:06:34.989631Z",
     "shell.execute_reply.started": "2022-02-13T08:06:32.40306Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.575"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the random forest classifier increased slightly due to pca and it took less time to train\n",
    "random_forest_classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "random_forest_classifier.fit(train_x_pca, train_y)\n",
    "rndforest_predictions = random_forest_classifier.predict(test_x_pca)\n",
    "accuracy_rndforest = accuracy_score(test_y, rndforest_predictions)\n",
    "accuracy_rndforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:07:57.587365Z",
     "iopub.status.busy": "2022-02-13T08:07:57.587072Z",
     "iopub.status.idle": "2022-02-13T08:08:04.085574Z",
     "shell.execute_reply": "2022-02-13T08:08:04.08496Z",
     "shell.execute_reply.started": "2022-02-13T08:07:57.587334Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TruncatedSVD(n_components=500, n_iter=10, random_state=42)"
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lsa/lsi - perform dimensionality reduction using svd\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "svd = TruncatedSVD(n_components=500, n_iter=10, random_state=42)\n",
    "svd.fit(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:08:07.501062Z",
     "iopub.status.busy": "2022-02-13T08:08:07.50084Z",
     "iopub.status.idle": "2022-02-13T08:08:07.708908Z",
     "shell.execute_reply": "2022-02-13T08:08:07.708294Z",
     "shell.execute_reply.started": "2022-02-13T08:08:07.50104Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 500), (1000, 500))"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x_svd = svd.transform(train_x)\n",
    "test_x_svd = svd.transform(test_x)\n",
    "train_x_svd.shape, test_x_svd.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:08:21.397733Z",
     "iopub.status.busy": "2022-02-13T08:08:21.396603Z",
     "iopub.status.idle": "2022-02-13T08:08:22.324696Z",
     "shell.execute_reply": "2022-02-13T08:08:22.324084Z",
     "shell.execute_reply.started": "2022-02-13T08:08:21.397657Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.582"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the svm classifier increased slightly due to svd and it took less time to train\n",
    "svm_classifier = svm.LinearSVC()\n",
    "svm_classifier.fit(train_x_svd, train_y)\n",
    "svm_predictions = svm_classifier.predict(test_x_svd)\n",
    "accuracy_svm = accuracy_score(test_y, svm_predictions)\n",
    "accuracy_svm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:09:02.283898Z",
     "iopub.status.busy": "2022-02-13T08:09:02.283655Z",
     "iopub.status.idle": "2022-02-13T08:09:02.324234Z",
     "shell.execute_reply": "2022-02-13T08:09:02.323316Z",
     "shell.execute_reply.started": "2022-02-13T08:09:02.283865Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.565"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the svm classifier increased slightly due to svd and it took less time to train\n",
    "gnb_classifier = GaussianNB()\n",
    "gnb_classifier.fit(train_x_svd, train_y)\n",
    "gnb_predictions = gnb_classifier.predict(test_x_svd)\n",
    "accuracy_gnb = accuracy_score(test_y, gnb_predictions)\n",
    "accuracy_gnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:09:45.735374Z",
     "iopub.status.busy": "2022-02-13T08:09:45.735144Z",
     "iopub.status.idle": "2022-02-13T08:09:46.353648Z",
     "shell.execute_reply": "2022-02-13T08:09:46.352854Z",
     "shell.execute_reply.started": "2022-02-13T08:09:45.735351Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.602"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the dectree classifier increased slightly due to svd and it took less time to train\n",
    "dectree_classifier = DecisionTreeClassifier(max_depth=2).fit(train_x_svd, train_y)\n",
    "dectree_predictions = dectree_classifier.predict(test_x_svd)\n",
    "accuracy_dtree = accuracy_score(test_y, dectree_predictions)\n",
    "accuracy_dtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T08:10:14.949934Z",
     "iopub.status.busy": "2022-02-13T08:10:14.949519Z",
     "iopub.status.idle": "2022-02-13T08:10:17.551606Z",
     "shell.execute_reply": "2022-02-13T08:10:17.550757Z",
     "shell.execute_reply.started": "2022-02-13T08:10:14.949891Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.585"
      ]
     },
     "execution_count": 313,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy of the random forest classifier increased slightly due to svd and it took less time to train\n",
    "random_forest_classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "random_forest_classifier.fit(train_x_svd, train_y)\n",
    "rndforest_predictions = random_forest_classifier.predict(test_x_svd)\n",
    "accuracy_rndforest = accuracy_score(test_y, rndforest_predictions)\n",
    "accuracy_rndforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "#############  Word Embedding ######################\n",
    "####################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:39:17.052983Z",
     "iopub.status.busy": "2022-02-13T09:39:17.052360Z",
     "iopub.status.idle": "2022-02-13T09:39:23.260400Z",
     "shell.execute_reply": "2022-02-13T09:39:23.259632Z",
     "shell.execute_reply.started": "2022-02-13T09:39:17.052888Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import feature_extraction\n",
    "from sklearn.metrics import accuracy_score\n",
    "import re\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:40:05.874752Z",
     "iopub.status.busy": "2022-02-13T09:40:05.874487Z",
     "iopub.status.idle": "2022-02-13T09:40:09.452559Z",
     "shell.execute_reply": "2022-02-13T09:40:09.451816Z",
     "shell.execute_reply.started": "2022-02-13T09:40:05.874723Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>This week , explained : spies , special counse...</td>\n",
       "      <td>The Russia investigation got real this week . ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Jolyon Palmer to leave Renault after Japanese ...</td>\n",
       "      <td>British driver Jolyon Palmer is to leave the R...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Former FBi Director James Comey Admits That He...</td>\n",
       "      <td>Liberals have been frothing at the mouth , wai...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Trump : ' I have n't changed my stance ' on China</td>\n",
       "      <td>President Trump said Tuesday that he has n't s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Trump Announces New Cabinet Member With Choice...</td>\n",
       "      <td>Newsmax reported Trump ’ s next move in the st...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Headline  \\\n",
       "0  This week , explained : spies , special counse...   \n",
       "1  Jolyon Palmer to leave Renault after Japanese ...   \n",
       "2  Former FBi Director James Comey Admits That He...   \n",
       "3  Trump : ' I have n't changed my stance ' on China   \n",
       "4  Trump Announces New Cabinet Member With Choice...   \n",
       "\n",
       "                                         articleBody  Label  \n",
       "0  The Russia investigation got real this week . ...      0  \n",
       "1  British driver Jolyon Palmer is to leave the R...      1  \n",
       "2  Liberals have been frothing at the mouth , wai...      0  \n",
       "3  President Trump said Tuesday that he has n't s...      1  \n",
       "4  Newsmax reported Trump ’ s next move in the st...      0  "
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset, test_dataset = load_dataset(dataset_name)\n",
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:40:10.852200Z",
     "iopub.status.busy": "2022-02-13T09:40:10.851800Z",
     "iopub.status.idle": "2022-02-13T09:40:10.858474Z",
     "shell.execute_reply": "2022-02-13T09:40:10.857579Z",
     "shell.execute_reply.started": "2022-02-13T09:40:10.852166Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((71420, 3), (6302, 3))"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.shape, test_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:40:12.229201Z",
     "iopub.status.busy": "2022-02-13T09:40:12.228766Z",
     "iopub.status.idle": "2022-02-13T09:40:12.270417Z",
     "shell.execute_reply": "2022-02-13T09:40:12.269586Z",
     "shell.execute_reply.started": "2022-02-13T09:40:12.229159Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.sample(n=5000, random_state=1)\n",
    "test_dataset = test_dataset.sample(n=1000, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:40:15.846966Z",
     "iopub.status.busy": "2022-02-13T09:40:15.846533Z",
     "iopub.status.idle": "2022-02-13T09:40:15.861148Z",
     "shell.execute_reply": "2022-02-13T09:40:15.860374Z",
     "shell.execute_reply.started": "2022-02-13T09:40:15.846928Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Preprocesses a string: Trimming, removing non-alphanumeric, lowercasing\n",
    "\"\"\"\n",
    "def preprocess(s):\n",
    "    return \" \".join(re.findall(r'\\w+', s, flags=re.UNICODE)).lower()\n",
    "\n",
    "\"\"\"\n",
    "    split the sentence into words using word tokenizer\n",
    "\"\"\"\n",
    "def word_tokenize(s):\n",
    "    return [t for t in nltk.word_tokenize(s)]\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "    The stopwords are a list of words that are very very common but don’t \n",
    "    provide useful information for most text analysis procedures. That is, \n",
    "    these words are ignored during most natural language processing tasks, \n",
    "    such as part-of-speech tagging, tokenization and parsing.\n",
    "    \n",
    "    While they are helpful for understanding the structure of sentences, they do \n",
    "    not help you understand the semantics of the sentences themselves.\n",
    "\"\"\"\n",
    "def remove_stopwords(l):\n",
    "    \"\"\"\n",
    "        Remove stopwords from the lemmatized list of words l\n",
    "    \"\"\"\n",
    "    return [w for w in l if w not in feature_extraction.text.ENGLISH_STOP_WORDS]\n",
    "\n",
    "\n",
    "def unigrams(line):\n",
    "    \"\"\"\n",
    "        Generate unigrams from a given string.\n",
    "        returns a list of unigrams\n",
    "    \"\"\"\n",
    "    line = preprocess(line)\n",
    "    tokenized_line = word_tokenize(line)\n",
    "    tokenized_line_wo_stopwords = remove_stopwords(tokenized_line)\n",
    "    return tokenized_line_wo_stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:40:17.487202Z",
     "iopub.status.busy": "2022-02-13T09:40:17.486777Z",
     "iopub.status.idle": "2022-02-13T09:40:17.725390Z",
     "shell.execute_reply": "2022-02-13T09:40:17.724674Z",
     "shell.execute_reply.started": "2022-02-13T09:40:17.487162Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/pradnesh/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/pradnesh/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:40:18.870930Z",
     "iopub.status.busy": "2022-02-13T09:40:18.870660Z",
     "iopub.status.idle": "2022-02-13T09:41:54.944667Z",
     "shell.execute_reply": "2022-02-13T09:41:54.943832Z",
     "shell.execute_reply.started": "2022-02-13T09:40:18.870901Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "headline unigrams train generated\n",
      "body unigrams train generated\n",
      "headline unigrams test generated\n",
      "body unigrams test generated\n"
     ]
    }
   ],
   "source": [
    "# generate unigrams\n",
    "train_dataset[\"Headline_unigrams\"] = train_dataset[\"Headline\"].map(lambda x: unigrams(x))\n",
    "print(\"headline unigrams train generated\")\n",
    "train_dataset[\"articleBody_unigrams\"] = train_dataset[\"articleBody\"].map(lambda x: unigrams(x))\n",
    "print(\"body unigrams train generated\")\n",
    "test_dataset[\"Headline_unigrams\"] = test_dataset[\"Headline\"].map(lambda x: unigrams(x))\n",
    "print(\"headline unigrams test generated\")\n",
    "test_dataset[\"articleBody_unigrams\"] = test_dataset[\"articleBody\"].map(lambda x: unigrams(x))\n",
    "print(\"body unigrams test generated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:42:02.904024Z",
     "iopub.status.busy": "2022-02-13T09:42:02.903426Z",
     "iopub.status.idle": "2022-02-13T09:42:02.908342Z",
     "shell.execute_reply": "2022-02-13T09:42:02.907516Z",
     "shell.execute_reply.started": "2022-02-13T09:42:02.903987Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Join the unigrams of headline and body as a single string\n",
    "\"\"\"\n",
    "def combine_headline_body_unigrams(x):\n",
    "    res = '%s %s' % (' '.join(x['Headline_unigrams']), ' '.join(x['articleBody_unigrams']))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:42:05.200097Z",
     "iopub.status.busy": "2022-02-13T09:42:05.199844Z",
     "iopub.status.idle": "2022-02-13T09:42:05.675806Z",
     "shell.execute_reply": "2022-02-13T09:42:05.674999Z",
     "shell.execute_reply.started": "2022-02-13T09:42:05.200069Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset[\"headline_plus_body\"] = train_dataset.apply(combine_headline_body_unigrams, axis=1)\n",
    "test_dataset[\"headline_plus_body\"] = test_dataset.apply(combine_headline_body_unigrams, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:42:53.886094Z",
     "iopub.status.busy": "2022-02-13T09:42:53.885830Z",
     "iopub.status.idle": "2022-02-13T09:42:54.352660Z",
     "shell.execute_reply": "2022-02-13T09:42:54.351948Z",
     "shell.execute_reply.started": "2022-02-13T09:42:53.886066Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "672"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# saving RAM\n",
    "import gc #garbage collector\n",
    "\n",
    "train_dataset.drop(['Headline_bigrams', 'articleBody_bigrams', 'Headline_unigrams', 'articleBody_unigrams', 'Headline', 'articleBody'], axis=1, errors='ignore', inplace=True)\n",
    "test_dataset.drop(['Headline_bigrams', 'articleBody_bigrams', 'Headline_unigrams', 'articleBody_unigrams', 'Headline', 'articleBody'], axis=1, errors='ignore', inplace=True)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:43:10.430205Z",
     "iopub.status.busy": "2022-02-13T09:43:10.429779Z",
     "iopub.status.idle": "2022-02-13T09:43:10.446619Z",
     "shell.execute_reply": "2022-02-13T09:43:10.445549Z",
     "shell.execute_reply.started": "2022-02-13T09:43:10.430164Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>headline_plus_body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49255</th>\n",
       "      <td>1</td>\n",
       "      <td>21k illegal aliens arrested trump far accordin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55535</th>\n",
       "      <td>1</td>\n",
       "      <td>university wisconsin passes dangerous new poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6633</th>\n",
       "      <td>1</td>\n",
       "      <td>diversity officially recognized fact checkers ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68666</th>\n",
       "      <td>0</td>\n",
       "      <td>confirmed syrian arab army takes control jorda...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9359</th>\n",
       "      <td>1</td>\n",
       "      <td>jets seeing new muhammad wilkerson according d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Label                                 headline_plus_body\n",
       "49255      1  21k illegal aliens arrested trump far accordin...\n",
       "55535      1  university wisconsin passes dangerous new poli...\n",
       "6633       1  diversity officially recognized fact checkers ...\n",
       "68666      0  confirmed syrian arab army takes control jorda...\n",
       "9359       1  jets seeing new muhammad wilkerson according d..."
      ]
     },
     "execution_count": 325,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:45:44.882049Z",
     "iopub.status.busy": "2022-02-13T09:45:44.881480Z",
     "iopub.status.idle": "2022-02-13T09:45:44.889059Z",
     "shell.execute_reply": "2022-02-13T09:45:44.887866Z",
     "shell.execute_reply.started": "2022-02-13T09:45:44.882008Z"
    }
   },
   "outputs": [],
   "source": [
    "def read_glove_vecs(glove_file):\n",
    "    with open(glove_file, 'r') as f:\n",
    "        word_to_vec_map = {}\n",
    "        for line in f:\n",
    "            line = line.strip().split()\n",
    "            curr_word = line[0]\n",
    "            word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64)\n",
    "            \n",
    "    return word_to_vec_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T09:45:49.567090Z",
     "iopub.status.busy": "2022-02-13T09:45:49.566827Z",
     "iopub.status.idle": "2022-02-13T09:46:26.260958Z",
     "shell.execute_reply": "2022-02-13T09:46:26.260269Z",
     "shell.execute_reply.started": "2022-02-13T09:45:49.567062Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_to_vec_map = read_glove_vecs('./Data-Set/glove.6B.300d.txt')\n",
    "type(word_to_vec_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:10:16.445681Z",
     "iopub.status.busy": "2022-02-13T10:10:16.445424Z",
     "iopub.status.idle": "2022-02-13T10:10:16.452260Z",
     "shell.execute_reply": "2022-02-13T10:10:16.451515Z",
     "shell.execute_reply.started": "2022-02-13T10:10:16.445652Z"
    }
   },
   "outputs": [],
   "source": [
    "def sentence_to_avg(sentence):\n",
    "    \"\"\"\n",
    "        sentence is already preprocessed\n",
    "        we represent the sentence by taking the average of its words' embeddings\n",
    "    \"\"\"\n",
    "    words = sentence.split()\n",
    "    avg = np.zeros(300)\n",
    "    count = 0\n",
    "    \n",
    "    for w in words:\n",
    "        if w in word_to_vec_map.keys():\n",
    "            count += 1\n",
    "            avg += word_to_vec_map[w]\n",
    "            \n",
    "    if count > 0:\n",
    "        avg = avg / count\n",
    "        \n",
    "    return avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:02:17.202581Z",
     "iopub.status.busy": "2022-02-13T10:02:17.202020Z",
     "iopub.status.idle": "2022-02-13T10:02:17.208992Z",
     "shell.execute_reply": "2022-02-13T10:02:17.208047Z",
     "shell.execute_reply.started": "2022-02-13T10:02:17.202543Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>headline_plus_body</th>\n",
       "      <th>sentence embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49255</th>\n",
       "      <td>1</td>\n",
       "      <td>21k illegal aliens arrested trump far accordin...</td>\n",
       "      <td>[-0.017819887739463573, 0.009724260919540227, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55535</th>\n",
       "      <td>1</td>\n",
       "      <td>university wisconsin passes dangerous new poli...</td>\n",
       "      <td>[-0.001058549019607837, -0.06027169187675072, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6633</th>\n",
       "      <td>1</td>\n",
       "      <td>diversity officially recognized fact checkers ...</td>\n",
       "      <td>[-0.07587987692307689, 0.09218232347140036, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68666</th>\n",
       "      <td>0</td>\n",
       "      <td>confirmed syrian arab army takes control jorda...</td>\n",
       "      <td>[0.04090239049999999, -0.0015157960000000124, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9359</th>\n",
       "      <td>1</td>\n",
       "      <td>jets seeing new muhammad wilkerson according d...</td>\n",
       "      <td>[-0.030955657575757575, 0.07464621212121214, 0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Label                                 headline_plus_body  \\\n",
       "49255      1  21k illegal aliens arrested trump far accordin...   \n",
       "55535      1  university wisconsin passes dangerous new poli...   \n",
       "6633       1  diversity officially recognized fact checkers ...   \n",
       "68666      0  confirmed syrian arab army takes control jorda...   \n",
       "9359       1  jets seeing new muhammad wilkerson according d...   \n",
       "\n",
       "                                      sentence embedding  \n",
       "49255  [-0.017819887739463573, 0.009724260919540227, ...  \n",
       "55535  [-0.001058549019607837, -0.06027169187675072, ...  \n",
       "6633   [-0.07587987692307689, 0.09218232347140036, 0....  \n",
       "68666  [0.04090239049999999, -0.0015157960000000124, ...  \n",
       "9359   [-0.030955657575757575, 0.07464621212121214, 0...  "
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[\"sentence embedding\"] = train_dataset[\"headline_plus_body\"].map(lambda x: sentence_to_avg(x))\n",
    "test_dataset[\"sentence embedding\"] = test_dataset[\"headline_plus_body\"].map(lambda x: sentence_to_avg(x))\n",
    "\n",
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:10:24.021638Z",
     "iopub.status.busy": "2022-02-13T10:10:24.021023Z",
     "iopub.status.idle": "2022-02-13T10:10:24.028831Z",
     "shell.execute_reply": "2022-02-13T10:10:24.027712Z",
     "shell.execute_reply.started": "2022-02-13T10:10:24.021595Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 300), (1000, 300))"
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x = train_dataset[\"sentence embedding\"].tolist()\n",
    "train_x = np.asarray(train_x)\n",
    "\n",
    "test_x = test_dataset[\"sentence embedding\"].tolist()\n",
    "test_x = np.asarray(test_x)\n",
    "\n",
    "train_x.shape, test_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:07:09.023221Z",
     "iopub.status.busy": "2022-02-13T10:07:09.022702Z",
     "iopub.status.idle": "2022-02-13T10:07:09.029537Z",
     "shell.execute_reply": "2022-02-13T10:07:09.028646Z",
     "shell.execute_reply.started": "2022-02-13T10:07:09.023186Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000,), (1000,))"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y = train_dataset[\"Label\"].values\n",
    "test_y = test_dataset[\"Label\"].values\n",
    "\n",
    "train_y.shape, test_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:07:12.353205Z",
     "iopub.status.busy": "2022-02-13T10:07:12.352411Z",
     "iopub.status.idle": "2022-02-13T10:07:12.361026Z",
     "shell.execute_reply": "2022-02-13T10:07:12.360307Z",
     "shell.execute_reply.started": "2022-02-13T10:07:12.353167Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we no longer require train_dataset and test_dataset, thus we can free up some RAM\n",
    "import gc\n",
    "\n",
    "del train_dataset, test_dataset\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:16:54.810050Z",
     "iopub.status.busy": "2022-02-13T10:16:54.809782Z",
     "iopub.status.idle": "2022-02-13T10:16:57.001304Z",
     "shell.execute_reply": "2022-02-13T10:16:57.000555Z",
     "shell.execute_reply.started": "2022-02-13T10:16:54.810021Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC()"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "svm_classifier = svm.LinearSVC()\n",
    "svm_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:16:59.047530Z",
     "iopub.status.busy": "2022-02-13T10:16:59.046904Z",
     "iopub.status.idle": "2022-02-13T10:16:59.062337Z",
     "shell.execute_reply": "2022-02-13T10:16:59.061619Z",
     "shell.execute_reply.started": "2022-02-13T10:16:59.047345Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.482"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_predictions = svm_classifier.predict(test_x)\n",
    "accuracy_svm = accuracy_score(test_y, svm_predictions)\n",
    "accuracy_svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:17:01.647065Z",
     "iopub.status.busy": "2022-02-13T10:17:01.646485Z",
     "iopub.status.idle": "2022-02-13T10:17:01.651885Z",
     "shell.execute_reply": "2022-02-13T10:17:01.650668Z",
     "shell.execute_reply.started": "2022-02-13T10:17:01.647026Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB()"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb_classifier = GaussianNB()\n",
    "gnb_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:17:02.275896Z",
     "iopub.status.busy": "2022-02-13T10:17:02.275401Z",
     "iopub.status.idle": "2022-02-13T10:17:02.280520Z",
     "shell.execute_reply": "2022-02-13T10:17:02.279727Z",
     "shell.execute_reply.started": "2022-02-13T10:17:02.275855Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.62"
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnb_predictions = gnb_classifier.predict(test_x)\n",
    "accuracy_gnb = accuracy_score(test_y, gnb_predictions)\n",
    "accuracy_gnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:17:03.022716Z",
     "iopub.status.busy": "2022-02-13T10:17:03.022199Z",
     "iopub.status.idle": "2022-02-13T10:17:03.031213Z",
     "shell.execute_reply": "2022-02-13T10:17:03.030428Z",
     "shell.execute_reply.started": "2022-02-13T10:17:03.022673Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(max_depth=2)"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The accuracy of DecisionTree\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dectree_classifier = DecisionTreeClassifier(max_depth=2)\n",
    "dectree_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:17:04.152950Z",
     "iopub.status.busy": "2022-02-13T10:17:04.152374Z",
     "iopub.status.idle": "2022-02-13T10:17:05.449730Z",
     "shell.execute_reply": "2022-02-13T10:17:05.449003Z",
     "shell.execute_reply.started": "2022-02-13T10:17:04.152911Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.549"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dectree_predictions = dectree_classifier.predict(test_x)\n",
    "accuracy_dtree = accuracy_score(test_y, dectree_predictions)\n",
    "accuracy_dtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-13T10:17:06.652144Z",
     "iopub.status.busy": "2022-02-13T10:17:06.651866Z",
     "iopub.status.idle": "2022-02-13T10:18:30.042611Z",
     "shell.execute_reply": "2022-02-13T10:18:30.041825Z",
     "shell.execute_reply.started": "2022-02-13T10:17:06.652095Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(max_depth=2, random_state=0)"
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "random_forest_classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "random_forest_classifier.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.604"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rndforest_predictions = random_forest_classifier.predict(test_x)\n",
    "accuracy_rndforest = accuracy_score(test_y, rndforest_predictions)\n",
    "accuracy_rndforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}