Logistic_Regression_on_sampled_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Logistic Regression on sampled data.ipynb",
      "provenance": [],
      "mount_file_id": "14UGW8HzpQaVx_aHdLy362N8i-hhGN7px",
      "authorship_tag": "ABX9TyPYP6R+o4PbYHPK5Tlz5Yly",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Aanisha/ACL_Abusive_Tamil_Comment_Classification/blob/main/Logistic_Regression_on_sampled_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Logistic Regression on the dataset\n",
        "\n",
        "The experiment here uses the data after the oversampling and under-sampling of the data."
      ],
      "metadata": {
        "id": "w946rFQWXhNe"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2zUtblVESanq",
        "outputId": "dd66ac10-f90a-4fa4-8dd2-dea021f122b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting indic-nlp-library\n",
            "  Downloading indic_nlp_library-0.81-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 4.8 MB/s \n",
            "\u001b[?25hCollecting morfessor\n",
            "  Downloading Morfessor-2.0.6-py3-none-any.whl (35 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from indic-nlp-library) (1.3.5)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from indic-nlp-library) (1.21.5)\n",
            "Collecting sphinx-rtd-theme\n",
            "  Downloading sphinx_rtd_theme-1.0.0-py2.py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 28.2 MB/s \n",
            "\u001b[?25hCollecting sphinx-argparse\n",
            "  Downloading sphinx_argparse-0.3.1-py2.py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->indic-nlp-library) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->indic-nlp-library) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->indic-nlp-library) (1.15.0)\n",
            "Requirement already satisfied: sphinx>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx-argparse->indic-nlp-library) (1.8.6)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.3.0)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.2.0)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.6.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (21.3)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.7.12)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.9.1)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.23.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (57.4.0)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.11.3)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.4)\n",
            "Requirement already satisfied: docutils<0.18,>=0.11 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.17.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2021.10.8)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.7)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.1.5)\n",
            "Installing collected packages: sphinx-rtd-theme, sphinx-argparse, morfessor, indic-nlp-library\n",
            "Successfully installed indic-nlp-library-0.81 morfessor-2.0.6 sphinx-argparse-0.3.1 sphinx-rtd-theme-1.0.0\n"
          ]
        }
      ],
      "source": [
        "# Downloading library\n",
        "\n",
        "!pip install indic-nlp-library"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd, numpy as np\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer"
      ],
      "metadata": {
        "id": "CoyLxBvjTCli"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Preprocessing the data"
      ],
      "metadata": {
        "id": "VVkflqT8bRdj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train = pd.read_csv('/content/Tamil_train_data.csv')\n",
        "test = pd.read_csv('/content/Tamil_test_data.csv')\n",
        "valid = pd.read_csv('/content/Tamil_valid_data.csv')"
      ],
      "metadata": {
        "id": "egDs_caZTY8q"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train = train[train.tag != 'Not-Tamil']"
      ],
      "metadata": {
        "id": "cdjAnJzmTyHI"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tags = {\"tag\":     {'Hope-Speech':0, 'None-of-the-above':7, 'Homophobia':1, 'Misandry':2,\n",
        "       'Counter-speech':3, 'Misogyny':4, 'Xenophobia':5, 'Transphobic':6}}"
      ],
      "metadata": {
        "id": "6gxDAUsAUNnm"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train = train.replace(tags)\n",
        "valid = valid.replace(tags)\n",
        "test = test.replace(tags)"
      ],
      "metadata": {
        "id": "RuE7vMtFUzkl"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train = pd.concat([train,valid],axis=0)"
      ],
      "metadata": {
        "id": "qXEcRozRVmT_"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hY6Lh9snVtUk",
        "outputId": "95c12cb2-1736-4f18-f85c-e0db8124397f"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10227, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train = train.drop(train[train['tag'] == 7].sample(frac=0.4).index)"
      ],
      "metadata": {
        "id": "kJrc_HuytwFe"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels = pd.read_csv(\"/content/Tamil_test_labels_data.csv\")\n",
        "\n",
        "test_labels = test_labels.replace(tags)\n",
        "test_labels = pd.merge(test_labels, test, on=['comments'])\n",
        "test_labels = test_labels.dropna()"
      ],
      "metadata": {
        "id": "X8LFjK5iyUH1"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gt = []\n",
        "co = []\n",
        "for com in range(len(list(test_labels['comments']))):\n",
        "  if test_labels['comments'][com] in list(test['comments']):\n",
        "\n",
        "    gt.append(test_labels['tag'][com])\n",
        "    co.append(test_labels['comments'][com])"
      ],
      "metadata": {
        "id": "f9J99E3IyZf_"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(co)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "701vIHx4yjCF",
        "outputId": "251a9b24-ebad-4c4a-a109-5cc73f3e4f46"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2555"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def oversample(df):\n",
        "    classes = [4,0,1,6,5]\n",
        "    most = 250\n",
        "    classes_list = []\n",
        "    for key in classes:\n",
        "        classes_list.append(df[df['tag'] == key]) \n",
        "    classes_sample = []\n",
        "    for i in range(len(classes_list)):\n",
        "        classes_sample.append(classes_list[i].sample(most, replace=True))\n",
        "    df_maybe = pd.concat(classes_sample)\n",
        "    final_df = pd.concat([df_maybe,df], axis=0)\n",
        "    final_df = final_df.reset_index(drop=True)\n",
        "    return final_df"
      ],
      "metadata": {
        "id": "vEKU-mQct2Vy"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train = oversample(train)"
      ],
      "metadata": {
        "id": "Jrl2mv3ot7YK"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Tokenization of the train data"
      ],
      "metadata": {
        "id": "rSnr5z6MbH1o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re, string\n",
        "from indicnlp.tokenize import indic_tokenize\n",
        "def tokenize(s): return indic_tokenize.trivial_tokenize(s)"
      ],
      "metadata": {
        "id": "ezKOgeFwU3gy"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n = train.shape[0]\n",
        "\n",
        "vec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,\n",
        "                      strip_accents='unicode', use_idf=1,\n",
        "               smooth_idf=1, sublinear_tf=1)\n",
        "\n",
        "\n",
        "trn_term_doc = vec.fit_transform(train['comments'])\n",
        "test_term_doc = vec.transform(co)"
      ],
      "metadata": {
        "id": "7pysTr8JVHj8"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Training the model on the training data"
      ],
      "metadata": {
        "id": "OqkZ_XE2a6he"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "label_cols = ['tag']"
      ],
      "metadata": {
        "id": "qzCweAheXmi6"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = trn_term_doc\n",
        "test_x = test_term_doc"
      ],
      "metadata": {
        "id": "GvxdUq6JVaLN"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def pr(y_i, y):\n",
        "    p = x[y==y_i].sum(0)\n",
        "    return (p+1) / ((y==y_i).sum()+1)"
      ],
      "metadata": {
        "id": "oQEV7lVzVSzL"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.svm as svm\n",
        "import sklearn.ensemble\n",
        "\n",
        "def get_mdl(y):\n",
        "    y = y.values\n",
        "    r = np.log(pr(1,y) / pr(0,y))\n",
        "    m = LogisticRegression(solver='newton-cg')\n",
        "    x_nb = x.multiply(r)\n",
        "    return m.fit(x_nb, y), r"
      ],
      "metadata": {
        "id": "R9d49XB_WiDm"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "preds = np.zeros((len(test), len(label_cols)))\n",
        "\n",
        "for i, j in enumerate(label_cols):\n",
        "    print('fit', j)\n",
        "    m,r = get_mdl(train[j])\n",
        "    preds = m.predict(test_x.multiply(r))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ONkHmHTuWk_f",
        "outputId": "91792052-f9eb-4dd4-af1c-67f6d5e2b0f4"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fit tag\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(preds)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Srov1HIuu5jp",
        "outputId": "77ba79e3-065b-4922-ee8e-9eb6f8d1c32f"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2555"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Preparing the test data"
      ],
      "metadata": {
        "id": "fv9cbwAeanmg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels = pd.read_csv(\"/content/Tamil_test_labels_data.csv\")"
      ],
      "metadata": {
        "id": "Rwy8XaVPXpDn"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g3r5dd-dbixZ",
        "outputId": "ed99d7f3-2e0d-48af-a0ee-5727e0a9cc68"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2559, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels = test_labels.replace(tags)"
      ],
      "metadata": {
        "id": "5av1jK3GX04z"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels = test_labels.dropna()"
      ],
      "metadata": {
        "id": "N4A_xXtRj4Qv"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rgci5vyzbpyJ",
        "outputId": "27094e89-ee91-4ac3-8f5e-ed7306a7cc15"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2556, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Testing the model using unseen test data"
      ],
      "metadata": {
        "id": "Zg8TuNq-atqE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn\n",
        "\n",
        "print(sklearn.metrics.classification_report(gt, preds))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fFyPVQ5NYQDR",
        "outputId": "bef50f7e-6143-48d3-f431-61b75c7a0379"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.26      0.16      0.20        95\n",
            "           1       0.51      0.33      0.40        64\n",
            "           2       0.60      0.47      0.53       419\n",
            "           3       0.50      0.03      0.06       135\n",
            "           4       0.57      0.08      0.13       105\n",
            "           5       0.67      0.02      0.03       120\n",
            "           6       0.20      0.05      0.08        60\n",
            "           7       0.67      0.91      0.77      1557\n",
            "\n",
            "    accuracy                           0.65      2555\n",
            "   macro avg       0.50      0.25      0.28      2555\n",
            "weighted avg       0.62      0.65      0.59      2555\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E0ZBmymLYUfH",
        "outputId": "368f2869-6535-42d7-9db6-a11401b43a39"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2556, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    }
  ]
}