MuRIL_on_original_dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MuRIL on original dataset.ipynb",
      "provenance": [],
      "mount_file_id": "1HjF_-_B1d3YYd0JGM9u0MIYsZXTtxf16",
      "authorship_tag": "ABX9TyOx7xbcVxS28rK6TLJe3p6K",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Aanisha/ACL_Abusive_Tamil_Comment_Classification/blob/main/MuRIL_on_original_dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Finetuned MuRIL on original dataset"
      ],
      "metadata": {
        "id": "WQoMJ7C96Ykg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1X8zu4eDFwyi",
        "outputId": "2b8ddfcb-5bc3-4c97-bf49-9b12ba405979"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 33.8 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 41.8 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
            "\u001b[K     |████████████████████████████████| 67 kB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers!=0.11.3,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 34.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.4.0 pyyaml-6.0 sacremoses-0.0.47 tokenizers-0.11.6 transformers-4.17.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "opCmOH5RuUGK"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import os\n",
        "import os\n",
        "import random\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm.auto import tqdm\n",
        "\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import TFAutoModel\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "import tensorflow.keras.backend as K\n",
        "import logging"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Pre-processing the data"
      ],
      "metadata": {
        "id": "w9r0tgEz7arD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train = pd.read_csv('/content/Tamil_train_data.csv')\n",
        "test = pd.read_csv('/content/Tamil_test_data.csv')\n",
        "valid = pd.read_csv('/content/Tamil_valid_data.csv')"
      ],
      "metadata": {
        "id": "dXRloJNEFCa3"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tags = {\"tag\":     {'Hope-Speech':0, 'None-of-the-above':7, 'Homophobia':1, 'Misandry':2,\n",
        "       'Counter-speech':3, 'Misogyny':4, 'Xenophobia':5, 'Transphobic':6}}"
      ],
      "metadata": {
        "id": "CqHSylt_jBL8"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test = test.replace(tags)"
      ],
      "metadata": {
        "id": "gsi743NljGgo"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Loading the model"
      ],
      "metadata": {
        "id": "cjAx5NQP7g3Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "config = {\n",
        "    \n",
        "    'seed' : 42,\n",
        "    'model': '/content/drive/MyDrive/Muril-base-cased',\n",
        "    'group': 'MURIL',\n",
        "    \n",
        "    'batch_size': 16,\n",
        "    'max_length': 64,\n",
        "    \n",
        "    'device' : 'GPU',\n",
        "    'epochs' : 2,\n",
        "\n",
        "    'test_size' : 0.1,\n",
        "    'lr': 5e-6,\n",
        "    'use_transfer_learning' : False,\n",
        "    \n",
        "    'use_wandb': True,\n",
        "    'wandb_mode' : 'online',\n",
        "}"
      ],
      "metadata": {
        "id": "azcZlZ6Xgefm"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_keras_model():\n",
        "    pretrained_model = TFAutoModel.from_pretrained(config['model'])\n",
        "    \n",
        "    input_ids = layers.Input(shape=(config['max_length']),\n",
        "                             name='input_ids', \n",
        "                             dtype=tf.int32)\n",
        "    token_type_ids = layers.Input(shape=(config['max_length'],),\n",
        "                                  name='token_type_ids', \n",
        "                                  dtype=tf.int32)\n",
        "    attention_mask = layers.Input(shape=(config['max_length'],),\n",
        "                                  name='attention_mask', \n",
        "                                  dtype=tf.int32)\n",
        "    embedding = pretrained_model(input_ids, \n",
        "                     token_type_ids=token_type_ids, \n",
        "                     attention_mask=attention_mask)[0]\n",
        "\n",
        "   \n",
        "\n",
        "    x1 = tf.keras.layers.Dropout(0.2)(embedding) \n",
        "    x1 = tf.keras.layers.Conv1D(1,1)(x1)\n",
        "    x1 = tf.keras.layers.Flatten()(x1)\n",
        "    x1 = tf.keras.layers.Dense(8, activation='softmax')(x1)\n",
        "   \n",
        "    #print(x1.shape)\n",
        "    \n",
        "    model = keras.Model(inputs=[input_ids, \n",
        "                                token_type_ids, \n",
        "                                attention_mask],\n",
        "                        outputs=x1)\n",
        "    \n",
        "    return model"
      ],
      "metadata": {
        "id": "xN1GUPO_hZip"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = get_keras_model()\n",
        "\n",
        "model.load_weights('/content/drive/MyDrive/Muril-base-cased/best_model_25.h5')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UWFKRjoPhr2m",
        "outputId": "49d68b7f-b1de-4134-c80a-0293ec5feaa8"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at /content/drive/MyDrive/Muril-base-cased were not used when initializing TFBertModel: ['mlm___cls']\n",
            "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some layers of TFBertModel were not initialized from the model checkpoint at /content/drive/MyDrive/Muril-base-cased and are newly initialized: ['bert/pooler/dense/bias:0', 'bert/pooler/dense/kernel:0']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Preparing test data"
      ],
      "metadata": {
        "id": "SvBOHBu57lpW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels = pd.read_csv(\"/content/Tamil_test_labels_data.csv\")\n",
        "\n",
        "test_labels = test_labels.replace(tags)\n",
        "test_labels = pd.merge(test_labels, test, on=['comments'])\n",
        "test_labels = test_labels.dropna()"
      ],
      "metadata": {
        "id": "Fi5SIKJpix8T"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ivSqXwE-xlzl",
        "outputId": "46a0b5ff-99d4-4d35-feab-871a7daedd19"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2555, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(config['model'])"
      ],
      "metadata": {
        "id": "3zsa9PJgjiyX"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_test = tokenizer(\n",
        "    text=test_labels.comments.tolist(),\n",
        "    add_special_tokens=True,\n",
        "    max_length = 64,\n",
        "    padding='max_length',\n",
        "    truncation=True, \n",
        "    return_tensors='tf',\n",
        "    return_token_type_ids = True,\n",
        "    return_attention_mask = True,\n",
        "    verbose = True)"
      ],
      "metadata": {
        "id": "ChTX90K7jSIn"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(x_test['input_ids'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qfa4OZZ5xuSY",
        "outputId": "be0649a4-0d62-4e6d-a89f-516dff7279ed"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2555"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preds = model.predict(x = {'input_ids':x_test['input_ids'], \n",
        "                    'token_type_ids':x_test['token_type_ids'], \n",
        "                    'attention_mask': x_test['attention_mask']}, verbose = 1, workers=4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YuhZ-oN3h3jQ",
        "outputId": "f4384c97-5252-412f-e597-597ecb61def4"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "80/80 [==============================] - 34s 288ms/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pr = []\n",
        "for p in preds:\n",
        "  pr.append(np.argmax(p))"
      ],
      "metadata": {
        "id": "qxJPqk7tnXmH"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Testing the model on unseen test data"
      ],
      "metadata": {
        "id": "Xj5XmQ2N7spm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn\n",
        "import pandas as pd, numpy as np\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "print(sklearn.metrics.classification_report(test_labels['tag'], pr))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "73Q9Zm8dmkip",
        "outputId": "a89a26dc-cc32-4c9d-ca1b-c1e7a41fac14"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.00      0.00      0.00        95\n",
            "           1       0.00      0.00      0.00        64\n",
            "           2       0.52      0.68      0.59       419\n",
            "           3       0.25      0.31      0.28       135\n",
            "           4       0.00      0.00      0.00       105\n",
            "           5       0.72      0.11      0.19       120\n",
            "           6       0.00      0.00      0.00        60\n",
            "           7       0.76      0.89      0.82      1557\n",
            "\n",
            "    accuracy                           0.68      2555\n",
            "   macro avg       0.28      0.25      0.23      2555\n",
            "weighted avg       0.60      0.68      0.62      2555\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "mmG0nfFjm5XA"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}