text_classification/sms_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMS Classification\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.455315Z",
     "start_time": "2020-08-28T00:21:47.681178Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from transformers import BertForSequenceClassification, BertTokenizer, TrainingArguments, Trainer\n",
    "from nlp import load_dataset, Dataset\n",
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.489767Z",
     "start_time": "2020-08-28T00:21:49.456750Z"
    }
   },
   "outputs": [],
   "source": [
    "sms = pd.read_csv(\"../corpus/sg_sms_corpus_en.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.497358Z",
     "start_time": "2020-08-28T00:21:49.491155Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48092, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.507885Z",
     "start_time": "2020-08-28T00:21:49.498437Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Bugis oso near wat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>I dunno until when... Lets go learn pilates...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Den only weekdays got special price... Haiz......</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Meet after lunch la...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                                               text\n",
       "0        0                              Bugis oso near wat...\n",
       "1        0  Go until jurong point, crazy.. Available only ...\n",
       "2        0     I dunno until when... Lets go learn pilates...\n",
       "3        0  Den only weekdays got special price... Haiz......\n",
       "4        0                             Meet after lunch la..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.516856Z",
     "start_time": "2020-08-28T00:21:49.509622Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "199"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# unique user_id\n",
    "sms.user_id.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.526814Z",
     "start_time": "2020-08-28T00:21:49.520230Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "129    4706\n",
       "9      4683\n",
       "105    3501\n",
       "102    2548\n",
       "126    1936\n",
       "       ... \n",
       "152      10\n",
       "140      10\n",
       "172      10\n",
       "183      10\n",
       "191      10\n",
       "Name: user_id, Length: 199, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms.user_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.532450Z",
     "start_time": "2020-08-28T00:21:49.527989Z"
    }
   },
   "outputs": [],
   "source": [
    "# filter for the top 3 users for classification\n",
    "sms = sms[sms.user_id.isin([9, 105, 129])].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.545002Z",
     "start_time": "2020-08-28T00:21:49.534971Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> User id : 9\n",
      "Look c... Buying but not so soon...\n",
      "m always like tis...pain u see..worst den monthly cramps!haha...\n",
      "What are you doing now ?\n",
      "Hey you on your way?\n",
      "u r more naggin thn my mum... ok, but i still hav to do my proj rite. how i wish i can go home now.. thn u goin alone ah? Ur bil1 not goin wf u?\n",
      "\n",
      ">>> User id : 105\n",
      "See can find people anot first eh.\n",
      "Haha sorryi just saw yourmessage.\n",
      "Okay  <#> pm my house.\n",
      "Hahaha still havent reach home.\n",
      "No carrot cake. Bought satay beehoon for you.\n",
      "\n",
      ">>> User id : 129\n",
      "Ohh okay!:P hahaha yeah the bride quite chio! Yes omg I watch untilthere liao yucks. Disgusting! That guy tasted it!!!! Yucks!!!!!!  Ohhyeah I kena taupok by shahid before, forgot who else! It was err veryrandom ah, during council chalet! Chris they all also taupok-ed me!Hahaha ooo! Tuna??\n",
      "Haha yup^^ sian I just got your msg! Haiz rest well my dear! (: yeapcan't wait!!!\n",
      "It's nice!!\n",
      "He's a very very weird guy, abit embarrassing:/ hahaha yeah youdefinitely won't recognise any of them le! Haha okay have fun and takecare): tonight sleep early! Haha ooo icic! The food is cool! Hahaha:P\n",
      "Oh my lucky!! >< psps! Yeap I enjoyed my shower very much, that's whyonly now then finish hehe:P home yet?\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# sample 5 rows for each user, can you spot the difference?\n",
    "for user_id in sms.user_id.unique():\n",
    "    print(f\">>> User id : {user_id}\")\n",
    "    print(\"\\n\".join(sms.loc[sms.user_id == user_id, \"text\"].sample(5, random_state=0).tolist()))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems like the 3rd user is more distinctive with longer sentence and words, and more usage of exclamation mark."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.550696Z",
     "start_time": "2020-08-28T00:21:49.546812Z"
    }
   },
   "outputs": [],
   "source": [
    "# remap user_id\n",
    "sms.user_id.replace([9, 105, 129], [0, 1, 2], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.562220Z",
     "start_time": "2020-08-28T00:21:49.552686Z"
    }
   },
   "outputs": [],
   "source": [
    "# stratified sampling for train-test split\n",
    "sms_train = pd.DataFrame()\n",
    "for user_id in sms.user_id.unique():\n",
    "    sms_train = pd.concat([sms_train, sms[sms.user_id == user_id].sample(frac=0.6)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.566140Z",
     "start_time": "2020-08-28T00:21:49.563407Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7735, 2)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.574522Z",
     "start_time": "2020-08-28T00:21:49.567617Z"
    }
   },
   "outputs": [],
   "source": [
    "sms_test = sms.drop(sms_train.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.579722Z",
     "start_time": "2020-08-28T00:21:49.575707Z"
    }
   },
   "outputs": [],
   "source": [
    "sms_train.reset_index(drop=True, inplace=True)\n",
    "sms_test.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.590221Z",
     "start_time": "2020-08-28T00:21:49.580712Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5155, 2)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.611591Z",
     "start_time": "2020-08-28T00:21:49.591560Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make sure no overlap of sms in train-test\n",
    "sms_train.text.isin(sms_test.text).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.628526Z",
     "start_time": "2020-08-28T00:21:49.613279Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = Dataset.from_pandas(sms_train)\n",
    "test_dataset = Dataset.from_pandas(sms_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:21:49.723555Z",
     "start_time": "2020-08-28T00:21:49.629790Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "792c60ce04994d5ca37c68c7116f9b74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2008cf568df48c98d95a7f8313a0f00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_dataset = train_dataset.map(lambda examples: {\"label\": examples[\"user_id\"]}, batched=True)\n",
    "test_dataset = test_dataset.map(lambda examples: {\"label\": examples[\"user_id\"]}, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:22:04.631247Z",
     "start_time": "2020-08-28T00:21:49.725224Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at zanelim/singbert-large-sg were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at zanelim/singbert-large-sg and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# allow up to 10 mins to download the model when running for the first time\n",
    "tokenizer = BertTokenizer.from_pretrained(\"zanelim/singbert-large-sg\")\n",
    "model = BertForSequenceClassification.from_pretrained(\"zanelim/singbert-large-sg\", \n",
    "                                                      num_labels=sms_train.user_id.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:22:04.634263Z",
     "start_time": "2020-08-28T00:22:04.632270Z"
    }
   },
   "outputs": [],
   "source": [
    "def tokenize(batch):\n",
    "    return tokenizer(batch[\"text\"], padding=True, truncation=True, max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:22:12.931850Z",
     "start_time": "2020-08-28T00:22:04.635210Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7efe818bf9a44b5583d98e29998d539b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ae52a9008d6419a80b9d87b4bdae563",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_dataset = train_dataset.map(tokenize, batched=True, batch_size=len(train_dataset))\n",
    "test_dataset = test_dataset.map(tokenize, batched=True, batch_size=len(test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:22:12.935687Z",
     "start_time": "2020-08-28T00:22:12.933020Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\n",
    "test_dataset.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:22:12.954418Z",
     "start_time": "2020-08-28T00:22:12.936701Z"
    }
   },
   "outputs": [],
   "source": [
    "# freeze weights of pre-trained model\n",
    "for param in model.base_model.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-28T00:22:12.964180Z",
     "start_time": "2020-08-28T00:22:12.957839Z"
    }
   },
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\n",
    "        \"accuracy\": acc,\n",
    "        \"f1\": f1,\n",
    "        \"precision\": precision,\n",
    "        \"recall\": recall\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T09:23:57.688633Z",
     "start_time": "2020-08-30T09:23:57.672790Z"
    }
   },
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"../data/sms_classification\",\n",
    "    num_train_epochs=15,\n",
    "    per_device_train_batch_size=200,\n",
    "    per_device_eval_batch_size=64,\n",
    "    warmup_steps=300,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir=\"../data/sms_classification\",\n",
    "    save_steps=1000,\n",
    "    save_total_limit=1\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    compute_metrics=compute_metrics,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=test_dataset\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T09:53:12.103266Z",
     "start_time": "2020-08-30T09:24:03.616882Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zane/miniconda3/envs/nlp/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py:26: UserWarning: \n",
      "    There is an imbalance between your GPUs. You may want to exclude GPU 1 which\n",
      "    has less than 75% of the memory or cores of GPU 0. You can do so by setting\n",
      "    the device_ids argument to DataParallel, or by setting the CUDA_VISIBLE_DEVICES\n",
      "    environment variable.\n",
      "  warnings.warn(imbalance_warn.format(device_ids[min_pos], device_ids[max_pos]))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a19a44541bb24e05b2b417494b050b1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=15.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84fa71defa164dda94c5d39101b2fc3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zane/miniconda3/envs/nlp/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bd68731ed6848758cf84f92e09437af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4ae83b0075a42568a05ccd35c313216",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "844d47edd22541f9a10b897ac2919d72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35ef946fde504e5bb8936de3aa049bf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9fcdc0740cf4dba89b36d5e05698811",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a442dcd3711b4f528cc3c8a94a495f23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66d479574c4146a590818a05417b525c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86f04cae8ada456aba17229c70860bc2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7003f964eb0c497dbc771ddb1746e60c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83ae44fba744444c9d4c6c917f2b3b09",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a6281d4d9aa4506b8e580fd0d7b6a27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "833144e9e6fe425984dddc006eb6a20c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01d6db543be449deb790e441c67e0141",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8f31313ff3e402bae03256f8df3af23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=20.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=300, training_loss=1.0349915971358616)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this will take around 30 mins on 2 GPUs\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T09:56:47.659822Z",
     "start_time": "2020-08-30T09:55:35.843611Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zane/miniconda3/envs/nlp/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py:26: UserWarning: \n",
      "    There is an imbalance between your GPUs. You may want to exclude GPU 1 which\n",
      "    has less than 75% of the memory or cores of GPU 0. You can do so by setting\n",
      "    the device_ids argument to DataParallel, or by setting the CUDA_VISIBLE_DEVICES\n",
      "    environment variable.\n",
      "  warnings.warn(imbalance_warn.format(device_ids[min_pos], device_ids[max_pos]))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42dd569033954f63b08a1dcd510b1dea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=41.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zane/miniconda3/envs/nlp/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.911719897898232,\n",
       " 'eval_accuracy': 0.6259941804073715,\n",
       " 'eval_f1': 0.6153501909755232,\n",
       " 'eval_precision': 0.6131202312817052,\n",
       " 'eval_recall': 0.6259941804073715,\n",
       " 'epoch': 15.0}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:02:31.925125Z",
     "start_time": "2020-08-30T10:02:29.080021Z"
    }
   },
   "outputs": [],
   "source": [
    "trainer.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:01:23.571480Z",
     "start_time": "2020-08-30T10:00:11.673935Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zane/miniconda3/envs/nlp/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py:26: UserWarning: \n",
      "    There is an imbalance between your GPUs. You may want to exclude GPU 1 which\n",
      "    has less than 75% of the memory or cores of GPU 0. You can do so by setting\n",
      "    the device_ids argument to DataParallel, or by setting the CUDA_VISIBLE_DEVICES\n",
      "    environment variable.\n",
      "  warnings.warn(imbalance_warn.format(device_ids[min_pos], device_ids[max_pos]))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35b3b99e760340a9834b3ef06c4272f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Prediction', max=41.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zane/miniconda3/envs/nlp/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "preds = trainer.predict(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:02:42.685389Z",
     "start_time": "2020-08-30T10:02:42.666859Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.67      0.73      0.70      1873\n",
      "           1       0.49      0.35      0.41      1400\n",
      "           2       0.65      0.72      0.68      1882\n",
      "\n",
      "    accuracy                           0.63      5155\n",
      "   macro avg       0.60      0.60      0.60      5155\n",
      "weighted avg       0.61      0.63      0.62      5155\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(preds.label_ids, preds.predictions.argmax(-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:05:44.928245Z",
     "start_time": "2020-08-30T10:05:44.908386Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1374</td>\n",
       "      <td>217</td>\n",
       "      <td>282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>447</td>\n",
       "      <td>493</td>\n",
       "      <td>460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>221</td>\n",
       "      <td>301</td>\n",
       "      <td>1360</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0    1     2\n",
       "0  1374  217   282\n",
       "1   447  493   460\n",
       "2   221  301  1360"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(preds.label_ids, preds.predictions.argmax(-1)), \n",
    "             index=range(3), columns=range(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected, the model does a much better job in classifying the 1st and 3rd user due to their distinctiveness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:09:46.568175Z",
     "start_time": "2020-08-30T10:09:46.561717Z"
    }
   },
   "outputs": [],
   "source": [
    "y_test_true = preds.predictions.argmax(-1)\n",
    "y_test_pred = sms_test.user_id.values\n",
    "y_test = pd.DataFrame({\"true\": y_test_true, \"pred\": y_test_pred})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:12:25.609384Z",
     "start_time": "2020-08-30T10:12:25.597497Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Wah...News really spread fast...Haha tonite dun nid formal rite?',\n",
       " 'then, come back more often to c grandma when u r free, take care...',\n",
       " 'HI LADY PLSE',\n",
       " 'Oh... Lk tt ah... Wat kind of jobs u wan... Waitress or office, i help u look out..',\n",
       " 'L CALL U',\n",
       " 'We meet at 1115 instead can? Btw, ü wanna go for my hall bash in embassy on e 18?',\n",
       " 'Yup... Centre pt all small ones leh... 300 pieces.',\n",
       " 'Really... Thgt u oredi booked tt lesson liao... Haha, gd...',\n",
       " 'I thk tmr still got one more lecture... I got no more tut so thk i wun b going to school...',\n",
       " 'Wah u so gd... Only 6 pts...']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's look at the examples where the model is correct for the 1st user\n",
    "sms_test.loc[(y_test.true == 0) & (y_test.pred == 0), \"text\"].sample(10).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:12:45.072283Z",
     "start_time": "2020-08-30T10:12:45.063515Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ü giving tuition rite? Ya applied for local ü',\n",
       " 'No lah. C u tom then ü',\n",
       " 'm so sweet rite.. i bought choc cake for xin, yummy..',\n",
       " 'Hi! How was your weekend? Did you have a good time?  :)',\n",
       " 'U still outside w friends?',\n",
       " 'Haha cant help much.',\n",
       " 'he said it is quite a good chance to practice before the real presentation lor, you might want to think about it.',\n",
       " 'Haha... So is he handsomer?',\n",
       " 'Erm wat break....? I think yr jc guy frens in army had a break last week rite?Hmmm, i think a nite job is quite suitable for ü.... Hee hee.',\n",
       " 'm on my way liao...']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's look at the examples where the model is incorrect for the 1st user\n",
    "sms_test.loc[(y_test.true != 0) & (y_test.pred == 0), \"text\"].sample(10).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:13:04.950615Z",
     "start_time": "2020-08-30T10:13:04.942247Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hahahaha nvm la.',\n",
       " ':-( neh mind',\n",
       " '<#> pm eh. Help me tell xt eh.',\n",
       " 'Outside. Why?',\n",
       " 'Nah I will leave yours out.',\n",
       " 'Any bubble tea for you?',\n",
       " 'Yo you looking for me?',\n",
       " \"Yeah, quite troublesome, but it's a one time thing mah.\",\n",
       " \"Oh ya. Can come down  <#> '\",\n",
       " 'Yup im still in school.']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's look at the examples where the model is correct for the 2nd user\n",
    "sms_test.loc[(y_test.true == 1) & (y_test.pred == 1), \"text\"].sample(10).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:13:27.888176Z",
     "start_time": "2020-08-30T10:13:27.878991Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Haha when's your exam?\",\n",
       " 'Haha eating with who?',\n",
       " 'Yes, burden off my shoulders! Haha',\n",
       " 'Mahjong tonight? Cally not free, see if you can find players.',\n",
       " 'Wah so good lol. Did he like use that to bao your red packet?! Lol',\n",
       " 'Lol what order is that?! Lol sugar level?',\n",
       " \"Yo ladies, if the movie changed to wednesday night, who's on?\",\n",
       " \"Hhahaha or probably you have not talk to me for the past few days,that's why the fever came. Nvm now im here already hahahaha\",\n",
       " 'Oh im at clubhouse. Later meeting at lt <#> a. I will be there by 6pm.',\n",
       " 'Yup to replenish your sodium. Should probably try some chivas too. XD']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's look at the examples where the model is incorrect for the 2nd user\n",
    "sms_test.loc[(y_test.true != 1) & (y_test.pred == 1), \"text\"].sample(10).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:13:48.878395Z",
     "start_time": "2020-08-30T10:13:48.870579Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Leaving amk! About half an hour bah! If you want can go cineleisure first!',\n",
       " 'Boo! Haha thanks for the chat:P have a safe trip home!',\n",
       " 'Haha ohh no super unglam!!!! ><next time dun laugh in front of youle! Hey you want to take 2 star on  <DATE>  oct? Jiayi Spencer JensSophie going most probably, can ask Lewis Nicky yuhan they all too!',\n",
       " 'Are you still here? Hehe I have sth to pass you:P',\n",
       " 'LOL AWESOME WHY HE SO WULIAO HAHAHA OKAY OUCH THAT IS PAINFUL I HOPETHERE IS NO HOPE IN YOUR FOOT RIGHT NOW',\n",
       " 'Anyway go sch with me tmr morning!! :D I can take  <#>  go meet youthen we take  <#> !',\n",
       " \"It's not!! I'm back!\",\n",
       " \"Haha uhh okay ah! he's a senior! His hairstyle needs to change though LOL.\",\n",
       " \"Haha ok ok ah I think but still won't do very well bah! Not sure leh!I felt like I could answer but then dunno if I missed anything outhaha:/ going to bathe now!\",\n",
       " 'Aw man haha fine:P see you later then!! Enjoy yourself ((:']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's look at the examples where the model is correct for the 3rd user\n",
    "sms_test.loc[(y_test.true == 2) & (y_test.pred == 2), \"text\"].sample(10).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-30T10:14:08.477116Z",
     "start_time": "2020-08-30T10:14:08.467688Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ohh x.x haha okay! Thanks:D do you think I can put my bag there?',\n",
       " 'Oh crap:x!!!!!7',\n",
       " \"Hihi ppl wanna volunteer to help out for a swimming event? On 4 & 5Nov, 7 am to 9 pm with 5 hours break in between on both days plus acompulsory training session on  <#>  Oct  <#>  -  <#> . Can get twoevent tshirts &  <#>  bucks! Reply with your nric, house address,emergency contact (name, number, relationship) plus your allergies!Let's volunteer tgt, so we sign up for uhh f&b and VIP room (1stchoice) or ushers (2nd) coz hehe I think these two move ard less bahand not so stress! Please reply if you're interested and jio anyoneelse! I need replies by tonight! I sent this to abbie, vivien, sophie,friend, Ivan, huikang and wenbo, I dun have Jimmy's number!\",\n",
       " 'Okay, I go down now?:P',\n",
       " 'Oh I dunno maybe he was attempting a raghav:P',\n",
       " 'You pig!! Ok!!! Want do um on Wednesday???',\n",
       " 'No! You!',\n",
       " \"Hahaha confusing! And no i'm not!!!\",\n",
       " 'LOL I ALSO DUN KNOW HAHAHA',\n",
       " \"Hahaha what I didn't see!!! What hack!? Woohoo I went shopping today LOL.\"]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's look at the examples where the model is incorrect for the 3rd user\n",
    "sms_test.loc[(y_test.true != 2) & (y_test.pred == 2), \"text\"].sample(10).tolist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NLP",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}