070-rnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recurrent Neural Network\n",
    "\n",
    "![LSTM](imgs/LSTM3-chain.png)\n",
    "\n",
    "![LSTM](imgs/LSTM2-notation.png)\n",
    "\n",
    "\n",
    "\n",
    "![LSTM](imgs/LSTM3-focus-f.png)\n",
    "\n",
    "![LSTM](imgs/LSTM3-focus-i.png)\n",
    "\n",
    "![LSTM](imgs/LSTM3-focus-C.png)\n",
    "\n",
    "![LSTM](imgs/LSTM3-focus-o.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## From pytorch documentation\n",
    "\n",
    "\\begin{array}{ll} \\\\\n",
    "    f_t = \\sigma(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\\\\n",
    "    i_t = \\sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n",
    "    g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{(t-1)} + b_{hg}) \\\\\n",
    "    c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n",
    "    o_t = \\sigma(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\\\    \n",
    "    h_t = o_t * \\tanh(c_t) \\\\\n",
    "\\end{array}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to define LSTM layer in pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 1.3358,  0.4016, -0.7162],\n",
       "         [-0.0836, -0.0075,  1.2336]],\n",
       "\n",
       "        [[-1.1825, -0.9264, -1.0846],\n",
       "         [ 0.7155, -1.5321,  0.5685]],\n",
       "\n",
       "        [[ 0.4086,  0.0886, -0.1730],\n",
       "         [ 0.8533, -0.2062, -1.0645]],\n",
       "\n",
       "        [[ 0.5711,  0.3063,  1.0305],\n",
       "         [-1.6005,  0.9858,  0.0909]],\n",
       "\n",
       "        [[ 0.3825, -0.6499,  1.1365],\n",
       "         [ 0.4503, -0.9128, -0.4787]],\n",
       "\n",
       "        [[ 0.7760,  0.6263, -0.5889],\n",
       "         [ 0.7740,  1.5630, -0.9118]],\n",
       "\n",
       "        [[ 1.2025, -0.1304,  0.0312],\n",
       "         [ 0.9756,  0.3158,  0.9231]],\n",
       "\n",
       "        [[-0.2801, -0.7592,  0.0316],\n",
       "         [-0.5743,  0.4620, -1.0581]],\n",
       "\n",
       "        [[ 0.0326,  0.3926, -0.5817],\n",
       "         [ 0.6513,  0.2424, -0.0168]],\n",
       "\n",
       "        [[-0.3728, -0.1267,  1.2081],\n",
       "         [ 0.5376,  0.2517,  0.9090]]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "\n",
    "batch_size = 2\n",
    "seq_len = 10\n",
    "input_size = 3\n",
    "hidden_size = 4 \n",
    "\n",
    "inputs = torch.randn(seq_len, batch_size, input_size)\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## hidden_0 = (h_0, c_0)\n",
    "hidden_0 = (torch.zeros(1, batch_size, hidden_size), torch.zeros(1, batch_size, hidden_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 2, 4])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out, lstm_hidden = lstm(inputs)\n",
    "lstm_out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1176, -0.0779, -0.1266, -0.1230],\n",
       "         [ 0.1297, -0.0501,  0.0050,  0.0971]],\n",
       "\n",
       "        [[ 0.0655,  0.0118, -0.1611,  0.0485],\n",
       "         [ 0.2832, -0.0866, -0.0079,  0.0955]],\n",
       "\n",
       "        [[ 0.1463, -0.0426, -0.1240,  0.0108],\n",
       "         [ 0.1705, -0.1128, -0.1397, -0.0414]],\n",
       "\n",
       "        [[ 0.1987, -0.0982, -0.0558,  0.0282],\n",
       "         [-0.0534, -0.0344, -0.1337,  0.0492]],\n",
       "\n",
       "        [[ 0.2735, -0.1196, -0.0231,  0.0840],\n",
       "         [ 0.1243, -0.0680, -0.1170,  0.0540]],\n",
       "\n",
       "        [[ 0.1202, -0.1452, -0.1386, -0.0779],\n",
       "         [ 0.0111, -0.1179, -0.2141, -0.1950]],\n",
       "\n",
       "        [[ 0.2105, -0.1802, -0.1016, -0.0722],\n",
       "         [ 0.2016, -0.1546, -0.0837, -0.0804]],\n",
       "\n",
       "        [[ 0.1864, -0.1185, -0.0759,  0.0444],\n",
       "         [ 0.0236, -0.0768, -0.2144, -0.1122]],\n",
       "\n",
       "        [[ 0.0952, -0.1128, -0.1545, -0.0309],\n",
       "         [ 0.1403, -0.1238, -0.1288, -0.0935]],\n",
       "\n",
       "        [[ 0.1655, -0.1065, -0.0386,  0.0946],\n",
       "         [ 0.1948, -0.1419, -0.0577, -0.0251]]], grad_fn=<StackBackward>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we want to put hidden inputs to zeros, there is no need to provide them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1176, -0.0779, -0.1266, -0.1230],\n",
       "         [ 0.1297, -0.0501,  0.0050,  0.0971]],\n",
       "\n",
       "        [[ 0.0655,  0.0118, -0.1611,  0.0485],\n",
       "         [ 0.2832, -0.0866, -0.0079,  0.0955]],\n",
       "\n",
       "        [[ 0.1463, -0.0426, -0.1240,  0.0108],\n",
       "         [ 0.1705, -0.1128, -0.1397, -0.0414]],\n",
       "\n",
       "        [[ 0.1987, -0.0982, -0.0558,  0.0282],\n",
       "         [-0.0534, -0.0344, -0.1337,  0.0492]],\n",
       "\n",
       "        [[ 0.2735, -0.1196, -0.0231,  0.0840],\n",
       "         [ 0.1243, -0.0680, -0.1170,  0.0540]],\n",
       "\n",
       "        [[ 0.1202, -0.1452, -0.1386, -0.0779],\n",
       "         [ 0.0111, -0.1179, -0.2141, -0.1950]],\n",
       "\n",
       "        [[ 0.2105, -0.1802, -0.1016, -0.0722],\n",
       "         [ 0.2016, -0.1546, -0.0837, -0.0804]],\n",
       "\n",
       "        [[ 0.1864, -0.1185, -0.0759,  0.0444],\n",
       "         [ 0.0236, -0.0768, -0.2144, -0.1122]],\n",
       "\n",
       "        [[ 0.0952, -0.1128, -0.1545, -0.0309],\n",
       "         [ 0.1403, -0.1238, -0.1288, -0.0935]],\n",
       "\n",
       "        [[ 0.1655, -0.1065, -0.0386,  0.0946],\n",
       "         [ 0.1948, -0.1419, -0.0577, -0.0251]]], grad_fn=<StackBackward>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out, lstm_hidden = lstm(inputs)\n",
    "lstm_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally the last output is the output of RRR. We can get it by"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.1655, -0.1065, -0.0386,  0.0946],\n",
       "        [ 0.1948, -0.1419, -0.0577, -0.0251]], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is often convient to have batches as the first dimension of the input. One can do it by adding `batch_first=True` parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-1.1078,  0.4207,  1.2509],\n",
       "         [-0.2133,  1.0294, -0.9465],\n",
       "         [-0.1794,  1.4713, -0.6459],\n",
       "         [-1.6491, -0.4083, -1.6127],\n",
       "         [ 0.0338,  0.0551,  0.2075],\n",
       "         [ 1.0826,  0.9102,  1.0004],\n",
       "         [-0.9677,  0.6797,  1.1882],\n",
       "         [-0.9318, -1.7912,  0.0443],\n",
       "         [ 0.1861, -0.3959,  1.2669],\n",
       "         [ 0.6288,  1.3873, -0.9654]],\n",
       "\n",
       "        [[-0.4574,  1.0188,  0.9024],\n",
       "         [ 0.0841,  0.5861, -0.3742],\n",
       "         [-0.5549, -0.7942, -0.9964],\n",
       "         [-1.1586, -0.7832, -1.2494],\n",
       "         [ 0.3433,  0.0775,  0.9570],\n",
       "         [ 0.6465, -1.2579, -1.5625],\n",
       "         [ 2.2202, -1.5127, -0.3422],\n",
       "         [ 0.3673, -1.4248,  0.0642],\n",
       "         [-0.1735, -0.7212,  1.2056],\n",
       "         [ 0.8488,  0.1269, -1.3305]]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_batch_first = nn.LSTM(input_size=input_size, hidden_size=hidden_size, batch_first=True) \n",
    "inputs_batch_first = torch.randn(batch_size, seq_len, input_size)\n",
    "inputs_batch_first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.0181, -0.0610, -0.0240,  0.0159],\n",
       "         [ 0.0533, -0.0131,  0.0635,  0.0826],\n",
       "         [ 0.0570, -0.0137,  0.1008,  0.0280],\n",
       "         [ 0.2066, -0.0136,  0.1808,  0.2782],\n",
       "         [ 0.0986, -0.0490,  0.0719,  0.1130],\n",
       "         [-0.0743, -0.0637,  0.0396, -0.1185],\n",
       "         [-0.0559, -0.0753,  0.0162, -0.0429],\n",
       "         [ 0.1037, -0.0649, -0.0008,  0.1253],\n",
       "         [-0.0448, -0.0937, -0.0143,  0.0078],\n",
       "         [-0.0010,  0.0029,  0.0514, -0.0971]],\n",
       "\n",
       "        [[-0.0323, -0.0575,  0.0141, -0.0524],\n",
       "         [ 0.0032, -0.0307,  0.0696, -0.0326],\n",
       "         [ 0.1795, -0.0241,  0.1158,  0.1892],\n",
       "         [ 0.2664, -0.0281,  0.1197,  0.2927],\n",
       "         [ 0.0346, -0.0785,  0.0291,  0.0354],\n",
       "         [ 0.2608, -0.0065,  0.0878,  0.2250],\n",
       "         [-0.0120, -0.0109,  0.0618, -0.0089],\n",
       "         [ 0.0467, -0.0372,  0.0401,  0.0651],\n",
       "         [-0.0372, -0.0780, -0.0146,  0.0263],\n",
       "         [ 0.0945,  0.0063,  0.0501,  0.0590]]], grad_fn=<TransposeBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out, lstm_hidden = lstm_batch_first(inputs_batch_first)\n",
    "lstm_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we get the finial output by:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0010,  0.0029,  0.0514, -0.0971],\n",
       "        [ 0.0945,  0.0063,  0.0501,  0.0590]], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out[:, -1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[38, 58, 76,  8, 25, 75, 35, 37, 13, 97],\n",
       "        [93, 35, 63, 85, 27, 54, 50, 23,  3, 88]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict_size = 100\n",
    "sentences = torch.randint(dict_size, (batch_size, seq_len))\n",
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 3\n",
    "embedding = nn.Embedding(dict_size, embedding_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.3717,  0.7268,  0.9202],\n",
       "         [-1.4053, -2.0243,  1.8774],\n",
       "         [-0.1911, -1.1051,  0.1172],\n",
       "         [-0.5170, -0.0370, -1.4053],\n",
       "         [-0.7114,  0.9241,  1.2172],\n",
       "         [-0.3384, -0.0622, -0.8514],\n",
       "         [ 1.3966,  0.4329, -0.7663],\n",
       "         [-0.8157, -0.1641,  0.3133],\n",
       "         [-0.4706,  1.0463,  0.0747],\n",
       "         [-0.6496, -1.0619,  0.6621]],\n",
       "\n",
       "        [[-0.6311, -0.1210, -1.2683],\n",
       "         [ 1.3966,  0.4329, -0.7663],\n",
       "         [ 0.2394,  0.7240, -0.7275],\n",
       "         [ 0.1625, -0.8712, -1.3705],\n",
       "         [-0.8190, -0.1620, -0.6818],\n",
       "         [-0.2097, -0.1759, -0.1939],\n",
       "         [-0.5872, -1.4678,  1.1135],\n",
       "         [-0.2881, -0.6247, -0.4135],\n",
       "         [ 1.1156,  2.5285,  0.6394],\n",
       "         [-0.2821,  1.4300, -0.3832]]], grad_fn=<EmbeddingBackward>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences_embedded = embedding(sentences)\n",
    "sentences_embedded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0570, -0.0736,  0.0134,  0.0969],\n",
       "        [-0.0008, -0.0312,  0.1075, -0.1063]], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out, _ = lstm_batch_first(sentences_embedded)\n",
    "lstm_out[:, -1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Natural Language Processing (NLP)\n",
    "\n",
    "Next we consider a dataset with text and the goal is to evaluate whether they are toxic or non-toxic.\n",
    "\n",
    "You can download the dataset in the following link:\n",
    "\n",
    "[here](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge). \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "comments_df = pd.read_csv(\"data/jigsaw-toxic-comment-classification-challenge/train.csv\")\n",
    "comments_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27250</th>\n",
       "      <td>I'm afraid that you didn't follow the history ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133958</th>\n",
       "      <td>Drmies, you really need to be de-syoped.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment_text\n",
       "27250   I'm afraid that you didn't follow the history ...\n",
       "133958           Drmies, you really need to be de-syoped."
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "label_colnames = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = \\\n",
    "    train_test_split(comments_df[['comment_text']], comments_df[label_colnames], random_state=667)\n",
    "X_train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "import nltk\n",
    "from nltk.stem import SnowballStemmer\n",
    "\n",
    "BAD_SYMBOLS_RE = re.compile('[^0-9a-z ]')\n",
    "STEMMER = SnowballStemmer('english')\n",
    "\n",
    "class TextPreprocessor:\n",
    "        \n",
    "    def transfrom_text(self, text):\n",
    "        text = text.lower()\n",
    "        text = re.sub(BAD_SYMBOLS_RE, \" \", text) # process bad symbols\n",
    "        # text = \" \".join([STEMMER.stem(word) for word in text.split()])\n",
    "        return text\n",
    "    \n",
    "    def transform(self, series):\n",
    "        return series.apply(lambda text: self.transfrom_text(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = TextPreprocessor()\n",
    "X_train_preprocessed = preprocessor.transform(X_train['comment_text'])\n",
    "X_test_preprocessed = preprocessor.transform(X_test['comment_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I'm afraid that you didn't follow the history of all this. (1) I don't want to obtain an email, I want to write it. (2) Every time when someone makes a remark, I tend to think that I am obligated to reply, since otherwise that person may think  that I neglect him/her, and don't think him or her worthy of a reply. But that leaves me no time to write that email. (3) Also, if I would take that time, then Rspeer could delete all my contributions in the mean time, as he already has started to do,  making it hard work again to restore it. (4) I recognize that wikipedia can be accessed any time of the day, but it might be proper for you and Rspeer and others reading my request for a time out to respect my wish for it. (5) Rspeer started  this discussion about my conduct on this talk page, by saying that there was a threat, while there wasn't. Now, do I have to take this seriously ? And have this nonsense here ? (6) Should I have entered my criticism on his behaviour on his talk page, instead of the borda fixed point page, where he occluded the argument on the article with his behavior ? (7) My impression now is that I have to ask for mediation to get some quiet time to formulate my argument on the integrity of science for the professors. True of false ?\n",
      "---------------------------------------------------------------------------------------------------------------------\n",
      "i m afraid that you didn t follow the history of all this   1  i don t want to obtain an email  i want to write it   2  every time when someone makes a remark  i tend to think that i am obligated to reply  since otherwise that person may think  that i neglect him her  and don t think him or her worthy of a reply  but that leaves me no time to write that email   3  also  if i would take that time  then rspeer could delete all my contributions in the mean time  as he already has started to do   making it hard work again to restore it   4  i recognize that wikipedia can be accessed any time of the day  but it might be proper for you and rspeer and others reading my request for a time out to respect my wish for it   5  rspeer started  this discussion about my conduct on this talk page  by saying that there was a threat  while there wasn t  now  do i have to take this seriously   and have this nonsense here    6  should i have entered my criticism on his behaviour on his talk page  instead of the borda fixed point page  where he occluded the argument on the article with his behavior    7  my impression now is that i have to ask for mediation to get some quiet time to formulate my argument on the integrity of science for the professors  true of false  \n"
     ]
    }
   ],
   "source": [
    "print(X_train[\"comment_text\"].iloc[0])\n",
    "print('---------------------------------------------------------------------------------------------------------------------')\n",
    "print(X_train_preprocessed.iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drmies, you really need to be de-syoped.\n",
      "---------------------------------------------------------------------------------------------------------------------\n",
      "drmies  you really need to be de syoped \n"
     ]
    }
   ],
   "source": [
    "print(X_train[\"comment_text\"].iloc[1])\n",
    "print('---------------------------------------------------------------------------------------------------------------------')\n",
    "\n",
    "print(X_train_preprocessed.iloc[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dicts(text):\n",
    "    word_set = set()\n",
    "    words = text.split()\n",
    "    for word in words:\n",
    "        word_set.add(word)\n",
    "    word_list = [\"<UNK>\", \"<PAD>\"] + sorted(list(word_set))\n",
    "    word2idx = {word_list[idx]: idx for idx in range(len(word_list))}\n",
    "    idx2word = {idx: word_list[idx] for idx in range(len(word_list))}\n",
    "    return word2idx, idx2word\n",
    "\n",
    "class Tokenizer:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.word2idx = None\n",
    "        self.idx2word = None\n",
    "        \n",
    "    def fit(self, X):\n",
    "        text = \" \".join(X)\n",
    "        self.word2idx, self.idx2word = create_dicts(text)\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return [self.transform_line(line) for line in X]\n",
    "        \n",
    "    def transform_line(self, line):\n",
    "        return [self.word2idx.get(word, 0) for word in line.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit(X_train_preprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tokenized = tokenizer.transform(X_train_preprocessed)\n",
    "X_test_tokenized = tokenizer.transform(X_test_preprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cutter:\n",
    "\n",
    "    def __init__(self, size=150):\n",
    "        self.size = size\n",
    "        \n",
    "    def transform(self, X):\n",
    "        new_X = []\n",
    "        for line in X:\n",
    "            new_line = line[:self.size]\n",
    "            new_line = new_line + [1] * (self.size - len(new_line))\n",
    "            new_X.append(new_line)\n",
    "        return new_X    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "cutter = Cutter()\n",
    "X_train_cutted = cutter.transform(X_train_tokenized)\n",
    "X_test_cutted = cutter.transform(X_test_tokenized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = torch.from_numpy(y_train.values)\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = TensorDataset(torch.tensor(X_train_cutted), torch.from_numpy(y_train.values).float())\n",
    "test_data = TensorDataset(torch.tensor(X_test_cutted), torch.from_numpy(y_test.values).float())\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)\n",
    "test_loader = DataLoader(test_data, shuffle=True, batch_size=len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMModel(nn.Module):\n",
    "    \n",
    "    def __init__(self, dict_size, output_size, embedding_dim, hidden_dim):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(dict_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, output_size)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        embeded = self.embedding(x)\n",
    "        lstm_out, _ = self.lstm(embeded)\n",
    "        lstm_out = lstm_out[:, -1]        \n",
    "        logits = self.fc(lstm_out)\n",
    "        out = self.sigmoid(logits)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_size = len(tokenizer.word2idx)\n",
    "output_size = len(label_colnames)\n",
    "embedding_dim = 3\n",
    "hidden_dim = 4\n",
    "\n",
    "lstm_model = LSTMModel(dict_size, output_size, embedding_dim, hidden_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Load train model\n",
    "\n",
    "#lstm_model.load_state_dict(torch.load(\"models/lstm_model2.pt\"))\n",
    "#lstm_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([119678, 150])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_torch = torch.tensor(X_train_cutted)\n",
    "X_test_torch = torch.tensor(X_test_cutted)\n",
    "X_train_torch.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9631, 0.4968, 0.8426, 0.0126, 0.6930, 0.1262],\n",
       "        [0.8758, 0.2331, 0.5415, 0.0112, 0.4926, 0.0973],\n",
       "        [0.8869, 0.2502, 0.5701, 0.0116, 0.5121, 0.1010],\n",
       "        ...,\n",
       "        [0.9571, 0.4502, 0.8042, 0.0143, 0.6883, 0.1372],\n",
       "        [0.9011, 0.2758, 0.6101, 0.0123, 0.5396, 0.1064],\n",
       "        [0.8824, 0.2431, 0.5585, 0.0115, 0.5041, 0.0994]],\n",
       "       grad_fn=<SigmoidBackward>)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_model(X_train_torch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.8865, 0.2496, 0.5691, 0.0116, 0.5114, 0.1008],\n",
       "        [0.8304, 0.1793, 0.4415, 0.0098, 0.4251, 0.0849],\n",
       "        [0.8873, 0.2509, 0.5712, 0.0117, 0.5128, 0.1011],\n",
       "        [0.8906, 0.2566, 0.5804, 0.0118, 0.5191, 0.1023],\n",
       "        [0.8663, 0.2197, 0.5182, 0.0109, 0.4769, 0.0943],\n",
       "        [0.9321, 0.3507, 0.7097, 0.0141, 0.6115, 0.1216],\n",
       "        [0.8996, 0.2729, 0.6057, 0.0122, 0.5365, 0.1058],\n",
       "        [0.8381, 0.1869, 0.4567, 0.0100, 0.4354, 0.0867],\n",
       "        [0.8567, 0.2076, 0.4961, 0.0105, 0.4620, 0.0916],\n",
       "        [0.8740, 0.2305, 0.5370, 0.0111, 0.4896, 0.0967],\n",
       "        [0.9118, 0.2980, 0.6422, 0.0128, 0.5621, 0.1109],\n",
       "        [0.8423, 0.1913, 0.4653, 0.0101, 0.4412, 0.0878],\n",
       "        [0.8688, 0.2232, 0.5244, 0.0110, 0.4811, 0.0951],\n",
       "        [0.8991, 0.2719, 0.6042, 0.0122, 0.5355, 0.1055],\n",
       "        [0.8676, 0.2215, 0.5214, 0.0109, 0.4790, 0.0947],\n",
       "        [0.8642, 0.2170, 0.5133, 0.0108, 0.4736, 0.0937],\n",
       "        [0.9246, 0.3295, 0.6839, 0.0136, 0.5922, 0.1173],\n",
       "        [0.8660, 0.2194, 0.5176, 0.0109, 0.4765, 0.0942],\n",
       "        [0.8895, 0.2547, 0.5773, 0.0118, 0.5170, 0.1019],\n",
       "        [0.8922, 0.2593, 0.5846, 0.0119, 0.5220, 0.1029],\n",
       "        [0.8978, 0.2694, 0.6004, 0.0121, 0.5329, 0.1050],\n",
       "        [0.8980, 0.2698, 0.6010, 0.0121, 0.5333, 0.1051],\n",
       "        [0.8752, 0.2321, 0.5399, 0.0112, 0.4916, 0.0971],\n",
       "        [0.9872, 0.2550, 0.8957, 0.0264, 0.7711, 0.1222],\n",
       "        [0.9089, 0.2917, 0.6332, 0.0127, 0.5558, 0.1096],\n",
       "        [0.8974, 0.2687, 0.5993, 0.0121, 0.5321, 0.1049],\n",
       "        [0.9923, 0.4394, 0.9432, 0.0324, 0.8471, 0.1673],\n",
       "        [0.9031, 0.2797, 0.6158, 0.0124, 0.5436, 0.1072],\n",
       "        [0.8865, 0.2496, 0.5691, 0.0116, 0.5114, 0.1008],\n",
       "        [0.8407, 0.1897, 0.4620, 0.0101, 0.4390, 0.0874],\n",
       "        [0.9067, 0.2871, 0.6267, 0.0126, 0.5512, 0.1087],\n",
       "        [0.8950, 0.2643, 0.5925, 0.0120, 0.5274, 0.1039]],\n",
       "       grad_fn=<SigmoidBackward>)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataiter = iter(train_loader)\n",
    "input_data, labels = dataiter.next()\n",
    "lstm_model(input_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=0.005\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(lstm_model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Batch: 300, Avg. Loss: 0.13982448821887375\n",
      "Epoch: 1, Batch: 600, Avg. Loss: 0.09684357973746956\n",
      "Epoch: 1, Batch: 900, Avg. Loss: 0.09239127966575325\n",
      "Epoch: 1, Batch: 1200, Avg. Loss: 0.08004220797680318\n",
      "Epoch: 1, Batch: 1500, Avg. Loss: 0.07414760378965488\n",
      "Epoch: 1, Batch: 1800, Avg. Loss: 0.06854366751232495\n",
      "Epoch: 1, Batch: 2100, Avg. Loss: 0.06648986662738025\n",
      "Epoch: 1, Batch: 2400, Avg. Loss: 0.06476836512641361\n",
      "Epoch: 1, Batch: 2700, Avg. Loss: 0.0662984246853739\n",
      "Epoch: 1, Batch: 3000, Avg. Loss: 0.06384452339572211\n",
      "Epoch: 1, Batch: 3300, Avg. Loss: 0.062040791804902254\n",
      "Epoch: 1, Batch: 3600, Avg. Loss: 0.060024392330087724\n",
      "Epoch: 2, Batch: 300, Avg. Loss: 0.05178234449975813\n",
      "Epoch: 2, Batch: 600, Avg. Loss: 0.05627863814899077\n",
      "Epoch: 2, Batch: 900, Avg. Loss: 0.05626081367100899\n",
      "Epoch: 2, Batch: 1200, Avg. Loss: 0.05146623851576199\n",
      "Epoch: 2, Batch: 1500, Avg. Loss: 0.05188001333658273\n",
      "Epoch: 2, Batch: 1800, Avg. Loss: 0.05484229912981391\n",
      "Epoch: 2, Batch: 2100, Avg. Loss: 0.05336530979800348\n",
      "Epoch: 2, Batch: 2400, Avg. Loss: 0.056038549875374884\n",
      "Epoch: 2, Batch: 2700, Avg. Loss: 0.0555807306842568\n",
      "Epoch: 2, Batch: 3000, Avg. Loss: 0.054106840852958464\n",
      "Epoch: 2, Batch: 3300, Avg. Loss: 0.05086597552755848\n",
      "Epoch: 2, Batch: 3600, Avg. Loss: 0.04864576347948362\n",
      "Epoch: 3, Batch: 300, Avg. Loss: 0.044393346928215276\n",
      "Epoch: 3, Batch: 600, Avg. Loss: 0.04692011113239763\n",
      "Epoch: 3, Batch: 900, Avg. Loss: 0.04851389403998231\n",
      "Epoch: 3, Batch: 1200, Avg. Loss: 0.047970315769392376\n",
      "Epoch: 3, Batch: 1500, Avg. Loss: 0.04487440358963795\n",
      "Epoch: 3, Batch: 1800, Avg. Loss: 0.04371842278943708\n",
      "Epoch: 3, Batch: 2100, Avg. Loss: 0.04748319591182129\n",
      "Epoch: 3, Batch: 2400, Avg. Loss: 0.04748354717429417\n",
      "Epoch: 3, Batch: 2700, Avg. Loss: 0.046908298761894306\n",
      "Epoch: 3, Batch: 3000, Avg. Loss: 0.04586710873894238\n",
      "Epoch: 3, Batch: 3300, Avg. Loss: 0.046629551838462555\n",
      "Epoch: 3, Batch: 3600, Avg. Loss: 0.0444724496640265\n",
      "Epoch: 4, Batch: 300, Avg. Loss: 0.0407817749743117\n",
      "Epoch: 4, Batch: 600, Avg. Loss: 0.04242100465033824\n",
      "Epoch: 4, Batch: 900, Avg. Loss: 0.04305085731243404\n",
      "Epoch: 4, Batch: 1200, Avg. Loss: 0.04118700043201291\n",
      "Epoch: 4, Batch: 1500, Avg. Loss: 0.04330247267653855\n",
      "Epoch: 4, Batch: 1800, Avg. Loss: 0.0412816199309115\n",
      "Epoch: 4, Batch: 2100, Avg. Loss: 0.039870876104105266\n",
      "Epoch: 4, Batch: 2400, Avg. Loss: 0.04044972470301824\n",
      "Epoch: 4, Batch: 2700, Avg. Loss: 0.0400532601919258\n",
      "Epoch: 4, Batch: 3000, Avg. Loss: 0.0416542849575247\n",
      "Epoch: 4, Batch: 3300, Avg. Loss: 0.04202086203692791\n",
      "Epoch: 4, Batch: 3600, Avg. Loss: 0.04190342301075967\n",
      "Epoch: 5, Batch: 300, Avg. Loss: 0.03776516022432285\n",
      "Epoch: 5, Batch: 600, Avg. Loss: 0.03726959547406295\n",
      "Epoch: 5, Batch: 900, Avg. Loss: 0.037269444060560396\n",
      "Epoch: 5, Batch: 1200, Avg. Loss: 0.03746880683669587\n",
      "Epoch: 5, Batch: 1500, Avg. Loss: 0.038771907316646925\n",
      "Epoch: 5, Batch: 1800, Avg. Loss: 0.037507141835715935\n",
      "Epoch: 5, Batch: 2100, Avg. Loss: 0.03911266412993428\n",
      "Epoch: 5, Batch: 2400, Avg. Loss: 0.03811632242577616\n",
      "Epoch: 5, Batch: 2700, Avg. Loss: 0.03984430216893088\n",
      "Epoch: 5, Batch: 3000, Avg. Loss: 0.039031947191106156\n",
      "Epoch: 5, Batch: 3300, Avg. Loss: 0.040518154080297485\n",
      "Epoch: 5, Batch: 3600, Avg. Loss: 0.038965241483141046\n",
      "Epoch: 6, Batch: 300, Avg. Loss: 0.035584362288257884\n",
      "Epoch: 6, Batch: 600, Avg. Loss: 0.03329620494589714\n",
      "Epoch: 6, Batch: 900, Avg. Loss: 0.03554757640435127\n",
      "Epoch: 6, Batch: 1200, Avg. Loss: 0.035181261537169724\n",
      "Epoch: 6, Batch: 1500, Avg. Loss: 0.035268940453145964\n",
      "Epoch: 6, Batch: 1800, Avg. Loss: 0.03694840702917039\n",
      "Epoch: 6, Batch: 2100, Avg. Loss: 0.03611939007809269\n",
      "Epoch: 6, Batch: 2400, Avg. Loss: 0.039125030625921986\n",
      "Epoch: 6, Batch: 2700, Avg. Loss: 0.0382748777269444\n",
      "Epoch: 6, Batch: 3000, Avg. Loss: 0.03728025376253451\n",
      "Epoch: 6, Batch: 3300, Avg. Loss: 0.035739916235130904\n",
      "Epoch: 6, Batch: 3600, Avg. Loss: 0.03560904623862977\n",
      "Epoch: 7, Batch: 300, Avg. Loss: 0.03094780127556684\n",
      "Epoch: 7, Batch: 600, Avg. Loss: 0.03484708650774943\n",
      "Epoch: 7, Batch: 900, Avg. Loss: 0.032533820772077886\n",
      "Epoch: 7, Batch: 1200, Avg. Loss: 0.03575895840787174\n",
      "Epoch: 7, Batch: 1500, Avg. Loss: 0.03431994686494969\n",
      "Epoch: 7, Batch: 1800, Avg. Loss: 0.032681026409118206\n",
      "Epoch: 7, Batch: 2100, Avg. Loss: 0.033953782436243876\n",
      "Epoch: 7, Batch: 2400, Avg. Loss: 0.03570418233211967\n",
      "Epoch: 7, Batch: 2700, Avg. Loss: 0.035395526474506674\n",
      "Epoch: 7, Batch: 3000, Avg. Loss: 0.03382279381262682\n",
      "Epoch: 7, Batch: 3300, Avg. Loss: 0.036866019758066006\n",
      "Epoch: 7, Batch: 3600, Avg. Loss: 0.03454752248915611\n",
      "Epoch: 8, Batch: 300, Avg. Loss: 0.032761811268501334\n",
      "Epoch: 8, Batch: 600, Avg. Loss: 0.03337784664142722\n",
      "Epoch: 8, Batch: 900, Avg. Loss: 0.03347868820438937\n",
      "Epoch: 8, Batch: 1200, Avg. Loss: 0.03329681222011762\n",
      "Epoch: 8, Batch: 1500, Avg. Loss: 0.029377179278041392\n",
      "Epoch: 8, Batch: 1800, Avg. Loss: 0.03147018156390307\n",
      "Epoch: 8, Batch: 2100, Avg. Loss: 0.03172023814297669\n",
      "Epoch: 8, Batch: 2400, Avg. Loss: 0.03167620531627714\n",
      "Epoch: 8, Batch: 2700, Avg. Loss: 0.03337981017379207\n",
      "Epoch: 8, Batch: 3000, Avg. Loss: 0.03471547071450914\n",
      "Epoch: 8, Batch: 3300, Avg. Loss: 0.03299397033144487\n",
      "Epoch: 8, Batch: 3600, Avg. Loss: 0.035196485685592055\n",
      "Epoch: 9, Batch: 300, Avg. Loss: 0.028607630795789495\n",
      "Epoch: 9, Batch: 600, Avg. Loss: 0.03114552316556607\n",
      "Epoch: 9, Batch: 900, Avg. Loss: 0.030606265409781674\n",
      "Epoch: 9, Batch: 1200, Avg. Loss: 0.03141503588003009\n",
      "Epoch: 9, Batch: 1500, Avg. Loss: 0.03192144944883087\n",
      "Epoch: 9, Batch: 1800, Avg. Loss: 0.03284474788864221\n",
      "Epoch: 9, Batch: 2100, Avg. Loss: 0.0306992270166423\n",
      "Epoch: 9, Batch: 2400, Avg. Loss: 0.03216731966424656\n",
      "Epoch: 9, Batch: 2700, Avg. Loss: 0.03249075842599268\n",
      "Epoch: 9, Batch: 3000, Avg. Loss: 0.0326811805353888\n",
      "Epoch: 9, Batch: 3300, Avg. Loss: 0.03377307411666455\n",
      "Epoch: 9, Batch: 3600, Avg. Loss: 0.033625676376201834\n",
      "Epoch: 10, Batch: 300, Avg. Loss: 0.031546816158806903\n",
      "Epoch: 10, Batch: 600, Avg. Loss: 0.031516385850676065\n",
      "Epoch: 10, Batch: 900, Avg. Loss: 0.027381662247983815\n",
      "Epoch: 10, Batch: 1200, Avg. Loss: 0.030159426596655978\n",
      "Epoch: 10, Batch: 1500, Avg. Loss: 0.030227046880924414\n",
      "Epoch: 10, Batch: 1800, Avg. Loss: 0.029143079974043456\n",
      "Epoch: 10, Batch: 2100, Avg. Loss: 0.03069378148567921\n",
      "Epoch: 10, Batch: 2400, Avg. Loss: 0.03197359722306525\n",
      "Epoch: 10, Batch: 2700, Avg. Loss: 0.03228701721294783\n",
      "Epoch: 10, Batch: 3000, Avg. Loss: 0.03070347033700576\n",
      "Epoch: 10, Batch: 3300, Avg. Loss: 0.032962560138306195\n",
      "Epoch: 10, Batch: 3600, Avg. Loss: 0.03134528333985751\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "n_epoch = 10\n",
    "print_every = 300\n",
    "\n",
    "loss_over_time = [] # to track the loss as the network trains\n",
    "    \n",
    "for epoch in range(n_epoch):\n",
    "    \n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for batch_i, (input_data, labels) in enumerate(train_loader):\n",
    "        # Zero gradients (just in case)\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # Forward pass, calculate predictions\n",
    "        output = lstm_model(input_data) \n",
    "        # Calculate loss\n",
    "        loss = criterion(output, labels)\n",
    "        ## Backward propagation\n",
    "        loss.backward()\n",
    "        ## Upade weights\n",
    "        optimizer.step()\n",
    "        \n",
    "        # print loss statistics\n",
    "        # to convert loss into a scalar and add it to running_loss, we use .item()\n",
    "        running_loss += loss.item()\n",
    "            \n",
    "        \n",
    "        if batch_i % print_every ==  print_every - 1:    # print everyx batches (\n",
    "                avg_loss = running_loss/print_every\n",
    "                # record and print the avg loss over the 100 batches\n",
    "                loss_over_time.append(avg_loss)\n",
    "                print('Epoch: {}, Batch: {}, Avg. Loss: {}'.format(epoch + 1, batch_i+1, avg_loss))\n",
    "                running_loss = 0.0\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hUVf7H8fd3ZpJJTyANktCroUMIRWyoCBZwXVDQVXBZ0fXnFtddy65r23VdXVdXV3RFQVRURGyoWBCwgBgInQQioaVAIJQktPTz+2OGkAZMGkNuvq/n4XHmtjmHi585c+6554oxBqWUUtZl83YBlFJKNS0NeqWUsjgNeqWUsjgNeqWUsjgNeqWUsjiHtwtQXUREhOnYsaO3i6GUUs3K6tWr9xtjImtbd84FfceOHUlOTvZ2MZRSqlkRkV2nWqddN0opZXEa9EopZXEa9EopZXEa9EopZXEa9EopZXEeBb2IjBaRNBFJF5H7a1l/oYisEZFSERlfy/oQEckSkRcao9BKKaU8d8agFxE7MB0YA8QDk0QkvtpmGcAU4O1THOZvwHf1L6ZSSqn68qRFnwikG2O2G2OKgbnAuMobGGN2GmM2AOXVdxaRQUA08FUjlPeUjhSV8syin1iXmdeUH6OUUs2OJ0EfC2RWep/lXnZGImID/g388QzbTRORZBFJzs3N9eTQNZSUlvP84q2syzhUr/2VUsqqmvpi7J3AQmNM1uk2MsbMMMYkGGMSIiNrvYP3jJw+rqoUldb4UaGUUi2aJ1MgZAPtKr2Pcy/zxDDgAhG5EwgCfEXkiDGmxgXdhvJz2AEoLNGgV0qpyjwJ+lVANxHphCvgJwI3enJwY8xNJ16LyBQgoSlCHsBmE3ztNgpLy5ri8Eop1WydsevGGFMK3AV8CWwG5hljUkTkMREZCyAig0UkC5gAvCwiKU1Z6FNxOmwUaYteKaWq8Gj2SmPMQmBhtWUPVXq9CleXzumOMRuYXecS1oHTx64teqWUqsZSd8b6+dgoLNGgV0qpyiwV9Np1o5RSNVkq6P187BRp141SSlVhuaDX4ZVKKVWVpYLe6dA+eqWUqs5SQe/qutEWvVJKVWaxoNcWvVJKVWepoHc6dBy9UkpVZ6mg9/PR4ZVKKVWdpYLe6bBr141SSlVjraD3sVGoF2OVUqoKSwW9n8NOcWk5xhhvF0Uppc4Z1gp6H9ec9DrEUimlTrJU0DsdrupoP71SSp1kqaDXFr1SStVksaDXFr1SSlVnqaB36nNjlVKqBksF/YkWvU5VrJRSJ1ks6LVFr5RS1Vkq6HXUjVJK1WSpoNdRN0opVZPFgl5b9EopVZ2lgv7kqBsNeqWUOsFaQV8x6ka7bpRS6gRLBf3JUTfaoldKqRMsFfQnRt1oi14ppU7yKOhFZLSIpIlIuojcX8v6C0VkjYiUisj4Ssv7i8gKEUkRkQ0ickNjFr46X7sNESjSFr1SSlU4Y9CLiB2YDowB4oFJIhJfbbMMYArwdrXlx4BbjDG9gNHAf0QkrKGFPk1Z8XPY9eEjSilVicODbRKBdGPMdgARmQuMA1JPbGCM2eleVyVhjTE/VXq9W0T2AZFAXoNLfgpOH5v20SulVCWedN3EApmV3me5l9WJiCQCvsC2WtZNE5FkEUnOzc2t66Gr8HPY9QHhSilVyVm5GCsibYE3gVuNMTVS2BgzwxiTYIxJiIyMbNBn+fnYKNRJzZRSqoInQZ8NtKv0Ps69zCMiEgJ8BvzFGPNj3YpXd06HXbtulFKqEk+CfhXQTUQ6iYgvMBFY4MnB3dt/CLxhjJlf/2J6zs/HpsMrlVKqkjMGvTGmFLgL+BLYDMwzxqSIyGMiMhZARAaLSBYwAXhZRFLcu18PXAhMEZF17j/9m6QmbtqiV0qpqjwZdYMxZiGwsNqyhyq9XoWrS6f6fnOAOQ0sY504fWwcLiw9mx+plFLnNEvdGQuuaRC060YppU6yXNA7HTa9M1YppSqxXND7+WgfvVJKVWbBoNdRN0opVZnlgl5H3SilVFWWC3rXnbHaoldKqROsF/QOO2XlhtIyDXullAILBv2Jxwlqq14ppVwsF/T6OEGllKrKekHvcAW9jrxRSikXywV9RdeNtuiVUgqwYtA7tOtGKaUqs1zQ+1W06LXrRimlwIJB76zoo9cWvVJKgQWD/kSLXp8bq5RSLhYMeu2jV0qpyiwX9E6Hu0WvwyuVUgqwYNBri14pparSoFdKKYuzXNBr141SSlVluaA/2aLXoFdKKbBg0Nttgo9dKNRx9EopBVgw6MF105SOo1dKKRdLBr3rKVPaoldKKbBo0OtzY5VS6iRrBr2PTUfdKKWUm0dBLyKjRSRNRNJF5P5a1l8oImtEpFRExldbN1lEtrr/TG6sgp+On8NOkbbolVIK8CDoRcQOTAfGAPHAJBGJr7ZZBjAFeLvavq2Bh4EhQCLwsIi0anixT8/Px6bDK5VSys2TFn0ikG6M2W6MKQbmAuMqb2CM2WmM2QBUT9crgEXGmIPGmEPAImB0I5T7tJwOu05TrJRSbp4EfSyQWel9lnuZJzzaV0SmiUiyiCTn5uZ6eOhT0xa9UkqddE5cjDXGzDDGJBhjEiIjIxt8PD8fHXWjlFIneBL02UC7Su/j3Ms80ZB9683p0FE3Sil1gidBvwroJiKdRMQXmAgs8PD4XwKjRKSV+yLsKPeyJqUteqWUOumMQW+MKQXuwhXQm4F5xpgUEXlMRMYCiMhgEckCJgAvi0iKe9+DwN9wfVmsAh5zL2tSGvRKKXWSw5ONjDELgYXVlj1U6fUqXN0yte07C5jVgDLWmdN9Mbas3GC3ydn8aKWUOuecExdjG1vvmFCKy8pJ3tnkPx6UUuqcZ8mgv6RnFE6Hjc835Xi7KEop5XWWDPogp4OLukfyxaYcysuNt4ujlFJeZcmgBxjTpw05BYWszczzdlGUUsqrLBv0l54XjY9d+GLTHm8XRSmlvMqyQR/i58MF3SJZuDEHY7T7RinVclk26AFG925Ddt5xNmbne7soSinlNZYO+lHx0dgEFm/e5+2iKKWU11g66MMCfIkJ8yfj4DFvF0UppbzG0kEPEBPmT/ah494uhlJKeY3lgz42zJ/sPA16pVTLZfmgjwnzI6egkDK9cUop1UK1gKD3p6zcsO9wobeLopRSXmH5oI8N8wdgt3bfKKVaqBYT9Fl6QVYp1UJZPujbVrTotetGKdUyWT7og5wOQv19tOtGKdViWT7owXVBVoNeKdVStYig17H0SqmWrIUEvZ8GvVKqxWoRQR8T5s/hwlIKCku8XRSllDrrWkzQA+zRkTdKqRaoRQW9XpBVSrVELSLo41q5gl776ZVSLVGLCPrIICc+dtEWvVKqRWoRQW+zCW1CdeSNUqpl8ijoRWS0iKSJSLqI3F/LeqeIvOtenyQiHd3LfUTkdRHZKCKbReSBxi2+52JC9aYppVTLdMagFxE7MB0YA8QDk0QkvtpmU4FDxpiuwLPAk+7lEwCnMaYPMAi4/cSXwNkWG+av890opVokT1r0iUC6MWa7MaYYmAuMq7bNOOB19+v5wKUiIoABAkXEAfgDxUBBo5S8jmLC/MkpKKS0rNwbH6+UUl7jSdDHApmV3me5l9W6jTGmFMgHwnGF/lFgD5ABPG2MOVj9A0Rkmogki0hybm5unSvhifbhAZSVG3YeONokx1dKqXNVU1+MTQTKgBigE3CPiHSuvpExZoYxJsEYkxAZGdkkBUno0AqApB01vmeUUsrSPAn6bKBdpfdx7mW1buPupgkFDgA3Al8YY0qMMfuA5UBCQwtdH50iAokMdpK0XYNeKdWyeBL0q4BuItJJRHyBicCCatssACa7X48HlhhjDK7umpEAIhIIDAW2NEbB60pEGNKpNUk7DuAqmlJKtQxnDHp3n/tdwJfAZmCeMSZFRB4TkbHuzWYC4SKSDvwBODEEczoQJCIpuL4wXjPGbGjsSnhqaOdw9hYUsevAMW8VQSmlzjqHJxsZYxYCC6ste6jS60JcQymr73ektuXeMrRzawCSdhygY0Sgl0ujlFJnR4u4M/aELpFBRAT5aj+9UqpFaVFBLyIkdmqtI2+UUi1Kiwp6gCGdwsnOO07mQe2nV0q1DC0v6N399D9uP+Dlkiil1NnR4oK+e1QwrQJ8tPtGKdVitLigt9mEoZ3DWbFNx9MrpVqGFhf0AMO7uPrpM7SfXinVArTMoO8aAcDydO2nV0pZX4sM+s4RgUSHOPlh235vF0UppZpciwx6EeH8LhGs2HaA8nLtp1dKWVuLDHqAYV3COXC0mJ/2HfZ2UZRSqkm12KDXfnqlVEvRYoM+NsyfjuEBrNB+eqWUxbXYoAdXqz5p+0F9jqxSytJadNCP6BrB4aJSPtu4x9tFUUqpJtOig35UfDT924Xx1482sTvvuLeLo5RSTaJFB73DbuO5if0pKzf8Yd46ynSopVLKglp00AN0CA/k4bG9+HH7QV7+bluVdceLy8g9XOSlkimlVONo8UEPMGFQHFf1bcvTX6axZMteAA4eLWbsC8sY9ey3HDpa7OUSKqVU/WnQ47pT9l/j+xIfE8Jv3l7Lyh0HuWVWEhkHj1FQWMrTX6V5u4hKKVVvGvRuAb4OXr1lMEF+Dq5/eQVpOYf53y8GccuwDry9MoNN2fneLqJSStWLBn0lbUL9mDl5MN2jg3h+4gAu6RnF7y/rTnigLw99vEnnxVFKNUsa9NX0jg3lq7svYkyftgCE+vtw3+ierMnI462kXV4unVJK1Z0GvQd+PjCOi7pH8tinqazepY8gVEo1Lxr0HrDZhOcnDiA2zJ875qwhJ7/Q20VSSimPadB7KDTAhxm3JHCsqJTb56ymuFTnx1FKNQ8eBb2IjBaRNBFJF5H7a1nvFJF33euTRKRjpXV9RWSFiKSIyEYR8Wu84p9d3aOD+ff1/Vifmce/dcilUqqZOGPQi4gdmA6MAeKBSSISX22zqcAhY0xX4FngSfe+DmAOcIcxphdwMVDSaKX3gtG923LTkPa8/N12lm3VKY6VUuc+T1r0iUC6MWa7MaYYmAuMq7bNOOB19+v5wKUiIsAoYIMxZj2AMeaAMaascYruPQ9eFU+3qCDunreOA0d0igSl1LnNk6CPBTIrvc9yL6t1G2NMKZAPhAPdASMiX4rIGhG5t+FF9j5/XzvPTxpA3rFiZny33dvFUUqp03KcheOPAAYDx4DFIrLaGLO48kYiMg2YBtC+ffsmLlLjOK9tCOe1DWHTbr1jVil1bvOkRZ8NtKv0Ps69rNZt3P3yocABXK3/74wx+40xx4CFwMDqH2CMmWGMSTDGJERGRta9Fl7SIzqYLXv04eJKqXObJ0G/CugmIp1ExBeYCCyots0CYLL79XhgiTHGAF8CfUQkwP0FcBGQ2jhF976ebUM4cLRYpzJWSp3Tzth1Y4wpFZG7cIW2HZhljEkRkceAZGPMAmAm8KaIpAMHcX0ZYIw5JCLP4PqyMMBCY8xnTVSXs65nm2AA0nIOExns9HJplFKqdh710RtjFuLqdqm87KFKrwuBCafYdw6uIZaW08Md9FtyChjRLcLLpVFKqdo19cVYS4sIchIR5Etazsl++ndXZZC04yB2EYL9fPjjFd0J8PXsr7mkrJw9eYW0a+2Pa3SqUko1nAZ9A/VoE0zaXlfQHysu5eEFKTgddvx8bOwtKKJ3bAjXDYzz6Fj/WLiZ15bvpEN4AKPio5k8vCNxrQKasvhKqRZA57ppoJ5tQvhp72HKyg3fpuVSWFLOSzcNZMX9lxId4uTrzXtr3a+83FBSdnK+nJ37j/Lmil1c0C2CjuGBzP5hJz9/6Qd27j96tqqilLIoDfoG6tEmmMKScnYdOMoXKTm0CvAhsVNrbDZhZM9ovvtpP0WlNW8GfuSTFEY8uYT0fa5fA//6Kg0fu41/T+jH679M5JPfjKCkzDDplR/ZdUDDXilVfxr0DXRi5M3G7HyWbN7HqPg2OOyuv9bL46M4UlRK0vaqc9hnHjzGW0kZ7C0oYtIrSXy0NpvPNuzhtgs7ExXi5z5uCHOmDuF4SRk3vpJEQWGzniJIKeVFGvQN1C0qGBGYtWwHh4tKGd2nTcW64V0i8Pex1+i+efGbdOw24c2piZSXG37/7joignyZdmHnKtvFx4Qwc3IC2XnHeScp46zURyllPRr0DeTva6djeCDrs/IJdjoY3iW8Yp2fj50R3SJYvHkfrvvHIDvvOPNXZzFxcDsu6BbJW7cNoXNEIH++8jyCnDWvjQ/q0JphncOZ/cPOKn36SinlKQ36RtAj2tV9c+l5UTgd9irrLj8vmuy842x2T5Xwv2+2AXDHRV0AVxfNkj9efNqRObdd2Ik9+YV8tmFPUxRfKWVxGvSNoGdbV9CP7t22xrpLekYhAtOXpnPPvPXMXZXB+EHtiAnz9/j4F3ePoktkIK98v73il4FSSnlKg74RXNWnLT8bEMvFPWpOyBYZ7GRQ+1Z8tnEPi7fs5co+bblnVPc6Hd9mE351QWdSdhewYvuBxiq2UqqFkHOthZiQkGCSk5O9XYxGlZNfSE5BIX1iQ7Hb6nfHa2FJGef/cwldooKYe9tQbPU8jlLKmtxTwCfUtk5b9GdBm1A/+rcLq3fIg+vC7h+v6MHKHQd5Z5WOwFFKeU6DvhmZOLgd53cN54mFW9idd9zbxVFKNRMa9M2IiPDP6/pSVm7484cb9cKsUsojGvTNTLvWAdw7ugffpOWyPL3mhVljDA99vInhTyzmiYWb+WmvPgFLqZZOg74ZmpTYnmCngwXrqz/REZ7+Ko03VuwiItjJzGU7GPXsd7yVtKvW4xhj2J57hLScw6TlHOawTrOglCXpNMXNkJ+Pnct7RfPFphz+dm3vipu0Zi7bwfSl25iU2J5//Kw3B44Wc8ebq3l+8VbGD4qrcTPXR+uyufvd9ZWOa+PqvjFMSmzPoA6tzmqdlFJNR1v0zdQ1/WIoKCzl+5/2A7B61yH+9mkqo3u14e/X9kZEiAhy8rvLurG3oIiP1tZs/b/1YwYdwwN48aaBvHDjAH42IJbPN+7h5y/9wB/mreNIUSkAKbvz+etHm9iee+Ss1lEp1Ti0Rd9MjegaQViAD59u2M3InlE89kkK0SFO/n19vyrDOEd0jaB3bAgvf7ud8YPaVaxL33eE5F2HuH9MT67s47qj9+q+MfzlqnhmfLuNF5amk7zzEL1jQ1i4MQeAtZmH+PDO8/Gxa/tAqeZE/49tpnzsNsb0bsOi1L28tTKD9Vn53De6J4HVJkYTEX59UVe27z/Klyk5FcvfS87EbhOuGxhbZfsgp4M/jOrBvNuHUVZu+CYtl7su6cq/xvdlU3YBLy7ddlbqp5RqPNqib8au6RvDOyszeWRBCv3bhXFt/9hatxvduw2dIgJ5YUk6I3tGYbcJ76/JYmTPKKKC/WrdJ6FjaxbfcxFFpeWE+vsAsCx9P/9dspXL4qPoFRPaZPVSSjUubdE3Y0M6hxMR5KSs3PDwNfGnnBbBbhP+cHl3UvcUMOF/K3jrx13sP1LMDQntTnt8Px97RcgDPDq2F60Cfbl3/gYdw69UM6JB34zZbcK9V/TgT1f0YED704+SuaZfDK/cksDO/Ud55JNUooKdtU7CdjphAb78aVQPUnYXsGrnoYYUXSl1FmnQN3PXD27H/13S1aNtL4+P5qO7zmdQh1bceXGXikce1sXV/doS7HQwd2XN+XbKyw0/pO9nT37V6Rme/jKNF5ZsrfNnKaUah/bRtzBdIoN4/9fD671/gK+DcQNieC85i4ev6UVogA+FJWW8szKD13/Yyc4Dx+gUEcjHd51PiJ9rVNALS9MB6BYdzBW92pzhE5RSjU1b9KrOJg5uT1FpOR+ty6a0rJxfz1nNo5+k0irQl/tG9yTz4DHumbeevQWFPPjRJvrFhdInNpT73t9ATn6hR59hjGHXgaPMXZnB45+l8sKSrbydlFHj14JS6sy0Ra/qrHesK7jfWZlB6u4Clqbl8rdre3Pz0A6A6w7bRz9JZX1mHseLy3jmhv4IcPV/l3H3u+uY86shp52yed/hQm6ZuZItOa55enztNordz8u9uEcks29NbPI6KmUlHrXoRWS0iKSJSLqI3F/LeqeIvOtenyQiHautby8iR0Tkj41TbOVtExPbsSXnMO8mZ/LbkV0rQh5gyvCOjO0Xw77DRTwwpiddIoPoHBnEI2N7sWL7Ae57fwNl5bWP2ikoLGHyrFVkHDzGI9fE8/UfLiTt76PZ8rfRTB3Rie+37ufAkaKzVU2lLOGMLXoRsQPTgcuBLGCViCwwxqRW2mwqcMgY01VEJgJPAjdUWv8M8HnjFVt529h+MTz39VYuPS+Kuy+v+mhEEeGp8X25PqEdw7uEVyyfMCiO3XnH+c/XWykuLeexcb34ZMMePt+4h/atA7igWyRv/riTrXsPM3PKYC7qfnJUkJ+PnQkJccxctoOFG/dw87COTVq/otIy7p2/gdJyw7PX98fXob2cqvnypOsmEUg3xmwHEJG5wDigctCPAx5xv54PvCAiYowxInItsAM42milVl4X7OfD8vtHnnI6BD8fOyO6RVRZJiL8/rLu+DpsPPVFGp9u2E25gS6RgWzMzmfuqkwA/nND/yohf0LPNiH0iA7m43W7GzXojTE89WUaPjZh2kVdcDps3PX2Whal7q1Y/99JAxv0hDClvMmToI8FMiu9zwKGnGobY0ypiOQD4SJSCNyH69fAKbttRGQaMA2gffv2HhdeeVd957y58+KutArwJXV3ARMS4ugTG0pZuWF9Vj7FpeUMq/QroLqx/WP415dpZB48RrvWAfX6/EWpe+kcGUiXyCAA/vP1Vl76xjW1w9srM+gSGUTSjoM8OrYXJWXl/P2zzQQ7N/LPn/dBpHHCfmNWPvuPFnFx98hGO6ZSp9LUF2MfAZ41xhw53T9mY8wMYAa4Hg7exGVS54BJiVW/0B128Whq5LH9XEH/yYbd3Hnx6e8f2JZ7hD9/sJHxg+KY4L4L+PUfdvLwghR8HTbuvaIHEUFOnnNP4/yLoR14/LNUknYc5MGrzmPy8I4AFBwv4fkl6Qzt0pqfDYirX4UrKSotY9qbyezJL2R4l3AevqYXPdoEN/i4Sp2KJ0GfDVS+Vz7Ovay2bbJExAGEAgdwtfzHi8hTQBhQLiKFxpgXGlxy1SK1ax3AoA6tWLDu9EG/JuMQU2evIv94CUk7DpJx8Bjdo4N55JMULu0ZhQj8/bPNACR2as0/ftYHX4eNebcPY9/hIqJDTs4BdPfl3Vm4KYdZy3Zybf/YBrfAP1iTzZ78Qm4Z1oEF63dz5fPfc/uFnSu6tZRqbJ4E/Sqgm4h0whXoE4Ebq22zAJgMrADGA0uMazKUC05sICKPAEc05FVDXds/hr9+nMLPX/qBq/u2pV+7MAQoLTdkHzrOttwjvPL9dqJD/HjvjuHM+G4b/13iumlrcMdWTL9pIE6HjfdWZ7F4816euK5vRcCKSJWQP7Fs8vCO/PWjTazJyGvQQ1lKy8p58Zt0+rUL49Gxvbj7su78Y+FmXvxmG0u27OPZG/pzXtuQeh9fqdqIJ5NTiciVwH8AOzDLGPO4iDwGJBtjFoiIH/AmMAA4CEw8cfG20jEewRX0T5/usxISEkxycnK9KqNahpKycl79fgcfr8uuGGtfXWKn1rx400AigpwYY/jft9tJ2nGA524YQGiAT637nM7RolKGPrGYi3tE8d9JA+q07+zlO7DZhEmJ7Vmwbjf3vLeeV29J4LL46IptFqXu5YEPNnKsuJQ5vxrCwDPMXdRUjhSVsig1h2v6xtRrigzlPSKy2hiTUOu6c20WQg16VRfbco+QcfAYAHYRYsL8iGsVgJ+P/Qx71t3jn6Xy2vKdLLtvJG1Ca5/eubrXlu/g0U9cA9Q6RwZSXFpOsJ8PC387okYX0N6CQq5/eQWHjhbz7u3DvNKyf+yTVGYt38GDV53Hry7ofNY/X9Xf6YJev7JVs9YlMohLekRxSY8oLuweSdeo4CYJeYBbhnWkzBjm/Fj1YeslZeVkHTpWY+rmRal7eezTVC6Pj+bVWxLAQNah49x1Sdda+/mjQ/yYM3UIAb4Obp6ZxEvfbGP+6iw2ZeefsWzHikuZuWwHeceKT7udMYaP12XzxaY9NaajOHCkiHdWZuCwCc8s+onsPJ1uwiq0Ra9UHUx7I5kftx9g8T0XExnsBOC++Rt4NzmTqGAnQzuHE+znoKCwlK9T99I9Ooh3pg0lwNdBSVk5qbsL6BsXetoLuun7jjDltZVkHXIFrcMmrPzLZbQO9D3lPtOXpvOvL9PoFRPCnKlDaHWKbWct28Fjn568BaZbVBCzf5lIbJg/T3+ZxvRv0pl9ayJ3vLma4V3CeXVywlkf/llaVs5ry3cyrn8MUSGe/XJS2qJXqtHcO7onhSXlPPpJCgDLtu7n3eRMxvRuw9DO4azccZAvNuWwMSuPwZ1a8+rkwQT4usY8+NhtrgvHZwjOrlFBLLtvJCmPXsHsWwdTWm5YvHlvxfoftu1n5NPfkOnusioqLWP2DzvpGhXE1n1HuPHVJA4erdmy/yZtH3//LJVR8dF8cOdwHro6npyCQm6emcSuA0d5fcVORvdqw0XdI7lnVHcWb9nH55tyahynMe3cf5R75q1n5/6T91O+8v0OHl+4ueIGOtVwOqmZUnXQNSqIu0Z25ZlFPzG6926e/GILnSICefaG/o3eZRTodHBR90jahvrxVereinsBXlu+k+37j/LQx5uYNWUwC9btJvdwEf+e0A+A295I5hevJvHu7UMJ9nNdeE7fd5jfvL2WHm1CePaG/gQ6HQxs34o+caHcPDOJK5/7nqPFZRVDVqcM78hH67J54IONxLcNoWNEYKPWDWBTdj5TXlvJ/iPFrM04xAd3Dmf/kSKeXfQTAGszzt7DbUrKyhGw7AVoa9ZKqSZ0x0Vd6BYVxG/eWUvmweP887o+TXZdQEQYFR/N91tzOV5cxv4jRSzdso/2rQNYmpbLpxv28Or3O+jZJpgLukVwYfdIXr55EGl7D3PnW2soLi1nTcYhrn/5R5w+Nl65ZVCVB8gP7tial24aRFFpOSXYPXUAAAxJSURBVBd1j6RPnOtZwA67jZduGoQI/OqNZA4XlgCucJ7z4y6mL03nic83M+O7bXy+cU/FrwtPJW0/wKQZP+Jrt/Gv8X3JOnScX89Zw5/mbyDAaefSnlGszcw7K4+s/Dp1L4mPf023Bz9n8ONfc8usleQfK2nyzz2btEWvVB35Omz88+d9mPC/Fdw4pD1DOp96yobGMKpXG15fsYvvt+aSeeg4peWGl28exL3zN/Cn+espLCnn6Qn9KrqELu4RxT+v68Of5m9gymsrWb3rEG1C/Zh9ayJxrWpOG3FJzygW/u6CGvcPtGsdwIs3DuTmWSv51evJFJWWsy4z7+TfQ6Xpo0P8HCT9+TL8fc/8hZe6u4Bfzl5F2zB/3pyaSNtQf9dzjeetB+C5if05XlzG4i372LH/KJ3dU1V4KnnnQfx87PSOPf0D7POPlzB9aTozvttO79gQbh7agT35hXy4Npu/fryJ5+s4jLY+9h8pYtayHUwd0YnwIGeTfY4GvVL1MKhDa7790yXEhPk3+WcldmpNiJ+Dr1L3VlzMPa9tCE9c14exLywjKtjJ2H4xVfaZkNCOPfmFPLPoJ/q3C2Pm5ITTBkn36NqnYBjeNYKHro7n4QUpdIoI5OFr4hnTuy2tAn1wOuzkHy/hm7R9/G7uOr5KzWFc/9jT1mVfQSFTX19FkJ+DOVOHVAxTvW5gHPnHS9iTX8jYfjFs3XcEgLUZeR4H/YEjRTz+2WY+WJtNqL8Pi+6+sOJibv7xEpZu2ceajEOsz8xjx/6jFBSWAjB5WAf+fNV5OB2uL6l2rQN4ZtFPXHpe1Bnr46kDR4pYn5XHseIyruzdFpt7grwHP9zEFyk5LE/fz9u3Da3ya6sx6agbpZqBu99dx8KNeygqLefRsb0q5uFZuHEPrQN9GVrLrwpjDKt2HqJvXGiDu5YyDx4jNsy/IqAqKy83XPDUUjpHBvLm1JPzHeYdKyYt5zBb9x2hsKQMp8PGvOQstuUeYd7tw07b4i4vN/R79CvG9o/h8Z/1OeV2xhhSdhfwyfrdzEvO5HBhKb8Y2oG5qzIY3iWCmZMTyD1cxMQZP7J9/1ECfO30jQulW1Qw7VsH0DcutMYvstKycia8vIL0fUeYM3UIrQN9CfHzqdeNdtl5x7nzrTWsr/RLaMrwjjx8TTxfpuRwx5w1jO7VhkWb93J+V1d56ztZ4OlG3WiLXqlmYFR8NB+uzcbHLlVa71f2aXvKfUSExE6tG+XzTzdTqM0mXDcwlulL08nJL6RNqF/F5HHV+diFl24adMZuFZtN6N8+jLUZJwNy8ea9hPr7MKhDK0SEZVv388Tnm0nZXYDDJlzUPZJ7R/ekRxtXiD/2aSovfeu6FyGnoJDZtw7mgm6RZ5xu2mG38Z8b+jPmue8ZN325qzwCn/7mAuJjPL+JLWV3Pre+torjJWXcO7oHA9u34quUvcxavgM/Hzvvr8kivm0I/71xAB+syeK+9zdy3/wNPD2hX61fqA2hQa9UM3Bh90icDhuX9Ig65Rh5b7puYBz/XZLOh2uzubhHJI9/tpkLukUwdUQnukcHE+TnoKikHF+HjVB/z1rGA9qF8cLSdI4WlbIl5zBTX3f90u8YHkDbUH9WbD9Au9b+/P3a3lzZp22V+wymDO/Ilyk5PPVFGgG+dmbfmlinL70O4YF88psRbMzKp6i0jAc/2sQHa7KIj4n3aP8ftu3ntteTCfX34f1fD6/oGkvs2Jq8Y8X879tt2G3Ca1MG42O3ccPg9uQeLuJwUSlNcduCBr1SzUCg08HcaUOJbdX01wTqo1NEIIM6tOK91Zl8vC6bEH8fnps4oOpNXnW892lAh1aUG1ifmceTX2whOsTJPZf34MO12ezYf5QHrzqPm4d1qOhbr8xmE56e0I8HP9rEnRd3qdcvmy6RQRXPLPh68z4+2bCbB648D7tNyD9ewqMLUsg8dIyjRWV0CA/g0XG9iAr2Y03GIX71ejJxrfx545dDqkyXYbMJT47vi5+vnW5RQVV+2fzfKe6YbgzaR6+UahRvJ2Xw5w83AvDalMFc0jOqQcfLO1ZM/8cW0SsmhJTdBTxzfT+uG9jw5wHUx6cbdnPX22t5+7YhDO8SwVNfbOHFb7YxrHM4Ab52lm/bT5DTwe8v685TX2yhVaAv790xjKjgs3dnr94Zq5Rqclf1bUuwn4Obh3ZocMgDhAX40jkykJTdBfRrF8a1jTQCpj4u7RlNoK+dBet2s+9wIa8t38nYfjG8M20oM6cMZsFdI2gV4MuDH20iwNc1ouhshvyZaNeNUqpRhPr7sOzekQT7NV6sDGzfiu25R3no6vhGv0BZF/6+dq7o1YaFG/cgIhSXlfOHy7tXrO8eHczHd53PGyt2cUWvNvV+zGVT0aBXSjWa+gxBPJ3fjOzKyJ5RDXrYS2MZ2z+GD9Zm887KDCYltq8xLUSAr4M7LuripdKdnga9Uuqc1SE8kA7hjT/PTn2c3zWC8EBfDheV8ttLT/+84nONBr1SSnnAx27jkbG9KCkrp23ouTn66VQ06JVSykPXVJtqornQUTdKKWVxGvRKKWVxGvRKKWVxGvRKKWVxGvRKKWVxGvRKKWVxGvRKKWVxGvRKKWVx59w0xSKSC+xqwCEigP2NVBxv07qcm7Qu5yYr1QXqXp8OxpjI2lacc0HfUCKSfKo5mZsbrcu5SetybrJSXaBx66NdN0opZXEa9EopZXFWDPoZ3i5AI9K6nJu0LucmK9UFGrE+luujV0opVZUVW/RKKaUq0aBXSimLs0zQi8hoEUkTkXQRud/b5akLEWknIktFJFVEUkTkd+7lrUVkkYhsdf/X+w/O9JCI2EVkrYh86n7fSUSS3OfnXRHx9XYZPSUiYSIyX0S2iMhmERnWXM+NiNzt/je2SUTeERG/5nJuRGSWiOwTkU2VltV6HsTleXedNojIQO+VvKZT1OVf7n9jG0TkQxEJq7TuAXdd0kTkirp+niWCXkTswHRgDBAPTBKReO+Wqk5KgXuMMfHAUOD/3OW/H1hsjOkGLHa/by5+B2yu9P5J4FljTFfgEDDVK6Wqn+eAL4wxPYF+uOrV7M6NiMQCvwUSjDG9ATswkeZzbmYDo6stO9V5GAN0c/+ZBrx0lsroqdnUrMsioLcxpi/wE/AAgDsLJgK93Pu86M48j1ki6IFEIN0Ys90YUwzMBcZ5uUweM8bsMcascb8+jCtIYnHV4XX3Zq8D13qnhHUjInHAVcCr7vcCjATmuzdpTnUJBS4EZgIYY4qNMXk003OD6/Gh/iLiAAKAPTSTc2OM+Q44WG3xqc7DOOAN4/IjECYibc9OSc+stroYY74yxpS63/4IxLlfjwPmGmOKjDE7gHRcmecxqwR9LJBZ6X2We1mzIyIdgQFAEhBtjNnjXpUDRHupWHX1H+BeoNz9PhzIq/SPuDmdn05ALvCauyvqVREJpBmeG2NMNvA0kIEr4POB1TTfcwOnPg/NPRN+CXzuft3gulgl6C1BRIKA94HfG2MKKq8zrnGw5/xYWBG5GthnjFnt7bI0EgcwEHjJGDMAOEq1bppmdG5a4WoddgJigEBqdh80W83lPJyJiPwFV3fuW411TKsEfTbQrtL7OPeyZkNEfHCF/FvGmA/ci/ee+Lnp/u8+b5WvDs4HxorITlxdaCNx9XGHubsLoHmdnywgyxiT5H4/H1fwN8dzcxmwwxiTa4wpAT7Adb6a67mBU5+HZpkJIjIFuBq4yZy8yanBdbFK0K8CurlHD/jiunCxwMtl8pi7D3smsNkY80ylVQuAye7Xk4GPz3bZ6soY84AxJs4Y0xHXeVhijLkJWAqMd2/WLOoCYIzJATJFpId70aVAKs3w3ODqshkqIgHuf3Mn6tIsz43bqc7DAuAW9+iboUB+pS6ec5KIjMbV5TnWGHOs0qoFwEQRcYpIJ1wXmFfW6eDGGEv8Aa7EdaV6G/AXb5enjmUfgesn5wZgnfvPlbj6thcDW4GvgdbeLmsd63Ux8Kn7dWf3P8504D3A6e3y1aEe/YFk9/n5CGjVXM8N8CiwBdgEvAk4m8u5Ad7BdW2hBNcvramnOg+A4BqJtw3YiGukkdfrcIa6pOPqiz+RAf+rtP1f3HVJA8bU9fN0CgSllLI4q3TdKKWUOgUNeqWUsjgNeqWUsjgNeqWUsjgNeqWUsjgNeqWUsjgNeqWUsrj/Bx9ejIdhFNEuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(loss_over_time)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print model's state_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embedding.weight \t torch.Size([153998, 3])\n",
      "lstm.weight_ih_l0 \t torch.Size([16, 3])\n",
      "lstm.weight_hh_l0 \t torch.Size([16, 4])\n",
      "lstm.bias_ih_l0 \t torch.Size([16])\n",
      "lstm.bias_hh_l0 \t torch.Size([16])\n",
      "fc.weight \t torch.Size([6, 4])\n",
      "fc.bias \t torch.Size([6])\n"
     ]
    }
   ],
   "source": [
    "for param_tensor in lstm_model.state_dict():\n",
    "    print(param_tensor, \"\\t\", lstm_model.state_dict()[param_tensor].size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(lstm_model.state_dict(), \"models/lstm_model2.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n",
    "\n",
    "def evaluate_classification(y, y_hat, y_proba):\n",
    "    return {\n",
    "        \"Accuracy\": accuracy_score(y, y_hat),\n",
    "        \"Precision\": precision_score(y, y_hat),\n",
    "        \"Recall\": recall_score(y, y_hat),\n",
    "        \"F1-score\": f1_score(y, y_hat),\n",
    "        \"AUC\": roc_auc_score(y, y_proba),\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.000002\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yc00122/.virtualenvs/py3.6.8-pytorch/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/Users/yc00122/.virtualenvs/py3.6.8-pytorch/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/Users/yc00122/.virtualenvs/py3.6.8-pytorch/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/Users/yc00122/.virtualenvs/py3.6.8-pytorch/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "num_classes = len(label_colnames)\n",
    "# initialize tensor and lists to monitor test loss and accuracy\n",
    "test_loss = torch.zeros(1)\n",
    "class_correct = list(0. for i in range(num_classes))\n",
    "class_total = list(0. for i in range(num_classes))\n",
    "\n",
    "# set the module to evaluation mode\n",
    "lstm_model.eval()\n",
    "\n",
    "# get the input images and their corresponding labels\n",
    "inputs, labels = test_loader.dataset.tensors\n",
    "\n",
    "# forward pass to get outputs\n",
    "outputs = lstm_model(inputs)\n",
    "\n",
    "# calculate the loss\n",
    "loss = criterion(outputs, labels)\n",
    "\n",
    "# update average test loss \n",
    "test_loss = test_loss + ((torch.ones(1) / (len(labels) + 1)) * (loss.data - test_loss))\n",
    "\n",
    "# get the predicted class from the maximum value in the output-list of class scores\n",
    "metrics = {}\n",
    "for j in range(num_classes):\n",
    "    # compare predictions to true label\n",
    "    predicted_class = np.round(outputs.data[:,j])\n",
    "    labels_class = labels.data[:,j]\n",
    "    class_total[j] = len(labels)\n",
    "    class_correct[j] = (labels_class==predicted_class).sum()\n",
    "    metrics[label_colnames[j]] = evaluate_classification(labels_class, predicted_class, outputs.data[:,j])\n",
    "    #(predicted_class == labels_class).sum()\n",
    "              \n",
    "print('Test Loss: {:.6f}\\n'.format(test_loss.numpy()[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "toxic\n",
      "{'Accuracy': 0.9471586493871105, 'Precision': 0.7083130772970345, 'Recall': 0.7624280481423339, 'F1-score': 0.7343749999999999, 'AUC': 0.947535114514326}\n",
      "severe_toxic\n",
      "{'Accuracy': 0.9900233123605645, 'Precision': 0.5, 'Recall': 0.2663316582914573, 'F1-score': 0.3475409836065574, 'AUC': 0.9861518950620937}\n",
      "obscene\n",
      "{'Accuracy': 0.975785225478154, 'Precision': 0.7770034843205574, 'Recall': 0.7508417508417509, 'F1-score': 0.7636986301369861, 'AUC': 0.9764002317818363}\n",
      "threat\n",
      "{'Accuracy': 0.9970420875842879, 'Precision': 0.0, 'Recall': 0.0, 'F1-score': 0.0, 'AUC': 0.9515710191863128}\n",
      "insult\n",
      "{'Accuracy': 0.9699947359185822, 'Precision': 0.7064372918978913, 'Recall': 0.6558475012879958, 'F1-score': 0.6802030456852792, 'AUC': 0.9676557668657957}\n",
      "identity_hate\n",
      "{'Accuracy': 0.9911262627528639, 'Precision': 0.0, 'Recall': 0.0, 'F1-score': 0.0, 'AUC': 0.9531048726402295}\n"
     ]
    }
   ],
   "source": [
    "for label in metrics:\n",
    "    print(label)\n",
    "    print(metrics[label])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Accuracy': 0.9785217122469273, 'Precision': 0.44862564225258056, 'Recall': 0.40590815976058964, 'F1-score': 0.42096960990480375, 'AUC': 0.9637364833417656}\n"
     ]
    }
   ],
   "source": [
    "total_evaluation = {}\n",
    "for metric in metrics['toxic']:\n",
    "    total_evaluation[metric] = 0\n",
    "    for label in metrics:\n",
    "        total_evaluation[metric] += metrics[label][metric]\n",
    "    total_evaluation[metric] /= num_classes\n",
    "print(total_evaluation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final project\n",
    "\n",
    "* Reduce vocabulary size by removing very unfrequent words\n",
    "* Tokenize dots, comas, etc, into `<dot>`, `<coma>` respectively.\n",
    "* Add another layer of LSTM\n",
    "* Rebalance tran dataset by repeating positive example \n",
    "* Play with parameters.\n",
    "\n",
    "## Extra\n",
    "\n",
    "One of the following:\n",
    "* Submit to kaggle\n",
    "* Create generator of comments."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}