Sentence Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# load all sentences that occured 8 times or more\n",
    "sentences_df = pd.read_csv('data/BookCorpus/sentences_counted_8+.csv')\n",
    "\n",
    "# load \"bad words\"\n",
    "bad_words = pd.read_csv('https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/en',header=None)\n",
    "bad_words.columns = ['word']\n",
    "\n",
    "# compute bad word flag\n",
    "bad_words_list = bad_words.word.tolist()\n",
    "sentences_df['contains_bad_word'] = sentences_df.sentence.str.contains(' | '.join(bad_words_list))\n",
    "\n",
    "# compute space count (sentence length)\n",
    "sentences_df['space_count'] = sentences_df.sentence.str.count(' ')\n",
    "sentences_df = sentences_df[['sentence','n','contains_bad_word','space_count']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overview and samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Note these are only sentences that occurred 8 or more times, not all sentences in the dataset*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>`` what 's that supposed to mean ? ''</td>\n",
       "      <td>1408</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>thank you for respecting the hard work of this...</td>\n",
       "      <td>1345</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>555</th>\n",
       "      <td>this ebook may not be re-sold or given away to...</td>\n",
       "      <td>1302</td>\n",
       "      <td>False</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625</th>\n",
       "      <td>if you would like to share this book with anot...</td>\n",
       "      <td>1193</td>\n",
       "      <td>False</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>674</th>\n",
       "      <td>`` what are you going to do ? ''</td>\n",
       "      <td>1113</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1003</th>\n",
       "      <td>if youre reading this book and did not purchas...</td>\n",
       "      <td>788</td>\n",
       "      <td>False</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053</th>\n",
       "      <td>`` what the hell are you doing ? ''</td>\n",
       "      <td>760</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1056</th>\n",
       "      <td>`` i do n't know what you 're talking about . ''</td>\n",
       "      <td>759</td>\n",
       "      <td>False</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1121</th>\n",
       "      <td>`` why did n't you tell me ? ''</td>\n",
       "      <td>721</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1468</th>\n",
       "      <td>`` what are we going to do ? ''</td>\n",
       "      <td>568</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence     n  \\\n",
       "494               `` what 's that supposed to mean ? ''  1408   \n",
       "536   thank you for respecting the hard work of this...  1345   \n",
       "555   this ebook may not be re-sold or given away to...  1302   \n",
       "625   if you would like to share this book with anot...  1193   \n",
       "674                    `` what are you going to do ? ''  1113   \n",
       "1003  if youre reading this book and did not purchas...   788   \n",
       "1053                `` what the hell are you doing ? ''   760   \n",
       "1056   `` i do n't know what you 're talking about . ''   759   \n",
       "1121                    `` why did n't you tell me ? ''   721   \n",
       "1468                    `` what are we going to do ? ''   568   \n",
       "\n",
       "      contains_bad_word  space_count  \n",
       "494               False            8  \n",
       "536               False           10  \n",
       "555               False           12  \n",
       "625               False           20  \n",
       "674               False            8  \n",
       "1003              False           31  \n",
       "1053              False            8  \n",
       "1056              False           11  \n",
       "1121              False            8  \n",
       "1468              False            8  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common sentences with at least 8 spaces\n",
    "sentences_df[(sentences_df.space_count>7)].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>52530</th>\n",
       "      <td>`` yeah , i 'll be fine . ''</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101240</th>\n",
       "      <td>you do n't have to worry about that . ''</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107979</th>\n",
       "      <td>`` she 's right , you know . ''</td>\n",
       "      <td>14</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148083</th>\n",
       "      <td>the hair rose on the back of my neck .</td>\n",
       "      <td>11</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200132</th>\n",
       "      <td>`` i 'll go with you , '' he said .</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189487</th>\n",
       "      <td>`` i 've had a lot of practice . ''</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204449</th>\n",
       "      <td>but in spite of wielding the knife and assurin...</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236182</th>\n",
       "      <td>i fought for it for an entire century , practi...</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225925</th>\n",
       "      <td>`` i was thinking , '' she said .</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232926</th>\n",
       "      <td>he caught me staring at him and smiled .</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 sentence   n  \\\n",
       "52530                        `` yeah , i 'll be fine . ''  27   \n",
       "101240           you do n't have to worry about that . ''  15   \n",
       "107979                    `` she 's right , you know . ''  14   \n",
       "148083             the hair rose on the back of my neck .  11   \n",
       "200132                `` i 'll go with you , '' he said .   9   \n",
       "189487                `` i 've had a lot of practice . ''   9   \n",
       "204449  but in spite of wielding the knife and assurin...   8   \n",
       "236182  i fought for it for an entire century , practi...   8   \n",
       "225925                  `` i was thinking , '' she said .   8   \n",
       "232926           he caught me staring at him and smiled .   8   \n",
       "\n",
       "        contains_bad_word  space_count  \n",
       "52530               False            8  \n",
       "101240              False            9  \n",
       "107979              False            8  \n",
       "148083              False            9  \n",
       "200132              False           10  \n",
       "189487              False            9  \n",
       "204449              False           35  \n",
       "236182              False           16  \n",
       "225925              False            8  \n",
       "232926              False            8  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# random sample of common sentences with at least 7 spaces\n",
    "sentences_df[(sentences_df.space_count>7)].sample(10).sort_values(by='n',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18074</th>\n",
       "      <td>i did n't have to wait long .</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22506</th>\n",
       "      <td>`` no , my lady .</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26340</th>\n",
       "      <td>he needed this .</td>\n",
       "      <td>49</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78048</th>\n",
       "      <td>`` what if you 're wrong ?</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129448</th>\n",
       "      <td>`` no , '' she said emphatically .</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130603</th>\n",
       "      <td>you just do n't . ''</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128519</th>\n",
       "      <td>julia rolled her eyes .</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175235</th>\n",
       "      <td>`` i 'm so glad you made it . ''</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178797</th>\n",
       "      <td>you will need your strength .</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207958</th>\n",
       "      <td>`` stevie ! ''</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  sentence   n  contains_bad_word  space_count\n",
       "18074        i did n't have to wait long .  67              False            7\n",
       "22506                    `` no , my lady .  56              False            5\n",
       "26340                     he needed this .  49              False            3\n",
       "78048           `` what if you 're wrong ?  19              False            6\n",
       "129448  `` no , '' she said emphatically .  12              False            7\n",
       "130603                you just do n't . ''  12              False            5\n",
       "128519             julia rolled her eyes .  12              False            4\n",
       "175235    `` i 'm so glad you made it . ''  10              False            9\n",
       "178797       you will need your strength .   9              False            5\n",
       "207958                      `` stevie ! ''   8              False            3"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# random sample of all common sentences\n",
    "sentences_df.sample(10).sort_values(by='n',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Noisy / redundant sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1003</th>\n",
       "      <td>if youre reading this book and did not purchas...</td>\n",
       "      <td>788</td>\n",
       "      <td>False</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9306</th>\n",
       "      <td>if you enjoyed this book , please encourage yo...</td>\n",
       "      <td>119</td>\n",
       "      <td>False</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10068</th>\n",
       "      <td>this book remains the copyrighted property of ...</td>\n",
       "      <td>111</td>\n",
       "      <td>False</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10340</th>\n",
       "      <td>although this is a free book , it remains the ...</td>\n",
       "      <td>109</td>\n",
       "      <td>False</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>smashwords edition , license notes this ebook ...</td>\n",
       "      <td>93</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15967</th>\n",
       "      <td>if you enjoyed this book , please return to sm...</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16523</th>\n",
       "      <td>if you 're reading this book and did not purch...</td>\n",
       "      <td>73</td>\n",
       "      <td>False</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16902</th>\n",
       "      <td>if you 're reading this book and did not purch...</td>\n",
       "      <td>72</td>\n",
       "      <td>False</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23623</th>\n",
       "      <td>if youre reading this book and did not purchas...</td>\n",
       "      <td>54</td>\n",
       "      <td>False</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26176</th>\n",
       "      <td>without limiting the rights under copyright re...</td>\n",
       "      <td>49</td>\n",
       "      <td>False</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                sentence    n  \\\n",
       "1003   if youre reading this book and did not purchas...  788   \n",
       "9306   if you enjoyed this book , please encourage yo...  119   \n",
       "10068  this book remains the copyrighted property of ...  111   \n",
       "10340  although this is a free book , it remains the ...  109   \n",
       "12408  smashwords edition , license notes this ebook ...   93   \n",
       "15967  if you enjoyed this book , please return to sm...   75   \n",
       "16523  if you 're reading this book and did not purch...   73   \n",
       "16902  if you 're reading this book and did not purch...   72   \n",
       "23623  if youre reading this book and did not purchas...   54   \n",
       "26176  without limiting the rights under copyright re...   49   \n",
       "\n",
       "       contains_bad_word  space_count  \n",
       "1003               False           31  \n",
       "9306               False           28  \n",
       "10068              False           22  \n",
       "10340              False           30  \n",
       "12408              False           14  \n",
       "15967              False           17  \n",
       "16523              False           32  \n",
       "16902              False           33  \n",
       "23623              False           32  \n",
       "26176              False           67  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# copyright sentences\n",
    "sentences_df[sentences_df.sentence.str.contains('copyright|smashwords')].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>153515</th>\n",
       "      <td>end if you enjoyed reading this book , you mig...</td>\n",
       "      <td>11</td>\n",
       "      <td>False</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 sentence   n  \\\n",
       "153515  end if you enjoyed reading this book , you mig...  11   \n",
       "\n",
       "        contains_bad_word  space_count  \n",
       "153515              False           29  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sentences about \"the author\"\n",
    "sentences_df[sentences_df.sentence.str.contains('other books')].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lite gender analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1503</th>\n",
       "      <td>`` i 'm sorry , '' he said .</td>\n",
       "      <td>555</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2657</th>\n",
       "      <td>`` i do n't know , '' he said .</td>\n",
       "      <td>343</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5570</th>\n",
       "      <td>`` i love you , '' he said .</td>\n",
       "      <td>184</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8543</th>\n",
       "      <td>`` what did he do to you ? ''</td>\n",
       "      <td>127</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8626</th>\n",
       "      <td>`` do n't worry , '' he said .</td>\n",
       "      <td>126</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9043</th>\n",
       "      <td>`` i love you , '' he whispered .</td>\n",
       "      <td>121</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9157</th>\n",
       "      <td>`` do you know where he is ? ''</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9666</th>\n",
       "      <td>`` you 're right , '' he said .</td>\n",
       "      <td>115</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10562</th>\n",
       "      <td>`` i 'm sorry , '' he whispered .</td>\n",
       "      <td>107</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11811</th>\n",
       "      <td>`` it 's okay , '' he said .</td>\n",
       "      <td>97</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                sentence    n  contains_bad_word  space_count\n",
       "1503        `` i 'm sorry , '' he said .  555              False            8\n",
       "2657     `` i do n't know , '' he said .  343              False            9\n",
       "5570        `` i love you , '' he said .  184              False            8\n",
       "8543       `` what did he do to you ? ''  127              False            8\n",
       "8626      `` do n't worry , '' he said .  126              False            8\n",
       "9043   `` i love you , '' he whispered .  121              False            8\n",
       "9157     `` do you know where he is ? ''  120              False            8\n",
       "9666     `` you 're right , '' he said .  115              False            8\n",
       "10562  `` i 'm sorry , '' he whispered .  107              False            8\n",
       "11811       `` it 's okay , '' he said .   97              False            8"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common sentences containing \"he\"\n",
    "sentences_df[(sentences_df.space_count>7)&(sentences_df.sentence.str.contains(' he '))].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>`` i 'm sorry , '' she said .</td>\n",
       "      <td>443</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3041</th>\n",
       "      <td>`` i do n't know , '' she said .</td>\n",
       "      <td>306</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4265</th>\n",
       "      <td>`` i 'm sorry , '' she whispered .</td>\n",
       "      <td>231</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8041</th>\n",
       "      <td>`` i love you , '' she whispered .</td>\n",
       "      <td>134</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9077</th>\n",
       "      <td>`` i love you , '' she said .</td>\n",
       "      <td>121</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9415</th>\n",
       "      <td>`` you 're right , '' she said .</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9780</th>\n",
       "      <td>`` i 'm fine , '' she said .</td>\n",
       "      <td>114</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9978</th>\n",
       "      <td>`` it 's okay , '' she said .</td>\n",
       "      <td>112</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13399</th>\n",
       "      <td>`` i ca n't , '' she said .</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15938</th>\n",
       "      <td>`` thank you , '' she said softly .</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  sentence    n  contains_bad_word  \\\n",
       "1996         `` i 'm sorry , '' she said .  443              False   \n",
       "3041      `` i do n't know , '' she said .  306              False   \n",
       "4265    `` i 'm sorry , '' she whispered .  231              False   \n",
       "8041    `` i love you , '' she whispered .  134              False   \n",
       "9077         `` i love you , '' she said .  121              False   \n",
       "9415      `` you 're right , '' she said .  117              False   \n",
       "9780          `` i 'm fine , '' she said .  114              False   \n",
       "9978         `` it 's okay , '' she said .  112              False   \n",
       "13399          `` i ca n't , '' she said .   87              False   \n",
       "15938  `` thank you , '' she said softly .   75              False   \n",
       "\n",
       "       space_count  \n",
       "1996             8  \n",
       "3041             9  \n",
       "4265             8  \n",
       "8041             8  \n",
       "9077             8  \n",
       "9415             8  \n",
       "9780             8  \n",
       "9978             8  \n",
       "13399            8  \n",
       "15938            8  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common sentences containing \"she\"\n",
    "sentences_df[(sentences_df.space_count>7)&(sentences_df.sentence.str.contains(' she '))].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lite \"bad word\" analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2340</th>\n",
       "      <td>`` what the fuck ? ''</td>\n",
       "      <td>383</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3809</th>\n",
       "      <td>`` son of a bitch ! ''</td>\n",
       "      <td>256</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3908</th>\n",
       "      <td>`` oh , shit . ''</td>\n",
       "      <td>250</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4472</th>\n",
       "      <td>shit , shit , shit .</td>\n",
       "      <td>222</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5560</th>\n",
       "      <td>`` son of a bitch . ''</td>\n",
       "      <td>184</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7108</th>\n",
       "      <td>`` oh , shit ! ''</td>\n",
       "      <td>149</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7794</th>\n",
       "      <td>`` you son of a bitch ! ''</td>\n",
       "      <td>137</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10298</th>\n",
       "      <td>`` shit , '' i said .</td>\n",
       "      <td>109</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11304</th>\n",
       "      <td>i suck in a breath .</td>\n",
       "      <td>101</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11649</th>\n",
       "      <td>shit , shit , shit !</td>\n",
       "      <td>98</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         sentence    n  contains_bad_word  space_count\n",
       "2340        `` what the fuck ? ''  383               True            5\n",
       "3809       `` son of a bitch ! ''  256               True            6\n",
       "3908            `` oh , shit . ''  250               True            5\n",
       "4472         shit , shit , shit .  222               True            5\n",
       "5560       `` son of a bitch . ''  184               True            6\n",
       "7108            `` oh , shit ! ''  149               True            5\n",
       "7794   `` you son of a bitch ! ''  137               True            7\n",
       "10298       `` shit , '' i said .  109               True            6\n",
       "11304        i suck in a breath .  101               True            5\n",
       "11649        shit , shit , shit !   98               True            5"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common sentences with \"bad words\"\n",
    "sentences_df[(sentences_df.contains_bad_word)&(sentences_df.space_count>4)].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11944</th>\n",
       "      <td>`` shit , '' he said .</td>\n",
       "      <td>96</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31631</th>\n",
       "      <td>`` fuck yes . ''</td>\n",
       "      <td>42</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36781</th>\n",
       "      <td>very sexy .</td>\n",
       "      <td>37</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47622</th>\n",
       "      <td>`` you 're such an ass . ''</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118326</th>\n",
       "      <td>you 're sexy .</td>\n",
       "      <td>13</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154854</th>\n",
       "      <td>`` sneaky bastard . ''</td>\n",
       "      <td>11</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142548</th>\n",
       "      <td>well , fuck me .</td>\n",
       "      <td>11</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157734</th>\n",
       "      <td>i want to fuck you .</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175546</th>\n",
       "      <td>`` what the fuck 's going on ? ''</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172264</th>\n",
       "      <td>just fuck me .</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 sentence   n  contains_bad_word  space_count\n",
       "11944              `` shit , '' he said .  96               True            6\n",
       "31631                    `` fuck yes . ''  42               True            4\n",
       "36781                         very sexy .  37               True            2\n",
       "47622         `` you 're such an ass . ''  29               True            7\n",
       "118326                     you 're sexy .  13               True            3\n",
       "154854             `` sneaky bastard . ''  11               True            4\n",
       "142548                   well , fuck me .  11               True            4\n",
       "157734               i want to fuck you .  10               True            5\n",
       "175546  `` what the fuck 's going on ? ''  10               True            8\n",
       "172264                     just fuck me .  10               True            3"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# random sentences with \"bad words\"\n",
    "sentences_df[(sentences_df.contains_bad_word)].sample(10).sort_values(by='n',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>n</th>\n",
       "      <th>contains_bad_word</th>\n",
       "      <th>space_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24378</th>\n",
       "      <td>`` what the fuck are you talking about ? ''</td>\n",
       "      <td>52</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25302</th>\n",
       "      <td>`` what the fuck are you doing ? ''</td>\n",
       "      <td>51</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40642</th>\n",
       "      <td>`` you scared the shit out of me . ''</td>\n",
       "      <td>34</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46609</th>\n",
       "      <td>`` you 're so full of shit . ''</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52958</th>\n",
       "      <td>`` you are so full of shit . ''</td>\n",
       "      <td>27</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54045</th>\n",
       "      <td>`` what the fuck do you think you 're doing ? ''</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55438</th>\n",
       "      <td>`` what the fuck are you doing here ? ''</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56448</th>\n",
       "      <td>you scared the shit out of me ! ''</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64022</th>\n",
       "      <td>`` what the fuck is going on ? ''</td>\n",
       "      <td>23</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64052</th>\n",
       "      <td>you scared the shit out of me . ''</td>\n",
       "      <td>23</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence   n  \\\n",
       "24378       `` what the fuck are you talking about ? ''  52   \n",
       "25302               `` what the fuck are you doing ? ''  51   \n",
       "40642             `` you scared the shit out of me . ''  34   \n",
       "46609                   `` you 're so full of shit . ''  30   \n",
       "52958                   `` you are so full of shit . ''  27   \n",
       "54045  `` what the fuck do you think you 're doing ? ''  26   \n",
       "55438          `` what the fuck are you doing here ? ''  26   \n",
       "56448                you scared the shit out of me ! ''  25   \n",
       "64022                 `` what the fuck is going on ? ''  23   \n",
       "64052                you scared the shit out of me . ''  23   \n",
       "\n",
       "       contains_bad_word  space_count  \n",
       "24378               True            9  \n",
       "25302               True            8  \n",
       "40642               True            9  \n",
       "46609               True            8  \n",
       "52958               True            8  \n",
       "54045               True           11  \n",
       "55438               True            9  \n",
       "56448               True            8  \n",
       "64022               True            8  \n",
       "64052               True            8  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# common longer sentences with \"bad words\"\n",
    "sentences_df[(sentences_df.contains_bad_word) & (sentences_df.space_count>7)].head(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}