Natural_Language_Processing_Workshop.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Natural Language Processing - Workshop.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPQ2QpUJ5PfigSGeDM7GILD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/victorescosta/Natural-Language-Processing-Workshop/blob/main/Natural_Language_Processing_Workshop.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qAlQhoxtpasX"
      },
      "source": [
        "Chapter 01 - Introduction to Natural Language Processing (NLP)\n",
        "===\n",
        "follow me on github: [victorescosta](https://github.com/victorescosta)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "czujt9RbK8WW"
      },
      "source": [
        "**Exercise 1.01** - Basic Text Analytics\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "Y1AEVWW8K4Q2",
        "outputId": "d7b143c6-0b63-4ab7-ad4d-c7757181caf3"
      },
      "source": [
        "#assign a sentence variable\n",
        "sentence = 'The quick brown fox jumps over the lazy dog'\n",
        "sentence"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'The quick brown fox jumps over the lazy dog'"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pwrCuDD5LbdM",
        "outputId": "10f5520c-d5fc-4fb5-8b77-8b212839df15"
      },
      "source": [
        "#checks if a word belongs to that text\n",
        "def find_word(word, sentence):\n",
        "    return word in sentence\n",
        "\n",
        "find_word('quick', sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q8BD3l6oLvKH",
        "outputId": "9c836d3b-4218-42e5-9ebc-a3095219b77a"
      },
      "source": [
        "#find out the index of the word\n",
        "def get_index(word, text):\n",
        "    return text.index(word)\n",
        "\n",
        "get_index('fox', sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "16"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DFo8fX8iL-O9",
        "outputId": "649884d8-f7d2-4793-c795-464c2f538149"
      },
      "source": [
        "#find the rank of the word\n",
        "get_index('dog', sentence.split())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "K2Gf0EB5lz0V",
        "outputId": "a4f7cc8c-ce9a-48d1-aa80-0ac50de6e20d"
      },
      "source": [
        "#print the third word of given text\n",
        "def get_word(text, rank):\n",
        "    return text.split()[rank]\n",
        "\n",
        "get_word(sentence, 2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'brown'"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "63_9X2slmc91",
        "outputId": "3272c779-0780-49fa-af2f-39eb68b2de82"
      },
      "source": [
        "#print the third word of given sentence in reverse order\n",
        "get_word(sentence, 2)[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'nworb'"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "zDnb_i4um1wt",
        "outputId": "ac4511c6-6055-4968-de45-6ec3cdeaf1ca"
      },
      "source": [
        "#concatenate the first and last words of the given sentence\n",
        "def concat_words(text):\n",
        "    \"\"\"\n",
        "    This method will concat first and last\n",
        "    words of given text\n",
        "    \"\"\"\n",
        "    words = text.split()\n",
        "    first_word = words[0]\n",
        "    last_word = words[len(words)-1]\n",
        "    return first_word + last_word\n",
        "\n",
        "concat_words(sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Thedog'"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "54-RgAVWqY7M",
        "outputId": "2807fe2a-480d-42d1-8ddc-34136fff54d8"
      },
      "source": [
        "#print words at even positions\n",
        "def get_even_position_words(text):\n",
        "    words = text.split()\n",
        "    return [words[i] for i in range (len(words))\n",
        "                                    if i%2 == 0]\n",
        "get_even_position_words(sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['The', 'brown', 'jumps', 'the', 'dog']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "FQN9E8N_vN_1",
        "outputId": "6af21cb4-5d8e-482c-858f-c77f6f607265"
      },
      "source": [
        "#last three letters of the text\n",
        "def get_last_n_letters(text, n):\n",
        "    return text[-n:]\n",
        "get_last_n_letters(sentence, 3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'dog'"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "erY2yz5Vvm5q",
        "outputId": "51b2609b-676f-4eb1-95b9-45da8c79230d"
      },
      "source": [
        "#print the text in reverse order\n",
        "def get_reverse(text):\n",
        "    return text[::-1]\n",
        "get_reverse(sentence)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'god yzal eht revo spmuj xof nworb kciuq ehT'"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "GQuWaZkSv-uO",
        "outputId": "62eeb6a4-0069-4fad-81aa-8f9a0c62f829"
      },
      "source": [
        "#print each word of the given text in reverse order, maintaining their sequence\n",
        "def get_word_reverse(text):\n",
        "    words = text.split()\n",
        "    return ' '.join([word[::-1] for word in words])\n",
        "get_word_reverse(sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ehT kciuq nworb xof spmuj revo eht yzal god'"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "oPyFsQv6u9En",
        "outputId": "7b3eeded-3324-40da-e090-0f47e0ced114"
      },
      "source": [
        "sentence"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'The quick brown fox jumps over the lazy dog'"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_I2WUVqxQAp"
      },
      "source": [
        "**Exercise 1.02** - Pre processing steps - Tokenization of a simple sentence\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BUjD4OHZEoZY"
      },
      "source": [
        "#new sentence\n",
        "sentence = 'I am reading NLP Fundamentals.'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "An6bHjQfxa3U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "42878087-aa8e-4f28-da5a-a272e7dfc7d5"
      },
      "source": [
        "from nltk import word_tokenize, download\n",
        "download(['punkt', 'averaged_perceptron_tagger', 'stopwords'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9N9j5OGJE8Uy",
        "outputId": "2de72b99-f27e-41dc-95c4-c5d37565808f"
      },
      "source": [
        "#tokenize a sentence\n",
        "def get_tokens(sentence):\n",
        "  words = word_tokenize(sentence)\n",
        "  return words\n",
        "\n",
        "print(get_tokens(\"I am reading NLP Fundamentals.\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I', 'am', 'reading', 'NLP', 'Fundamentals', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PkWPK082G13-"
      },
      "source": [
        "**Exercise 1.03** - Pre processing steps - PoS tagging"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vRBpO4IbHJIs"
      },
      "source": [
        "complete list of PoS tags in NLTK [PoS-List](https://pythonprogramming.net/natural-language-toolkit-nltk-part-speech-tagging/)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54XjmoDeH3VN"
      },
      "source": [
        "from nltk import word_tokenize, pos_tag"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VQKIhsuSIqEr",
        "outputId": "0a757f24-4850-4ed0-f989-6e5604986968"
      },
      "source": [
        "#find the tokens in sentence\n",
        "def get_tokens(sentence):\n",
        "  words = word_tokenize(sentence)\n",
        "  return words\n",
        "\n",
        "#print the tokens\n",
        "\n",
        "words = get_tokens(\"I am reading NLP Fundamentals.\")\n",
        "print (words)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I', 'am', 'reading', 'NLP', 'Fundamentals', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GmKwusmgJMdy",
        "outputId": "b9a43e2b-5ab9-43b5-f4c7-925b328983c2"
      },
      "source": [
        "#pos_tag method\n",
        "def get_pos(words):\n",
        "  return pos_tag(words)\n",
        "get_pos(words)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('I', 'PRP'),\n",
              " ('am', 'VBP'),\n",
              " ('reading', 'VBG'),\n",
              " ('NLP', 'NNP'),\n",
              " ('Fundamentals', 'NNS'),\n",
              " ('.', '.')]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zUVeXAWzJkBm"
      },
      "source": [
        "**Exercise 1.04** - Pre processing steps - Stop Word Removal"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fv3PME2bJrF1",
        "outputId": "3f50de28-1135-4296-a04c-0762b650966e"
      },
      "source": [
        "from nltk import download\n",
        "download('stopwords')\n",
        "from nltk import word_tokenize\n",
        "from nltk.corpus import stopwords"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CcCgGLZlKZQz"
      },
      "source": [
        "#stop words list. you can change to your following language\n",
        "stop_words = stopwords.words('english') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2R60nuSWKlF5",
        "outputId": "09cbd41b-8ba9-4bf1-aa2f-01c0961fbb7c"
      },
      "source": [
        "print(stop_words)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Q14DTa0LWq1"
      },
      "source": [
        "#tokenize this sentence\n",
        "sentence = \"I am learning Python. It is one of the \"\\\n",
        "\"most popular programming languages\"\n",
        "sentence_words = word_tokenize(sentence)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hG7kEs_rLuqa",
        "outputId": "c2105ea6-feb8-48ac-96fe-ec76a92ed79d"
      },
      "source": [
        "#printing the list of tokens\n",
        "print(sentence_words)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I', 'am', 'learning', 'Python', '.', 'It', 'is', 'one', 'of', 'the', 'most', 'popular', 'programming', 'languages']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UhYTIkXlMaW3"
      },
      "source": [
        "#remove stop words. loop for every token and check if it`s a stopword\n",
        "def remove_stop_words(sentence_words, stop_words):\n",
        "  return ' '.join([word for word in sentence_words if \\\n",
        "                   word not in stop_words])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gYKfgCNwNA3t",
        "outputId": "048be841-a5cb-4e0a-ca3a-b7cb5591967d"
      },
      "source": [
        "#check wheter the stop words are filtered out from the sentence\n",
        "print(remove_stop_words(sentence_words, stop_words))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I learning Python . It one popular programming languages\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vy23bbRvNRwV",
        "outputId": "b879ecec-25ab-4430-a568-053d118bd09b"
      },
      "source": [
        "#add your own stop words to the stop word lists\n",
        "stop_words.extend(['I', 'It', 'one'])\n",
        "print(remove_stop_words(sentence_words, stop_words))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning Python . popular programming languages\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7X5P8v5fN7j5"
      },
      "source": [
        "**Exercise 1.05** - Pre processing steps - Text Normalization (*replacing words*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISRpFyIuOEOD"
      },
      "source": [
        "#replace text in a given sentence\n",
        "sentence = \"I visited the US from the UK on 22-10-18\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O2famQCcQTBd"
      },
      "source": [
        "#replace \"US\" with \"United States\", \"UK\" with \"United Kigdom\", \"18\"  with \"2018\" using replace\n",
        "def normalize(text):\n",
        "    return text.replace(\"US\", \"United States\")\\\n",
        "               .replace(\"UK\", \"United Kingdom\")\\\n",
        "               .replace(\"-18\", \"-2018\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xIe3GE55RCDV",
        "outputId": "c7f2b9b8-21bf-4174-d11a-2ebf722cd09d"
      },
      "source": [
        "#see whether the text has been normalized\n",
        "normalized_sentence = normalize(sentence)\n",
        "print(normalized_sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I visited the United States from the United Kingdom on 22-10-2018\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X8dIZynfRarh",
        "outputId": "17c5538e-05bb-495e-adb1-f0eb818d1d2e"
      },
      "source": [
        "#normalized_sentence again\n",
        "normalized_sentence = normalize('US and UK are two superpowers')\n",
        "print(normalized_sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "United States and United Kingdom are two superpowers\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-4Rn7uDxTkin"
      },
      "source": [
        "*autocorrect* - library for misspelled words. spell() function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wfTRjOX3T3Qa"
      },
      "source": [
        "**Exercise 1.06** - Pre processing steps - Text Normalization (*Spelling Correction of a Word and a Sentence*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ueQ7X-VhaiUQ",
        "outputId": "dfd977c1-beae-4c6b-ac85-610f99f6e02d"
      },
      "source": [
        "#autocorrect module\n",
        "!pip install autocorrect"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting autocorrect\n",
            "  Downloading autocorrect-2.5.1.tar.gz (622 kB)\n",
            "\u001b[?25l\r\u001b[K     |▌                               | 10 kB 15.9 MB/s eta 0:00:01\r\u001b[K     |█                               | 20 kB 19.3 MB/s eta 0:00:01\r\u001b[K     |█▋                              | 30 kB 13.9 MB/s eta 0:00:01\r\u001b[K     |██                              | 40 kB 11.2 MB/s eta 0:00:01\r\u001b[K     |██▋                             | 51 kB 5.3 MB/s eta 0:00:01\r\u001b[K     |███▏                            | 61 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |███▊                            | 71 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |████▏                           | 81 kB 6.2 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 92 kB 4.8 MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 102 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 112 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 122 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 133 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 143 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████                        | 153 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 163 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 174 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 184 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████                      | 194 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 204 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 215 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 225 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 235 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 245 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 256 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 266 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 276 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 286 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 296 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 307 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 317 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 327 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 337 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 348 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 358 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 368 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 378 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 389 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 399 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 409 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 419 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 430 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 440 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 450 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 460 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 471 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 481 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 491 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 501 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 512 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 522 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 532 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 542 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 552 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 563 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 573 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 583 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 593 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 604 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 614 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 622 kB 5.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: autocorrect\n",
            "  Building wheel for autocorrect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for autocorrect: filename=autocorrect-2.5.1-py3-none-any.whl size=621799 sha256=d6a33e150036c5658dbbb895c8270f120b044992a880ac05eaf2773b33a50bf4\n",
            "  Stored in directory: /root/.cache/pip/wheels/90/1e/f2/82ace0d24e69c1184e25716fc1aa0fc89313f23c0f45d45051\n",
            "Successfully built autocorrect\n",
            "Installing collected packages: autocorrect\n",
            "Successfully installed autocorrect-2.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_08mwTLwTCF0"
      },
      "source": [
        "#autocorrect libraries\n",
        "from nltk import word_tokenize\n",
        "from autocorrect import Speller"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "PVsJQKPmaqwg",
        "outputId": "c497f365-f3e7-4373-8f76-ea5b981d8c93"
      },
      "source": [
        "#correct a word using spell() function\n",
        "spell = Speller(lang='en')\n",
        "spell('Natureal')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Natural'"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QwUjr16da78K",
        "outputId": "8660ef38-b816-4569-8d97-ddab0c691aab"
      },
      "source": [
        "#to correct a sentence, first tokenize it into tokens, and then loop each token\n",
        "sentence = word_tokenize(\"Ntural Luanguage Processin deals with \"\\\n",
        "                         \"the art of extracting insightes from \"\\\n",
        "                         \"Natual Languaes\")\n",
        "print(sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Ntural', 'Luanguage', 'Processin', 'deals', 'with', 'the', 'art', 'of', 'extracting', 'insightes', 'from', 'Natual', 'Languaes']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZMBZXoicBVH"
      },
      "source": [
        "#loop each token in sentence\n",
        "def correct_spelling(tokens):\n",
        "    sentence_corrected = ' '.join([spell(word) \\\n",
        "                                  for word in tokens])\n",
        "    return sentence_corrected"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxvZqpfWcwKr",
        "outputId": "51d7da85-bac3-410d-a57c-2470d639015a"
      },
      "source": [
        "#print the correct sentence\n",
        "print(correct_spelling(sentence))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Natural Language Processing deals with the art of extracting insights from Natural Languages\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wD4wvHBxdLNv"
      },
      "source": [
        "**Exercise 1.07** - Pre processing steps - Text Normalization (*Stemming*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dIlR2EIQc448"
      },
      "source": [
        "#stemming words into their base forms. import necessary libraries\n",
        "from nltk import stem"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "cmtt-8ldeBjC",
        "outputId": "9e59ad11-5d7a-4fe7-dbdb-e78eefd0cc32"
      },
      "source": [
        "#function to stem words\n",
        "def get_stems(word, stemmer):\n",
        "    return stemmer.stem(word)\n",
        "\n",
        "porterStem = stem.PorterStemmer()\n",
        "get_stems(\"battling\", porterStem)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'battl'"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "S2QmGUL4e1w-",
        "outputId": "1dd73df8-811b-48ba-83ef-950a306804c1"
      },
      "source": [
        "#another method to stem words\n",
        "stemmer = stem.SnowballStemmer(\"english\")\n",
        "get_stems(\"battling\", stemmer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'battl'"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p-2gIYQWf2t1"
      },
      "source": [
        "**Exercise 1.08** - Pre processing steps - Text Normalization (*Lemmatization*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tDEmQKcDf1o7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "11ea6c10-2c71-4c95-91cc-93e337b80724"
      },
      "source": [
        "#installing necessary libraries\n",
        "from nltk import download\n",
        "download('wordnet')\n",
        "from nltk.stem.wordnet import WordNetLemmatizer"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z9FguS1aERjD"
      },
      "source": [
        "#creating an object to WordNetLemmatizer class\n",
        "lemmatizer = WordNetLemmatizer()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "o8rYHMRJEbps",
        "outputId": "68d16c32-fb6d-45bf-fefe-15700b73f5f4"
      },
      "source": [
        "#get lemma function\n",
        "\n",
        "def get_lemma(word):\n",
        "    return lemmatizer.lemmatize(word)\n",
        "get_lemma('evaluations')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'evaluation'"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HYmT1nY3FeDN"
      },
      "source": [
        "**Exercise 1.09** - Pre processing steps - Text Normalization (*NER - Named Entity Recognition - PoS Tag*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zwAYRDxZFdIS",
        "outputId": "9914f0de-8f4d-463a-9794-c406f8ab52e2"
      },
      "source": [
        "#necessary libraries\n",
        "from nltk import download\n",
        "from nltk import pos_tag\n",
        "from nltk import ne_chunk\n",
        "from nltk import word_tokenize\n",
        "download('maxent_ne_chunker')\n",
        "download('words')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package maxent_ne_chunker to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/words.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OtCDfbQZMCf7"
      },
      "source": [
        "#declare a sentence\n",
        "sentence = \"We are reading a book published by Packt \"\\\n",
        "           \"which is based out of Birmingham.\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tCItVQsUMcXj",
        "outputId": "c2d8c042-65c6-48ca-bd0c-90d54546f3f7"
      },
      "source": [
        "#find named entities from the preceding text\n",
        "\n",
        "def get_ner(text):\n",
        "    i = ne_chunk(pos_tag(word_tokenize(text)), binary=True)\n",
        "    return [a for a in i if len(a)==1]\n",
        "get_ner(sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Tree('NE', [('Packt', 'NNP')]), Tree('NE', [('Birmingham', 'NNP')])]"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDm1oVJoRxY3"
      },
      "source": [
        "complete list of PoS tags in NLTK [PoS-List](https://pythonprogramming.net/natural-language-toolkit-nltk-part-speech-tagging/)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O6QOpAJezlu-"
      },
      "source": [
        "**Exercise 1.10** - Pre processing steps - Text Normalization (*Word Sense Disambiguation*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6QgLKB1_0K1u",
        "outputId": "9759fc4c-488a-4908-fdce-589c6ad80423"
      },
      "source": [
        "#import necessary libraries\n",
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "from nltk.wsd import lesk\n",
        "from nltk import word_tokenize"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "20d5FDDw0iAw"
      },
      "source": [
        "#declare variables to analyze ambiguity\n",
        "sentence1 = \"Keep your savings in the bank\"\n",
        "sentence2 = \"It's so risky to drive over the banks of the road\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8NaNEyxV0yvG",
        "outputId": "a2f3dfd7-b40c-4c58-ce3c-19e319d999de"
      },
      "source": [
        "#find the sense of the word 'bank' with Lesk algorithm in nltk.wsd pg 80\n",
        "def get_synset(sentence, word):\n",
        "    return lesk(word_tokenize(sentence), word)\n",
        "get_synset(sentence1, 'bank') "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Synset('savings_bank.n.02')"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Et0jWzzE1h40",
        "outputId": "0c6413ae-9e95-4386-9010-27e11a10b0c9"
      },
      "source": [
        "#check background synsets definitions in Lesk algorithm for more info\n",
        "get_synset(sentence2, 'bank')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Synset('bank.v.07')"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XR2wV37j2Xh1"
      },
      "source": [
        "**Exercise 1.11** - Pre processing steps - Text Normalization (*Sentence Boundary Detection*)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bl1avmcf2Uwh"
      },
      "source": [
        "#import necessary libraries\n",
        "import nltk\n",
        "from nltk.tokenize import sent_tokenize"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YwGgqYHujfNo",
        "outputId": "0ea8a640-e9b4-487a-a314-0f822b674509"
      },
      "source": [
        "#detecting sentence boundary with sent_tokenize()\n",
        "def get_sentences(text):\n",
        "    return sent_tokenize(text)\n",
        "get_sentences(\"We are reading a book. Do you know who is \"\\\n",
        "              \"the publisher? It is Packt. Packt is based \"\\\n",
        "              \"out of Birmingham\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['We are reading a book.',\n",
              " 'Do you know who is the publisher?',\n",
              " 'It is Packt.',\n",
              " 'Packt is based out of Birmingham']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cafnba3JkMV4",
        "outputId": "4664e15d-5da1-4a03-f331-c3e15ae63954"
      },
      "source": [
        "#applying sent_tokenize method\n",
        "get_sentences(\"Mr. Donald John Trump is the current \"\\\n",
        "              \"president of the USA. Before joining \"\\\n",
        "              \"politics, he was a businessman.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Mr. Donald John Trump is the current president of the USA.',\n",
              " 'Before joining politics, he was a businessman.']"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tQc9kTU3vjJp"
      },
      "source": [
        "**Activity 1.01** - Preprocessing of Raw Text  \n",
        "---\n",
        "[file.txt](https://packt.live/30cu54z) - you can paste it in this colab notebook"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2jlBIXl-_zmA",
        "outputId": "614c2e4b-123b-406a-9be7-baa5b95d3feb"
      },
      "source": [
        "#module to install\n",
        "!pip install autocorrect"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting autocorrect\n",
            "  Downloading autocorrect-2.6.0.tar.gz (622 kB)\n",
            "\u001b[?25l\r\u001b[K     |▌                               | 10 kB 20.5 MB/s eta 0:00:01\r\u001b[K     |█                               | 20 kB 25.4 MB/s eta 0:00:01\r\u001b[K     |█▋                              | 30 kB 14.6 MB/s eta 0:00:01\r\u001b[K     |██                              | 40 kB 10.9 MB/s eta 0:00:01\r\u001b[K     |██▋                             | 51 kB 7.2 MB/s eta 0:00:01\r\u001b[K     |███▏                            | 61 kB 7.6 MB/s eta 0:00:01\r\u001b[K     |███▊                            | 71 kB 7.3 MB/s eta 0:00:01\r\u001b[K     |████▏                           | 81 kB 8.1 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 92 kB 7.8 MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 102 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 112 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 122 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 133 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 143 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████                        | 153 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 163 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 174 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 184 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████                      | 194 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 204 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 215 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 225 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 235 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 245 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 256 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 266 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 276 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 286 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 296 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 307 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 317 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 327 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 337 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 348 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 358 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 368 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 378 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 389 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 399 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 409 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 419 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 430 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 440 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 450 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 460 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 471 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 481 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 491 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 501 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 512 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 522 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 532 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 542 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 552 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 563 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 573 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 583 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 593 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 604 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 614 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 622 kB 6.8 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: autocorrect\n",
            "  Building wheel for autocorrect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for autocorrect: filename=autocorrect-2.6.0-py3-none-any.whl size=622249 sha256=53c20709232499efb0c2148f1fefc5a1c6ae8a81c9140c3185870159e933eeb8\n",
            "  Stored in directory: /root/.cache/pip/wheels/fa/ce/aa/bc894efbe0541ce91dea21561d01d319783986d9787a8e9f58\n",
            "Successfully built autocorrect\n",
            "Installing collected packages: autocorrect\n",
            "Successfully installed autocorrect-2.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKp9pS46vxe7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6f206d5-f065-4f32-e444-20def0ef79c3"
      },
      "source": [
        "#import the necessary libraries\n",
        "from nltk import word_tokenize, download\n",
        "download(['punkt', 'averaged_perceptron_tagger', 'stopwords', 'wordnet',\n",
        "          'maxent_ne_chunker', 'words'])\n",
        "from nltk import word_tokenize, pos_tag\n",
        "from nltk.corpus import stopwords\n",
        "from autocorrect import Speller\n",
        "from nltk import ne_chunk\n",
        "from nltk import stem\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "from nltk.wsd import lesk\n",
        "from nltk.tokenize import sent_tokenize\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package maxent_ne_chunker to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package maxent_ne_chunker is already up-to-date!\n",
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/words.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eHFbjazGAGj5",
        "outputId": "d9acb214-2c5f-48e6-b5ef-ccec19593ef7"
      },
      "source": [
        "#load the text corpus to a variable\n",
        "txt = open(\"file.txt\", \"r\").read()\n",
        "print(txt)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The reader of this course should have a basic knowledge of the Python programming lenguage.\n",
            "He/she must have knowldge of data types in Python.He should be able to write functions,\n",
            " and also have the ability to import and use libraries and packages in Python. Familiarity\n",
            " with basic linguistics and probability is assumed although not required to fully\n",
            " complete this course.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zpx9bvLRCe-1",
        "outputId": "05cc762a-7e74-4077-926e-a2345846655e"
      },
      "source": [
        "#tokenize the text\n",
        "def get_tokens(sentence):\n",
        "  words = word_tokenize(sentence)\n",
        "  return words\n",
        "\n",
        "txtTokens = get_tokens(txt)\n",
        "\n",
        "#and print the first 20 tokens. slice notation used\n",
        "print(txtTokens[:20])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['The', 'reader', 'of', 'this', 'course', 'should', 'have', 'a', 'basic', 'knowledge', 'of', 'the', 'Python', 'programming', 'lenguage', '.', 'He/she', 'must', 'have', 'knowldge']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KIGgvwh1IkAD",
        "outputId": "a5063777-99f6-4e4d-da94-b66aa1f1918f"
      },
      "source": [
        "#apply spelling correction on each token \n",
        "#loop each token in sentence\n",
        "spell = Speller(lang='en')\n",
        "def correct_spelling(tokens):\n",
        "    sentence_corrected = ' '.join([spell(word) \\\n",
        "                                  for word in tokens])\n",
        "    return sentence_corrected\n",
        "\n",
        "correctTxt = correct_spelling(txtTokens) #applying correct_spelling function in the txtTokens\n",
        "correctTxt = get_tokens(correctTxt) #tokenize correct txt\n",
        "\n",
        "#and print the initial 20 tokens corrected\n",
        "print(correctTxt[:20])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['The', 'reader', 'of', 'this', 'course', 'should', 'have', 'a', 'basic', 'knowledge', 'of', 'the', 'Python', 'programming', 'language', '.', 'He/she', 'must', 'have', 'knowledge']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TXgCDt9oU8wB",
        "outputId": "3ea752fd-2758-4ec6-fe19-539d2ede4aef"
      },
      "source": [
        "#applying PoS tags to corrected tokens\n",
        "print(pos_tag(correctTxt))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('The', 'DT'), ('reader', 'NN'), ('of', 'IN'), ('this', 'DT'), ('course', 'NN'), ('should', 'MD'), ('have', 'VB'), ('a', 'DT'), ('basic', 'JJ'), ('knowledge', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Python', 'NNP'), ('programming', 'NN'), ('language', 'NN'), ('.', '.'), ('He/she', 'NNP'), ('must', 'MD'), ('have', 'VB'), ('knowledge', 'NN'), ('of', 'IN'), ('data', 'NNS'), ('types', 'NNS'), ('in', 'IN'), ('Python.He', 'NNP'), ('should', 'MD'), ('be', 'VB'), ('able', 'JJ'), ('to', 'TO'), ('write', 'VB'), ('functions', 'NNS'), (',', ','), ('and', 'CC'), ('also', 'RB'), ('have', 'VBP'), ('the', 'DT'), ('ability', 'NN'), ('to', 'TO'), ('import', 'NN'), ('and', 'CC'), ('use', 'NN'), ('libraries', 'NNS'), ('and', 'CC'), ('packages', 'NNS'), ('in', 'IN'), ('Python', 'NNP'), ('.', '.'), ('Familiarity', 'NN'), ('with', 'IN'), ('basic', 'JJ'), ('linguistics', 'NNS'), ('and', 'CC'), ('probability', 'NN'), ('is', 'VBZ'), ('assumed', 'VBN'), ('although', 'IN'), ('not', 'RB'), ('required', 'VBN'), ('to', 'TO'), ('fully', 'RB'), ('complete', 'VB'), ('this', 'DT'), ('course', 'NN'), ('.', '.')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "POeKXDIta9Hk",
        "outputId": "8a65359f-9be3-4835-bc1e-ad8c7b17af72"
      },
      "source": [
        "#remove stop words from corrected token list\n",
        "#stop words list. you can change to your following language\n",
        "stop_words = stopwords.words('english') \n",
        "\n",
        "#remove stop words. loop for every token and check if it`s a stopword\n",
        "def remove_stop_words(sentence_words, stop_words):\n",
        "  return ' '.join([word for word in sentence_words if \\\n",
        "                   word not in stop_words])\n",
        "\n",
        "#and print the initial 20 tokens\n",
        "txT = remove_stop_words(correctTxt, stop_words)\n",
        "txT = word_tokenize(txT)\n",
        "print(txT[:20])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['The', 'reader', 'course', 'basic', 'knowledge', 'Python', 'programming', 'language', '.', 'He/she', 'must', 'knowledge', 'data', 'types', 'Python.He', 'able', 'write', 'functions', ',', 'also']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rB6-V9iZiLp-",
        "outputId": "c933c72f-c897-41d3-dccd-69db9a11a5c2"
      },
      "source": [
        "#apply stemming and lemmantization to the corrected token list\n",
        "#function to stem words using snowball stemmer\n",
        "stemmer = stem.SnowballStemmer(\"english\")\n",
        "def get_stems(tokens):\n",
        "    sentence_stemmed = ' '.join([stemmer.stem(word) \\\n",
        "                                  for word in tokens])\n",
        "    return sentence_stemmed\n",
        "\n",
        "#function to lemmantize words\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "def get_lemma(tokens):\n",
        "    sentence_lemma = ' '.join([lemmatizer.lemmatize(word) \\\n",
        "                              for word in tokens])\n",
        "    return sentence_lemma\n",
        "#print the initial 20 tokens\n",
        "word_tokenize(get_stems(correctTxt[:20]))\n",
        "word_tokenize(get_lemma(correctTxt[:20]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['The',\n",
              " 'reader',\n",
              " 'of',\n",
              " 'this',\n",
              " 'course',\n",
              " 'should',\n",
              " 'have',\n",
              " 'a',\n",
              " 'basic',\n",
              " 'knowledge',\n",
              " 'of',\n",
              " 'the',\n",
              " 'Python',\n",
              " 'programming',\n",
              " 'language',\n",
              " '.',\n",
              " 'He/she',\n",
              " 'must',\n",
              " 'have',\n",
              " 'knowledge']"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n-Rd0CRRkCr8",
        "outputId": "d14e3d4b-6bfc-4864-a155-496269139a69"
      },
      "source": [
        "#detect sentence boundaries in the given text\n",
        "print(sent_tokenize(txt))\n",
        "#and print the total number of sentences\n",
        "len(sent_tokenize(txt))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['The reader of this course should have a basic knowledge of the Python programming lenguage.', 'He/she must have knowldge of data types in Python.He should be able to write functions,\\n and also have the ability to import and use libraries and packages in Python.', 'Familiarity\\n with basic linguistics and probability is assumed although not required to fully\\n complete this course.']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oCwlwx6Uqfln"
      },
      "source": [
        "**FINISHED THIS CHAPTER. GLÓRIA A DEUS. 六六六**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rkWSO_da95fQ"
      },
      "source": [
        "Chapter 02 - Feature Extraction Methods\n",
        "===\n",
        "follow me on github: [victorescosta](https://github.com/victorescosta)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qjmGaE-OMKYx"
      },
      "source": [
        "**Exercise 2.01** - Text Cleaning and Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eil6xCOv9gJH"
      },
      "source": [
        "#necessary libraries\n",
        "import re "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UcxW0-jPMfp_"
      },
      "source": [
        "#clean_text() method to delete (and replace to ' ') all chars other than alphanum,\n",
        "#whitespaces and split then text into tokens\n",
        "def clean_text(sentence):\n",
        "    return re.sub(r'([^\\s\\w]|_)+', ' ',\n",
        "                  sentence).split()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8z8hfmPYMi48",
        "outputId": "91a143f2-f279-4f8f-cc58-f73be0b916f1"
      },
      "source": [
        "#store the sentence variable and pass to the function\n",
        "sentence = 'Sunil tweeted, \"Witnessing 70th Republic Day \"\\\n",
        "            \"of India from Rajpath, New Delhi. \"\\\n",
        "            \"Mesmerizing performance by Indian Army! \"\\\n",
        "            \"@indian_army #India #70thRepublic_Day. \"\\\n",
        "            \"For more photos ping me sunil@photoking.com :)\"'\n",
        "clean_text(sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Sunil',\n",
              " 'tweeted',\n",
              " 'Witnessing',\n",
              " '70th',\n",
              " 'Republic',\n",
              " 'Day',\n",
              " 'of',\n",
              " 'India',\n",
              " 'from',\n",
              " 'Rajpath',\n",
              " 'New',\n",
              " 'Delhi',\n",
              " 'Mesmerizing',\n",
              " 'performance',\n",
              " 'by',\n",
              " 'Indian',\n",
              " 'Army',\n",
              " 'indian',\n",
              " 'army',\n",
              " 'India',\n",
              " '70thRepublic',\n",
              " 'Day',\n",
              " 'For',\n",
              " 'more',\n",
              " 'photos',\n",
              " 'ping',\n",
              " 'me',\n",
              " 'sunil',\n",
              " 'photoking',\n",
              " 'com']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yZwzYUX_O4r7"
      },
      "source": [
        "**Exercise 2.02** - Extracting n-grams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6NBKCinSPDXD"
      },
      "source": [
        "#using custom-defined functions and with libs (nltk and textblob)\n",
        "#import necessary libraries\n",
        "import re"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tDuUiPq4Ro2m"
      },
      "source": [
        "#function to extract n-grams\n",
        "def n_gram_extractor(sentence, n):\n",
        "    tokens = re.sub(r'([^\\s\\w]|_)+', ' ',\n",
        "                    sentence).split()\n",
        "    for i in range(len(tokens)-n+1):\n",
        "        print(tokens[i:i+n])"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "alUjxj4HTHc4",
        "outputId": "f68c02cd-306f-496c-f797-44b3141dae0e"
      },
      "source": [
        "#testing it\n",
        "n_gram_extractor('The cute little boy is playing with the kitten.', \\\n",
        "                 3)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['The', 'cute', 'little']\n",
            "['cute', 'little', 'boy']\n",
            "['little', 'boy', 'is']\n",
            "['boy', 'is', 'playing']\n",
            "['is', 'playing', 'with']\n",
            "['playing', 'with', 'the']\n",
            "['with', 'the', 'kitten']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1GxGukggThFX",
        "outputId": "afc71bc6-bfa6-4ac1-cb97-ea082a98352c"
      },
      "source": [
        "#check ngrams using nltk\n",
        "from nltk import ngrams\n",
        "\n",
        "list(ngrams('The cute little boy is playing with the kitten.'\\\n",
        "            .split(), 3))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('The', 'cute', 'little'),\n",
              " ('cute', 'little', 'boy'),\n",
              " ('little', 'boy', 'is'),\n",
              " ('boy', 'is', 'playing'),\n",
              " ('is', 'playing', 'with'),\n",
              " ('playing', 'with', 'the'),\n",
              " ('with', 'the', 'kitten.')]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CGiknMt9VSsB",
        "outputId": "34a7cc04-4b95-4054-da41-97ae867a412c"
      },
      "source": [
        "from nltk import download\n",
        "download('punkt')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D0-ULOafUYzf",
        "outputId": "fb8d4d84-901a-4687-c1bf-f1400aca7c42"
      },
      "source": [
        "#check ngrams using textblob\n",
        "!pip install -U textblob\n",
        "from textblob import TextBlob\n",
        "\n",
        "blob = TextBlob(\"The cute little boy is playing with the kitten.\")\n",
        "blob.ngrams(n=3)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: textblob in /usr/local/lib/python3.7/dist-packages (0.17.1)\n",
            "Requirement already satisfied: nltk>=3.1 in /usr/local/lib/python3.7/dist-packages (from textblob) (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk>=3.1->textblob) (1.15.0)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[WordList(['The', 'cute', 'little']),\n",
              " WordList(['cute', 'little', 'boy']),\n",
              " WordList(['little', 'boy', 'is']),\n",
              " WordList(['boy', 'is', 'playing']),\n",
              " WordList(['is', 'playing', 'with']),\n",
              " WordList(['playing', 'with', 'the']),\n",
              " WordList(['with', 'the', 'kitten'])]"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zeuT_m9iXGB_"
      },
      "source": [
        "**Exercise 2.03** - Tokenizing Text with Keras and TextBlob"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4a8qb8SXdnZ"
      },
      "source": [
        "#import necessary libraries and declaring sentence var\n",
        "from keras.preprocessing.text import text_to_word_sequence\n",
        "from textblob import TextBlob\n",
        "\n",
        "sentence = 'Sunil tweeted, \"Witnessing 70th Republic Day \"\\\n",
        "            \"of India from Rajpath, New Delhi. \"\\\n",
        "            \"Mesmerizing performance by Indian Army !\"\\\n",
        "            \"Awesome airshow! @india_official \"\\\n",
        "            \"@indian_army #India #70thRepublic_Day. \"\\\n",
        "            \"For more photos ping me sunil@photoking.com :)\"'"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M4lB3cZuYah5",
        "outputId": "e1136af1-57cc-493e-bdf7-f409b2b75029"
      },
      "source": [
        "#tokenize using keras\n",
        "def get_keras_tokens(text):\n",
        "    return text_to_word_sequence(text)\n",
        "get_keras_tokens(sentence)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sunil',\n",
              " 'tweeted',\n",
              " 'witnessing',\n",
              " '70th',\n",
              " 'republic',\n",
              " 'day',\n",
              " 'of',\n",
              " 'india',\n",
              " 'from',\n",
              " 'rajpath',\n",
              " 'new',\n",
              " 'delhi',\n",
              " 'mesmerizing',\n",
              " 'performance',\n",
              " 'by',\n",
              " 'indian',\n",
              " 'army',\n",
              " 'awesome',\n",
              " 'airshow',\n",
              " 'india',\n",
              " 'official',\n",
              " 'indian',\n",
              " 'army',\n",
              " 'india',\n",
              " '70threpublic',\n",
              " 'day',\n",
              " 'for',\n",
              " 'more',\n",
              " 'photos',\n",
              " 'ping',\n",
              " 'me',\n",
              " 'sunil',\n",
              " 'photoking',\n",
              " 'com']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NmyybjMvYwW3",
        "outputId": "a72db616-93f2-49b1-fc3f-0c273e819ea3"
      },
      "source": [
        "#tokenizing using textblob lib\n",
        "def get_textblob_tokens(text):\n",
        "    blob = TextBlob(text)\n",
        "    return blob.words\n",
        "get_textblob_tokens(sentence)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "WordList(['Sunil', 'tweeted', 'Witnessing', '70th', 'Republic', 'Day', 'of', 'India', 'from', 'Rajpath', 'New', 'Delhi', 'Mesmerizing', 'performance', 'by', 'Indian', 'Army', 'Awesome', 'airshow', 'india_official', 'indian_army', 'India', '70thRepublic_Day', 'For', 'more', 'photos', 'ping', 'me', 'sunil', 'photoking.com'])"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wEQhiEmIabph"
      },
      "source": [
        "**Exercise 2.04** - Tokenizing Text Using Various Tokenizers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gYpry_XMalkC"
      },
      "source": [
        "#"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}