indicTrans_python_interface.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "indicTrans_python_interface.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cmrajan/Go-figure/blob/master/indicTrans_python_interface.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjfzxXZLHed_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cf6982bd-ba8d-46aa-bdf5-84318e61149a"
      },
      "source": [
        "# clone the repo for running evaluation\n",
        "!git clone https://github.com/AI4Bharat/indicTrans.git\n",
        "%cd indicTrans\n",
        "# clone requirements repositories\n",
        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_library.git\n",
        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_resources.git\n",
        "!git clone https://github.com/rsennrich/subword-nmt.git\n",
        "%cd .."
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'indicTrans'...\n",
            "remote: Enumerating objects: 498, done.\u001b[K\n",
            "remote: Counting objects: 100% (201/201), done.\u001b[K\n",
            "remote: Compressing objects: 100% (80/80), done.\u001b[K\n",
            "remote: Total 498 (delta 160), reused 135 (delta 120), pack-reused 297\u001b[K\n",
            "Receiving objects: 100% (498/498), 1.49 MiB | 6.17 MiB/s, done.\n",
            "Resolving deltas: 100% (287/287), done.\n",
            "/content/indicTrans\n",
            "Cloning into 'indic_nlp_library'...\n",
            "remote: Enumerating objects: 1325, done.\u001b[K\n",
            "remote: Counting objects: 100% (147/147), done.\u001b[K\n",
            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
            "remote: Total 1325 (delta 84), reused 89 (delta 41), pack-reused 1178\u001b[K\n",
            "Receiving objects: 100% (1325/1325), 9.57 MiB | 7.39 MiB/s, done.\n",
            "Resolving deltas: 100% (688/688), done.\n",
            "Cloning into 'indic_nlp_resources'...\n",
            "remote: Enumerating objects: 133, done.\u001b[K\n",
            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
            "remote: Total 133 (delta 0), reused 2 (delta 0), pack-reused 126\u001b[K\n",
            "Receiving objects: 100% (133/133), 149.77 MiB | 13.71 MiB/s, done.\n",
            "Resolving deltas: 100% (51/51), done.\n",
            "Cloning into 'subword-nmt'...\n",
            "remote: Enumerating objects: 580, done.\u001b[K\n",
            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
            "remote: Total 580 (delta 0), reused 1 (delta 0), pack-reused 576\u001b[K\n",
            "Receiving objects: 100% (580/580), 237.41 KiB | 1.60 MiB/s, done.\n",
            "Resolving deltas: 100% (349/349), done.\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IeYW2BJhlJvx",
        "outputId": "865018e1-34fc-46dc-fe23-caaab4d52f8a"
      },
      "source": [
        "# Install the necessary libraries\n",
        "!pip install sacremoses pandas mock sacrebleu tensorboardX pyarrow indic-nlp-library\n",
        "! pip install mosestokenizer subword-nmt\n",
        "# Install fairseq from source\n",
        "!git clone https://github.com/pytorch/fairseq.git\n",
        "%cd fairseq\n",
        "# !git checkout da9eaba12d82b9bfc1442f0e2c6fc1b895f4d35d\n",
        "!pip install --editable ./\n",
        "\n",
        "%cd .."
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[?25l\r\u001b[K     |▍                               | 10 kB 31.3 MB/s eta 0:00:01\r\u001b[K     |▊                               | 20 kB 9.6 MB/s eta 0:00:01\r\u001b[K     |█                               | 30 kB 8.5 MB/s eta 0:00:01\r\u001b[K     |█▌                              | 40 kB 7.9 MB/s eta 0:00:01\r\u001b[K     |█▉                              | 51 kB 4.4 MB/s eta 0:00:01\r\u001b[K     |██▏                             | 61 kB 4.7 MB/s eta 0:00:01\r\u001b[K     |██▋                             | 71 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███                             | 81 kB 4.8 MB/s eta 0:00:01\r\u001b[K     |███▎                            | 92 kB 4.6 MB/s eta 0:00:01\r\u001b[K     |███▋                            | 102 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████                            | 112 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████▍                           | 122 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 133 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 143 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 153 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 163 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 174 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 184 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████                         | 194 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 204 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 215 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████                        | 225 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 235 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 245 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 256 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 266 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 276 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 286 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 296 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 307 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 317 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 327 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 337 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 348 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 358 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 368 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 378 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 389 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 399 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 409 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 419 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 430 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 440 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 450 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 460 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 471 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 481 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 491 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 501 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 512 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 522 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 532 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 542 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 552 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 563 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 573 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 583 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 593 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 604 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 614 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 624 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 634 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 645 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 655 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 665 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 675 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 686 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 696 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 706 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 716 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 727 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 737 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 747 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 757 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 768 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 778 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 788 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 798 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 808 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 819 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 829 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 839 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 849 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 860 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 870 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 880 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 890 kB 4.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 895 kB 4.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
            "Collecting mock\n",
            "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.0.0-py3-none-any.whl (90 kB)\n",
            "\u001b[K     |████████████████████████████████| 90 kB 9.1 MB/s \n",
            "\u001b[?25hCollecting tensorboardX\n",
            "  Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n",
            "\u001b[K     |████████████████████████████████| 124 kB 53.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (3.0.0)\n",
            "Collecting indic-nlp-library\n",
            "  Downloading indic_nlp_library-0.81-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 5.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.41.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.1)\n",
            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.19.5)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (0.8.9)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.3.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.17.3)\n",
            "Collecting sphinx-argparse\n",
            "  Downloading sphinx-argparse-0.2.5.tar.gz (12 kB)\n",
            "Collecting sphinx-rtd-theme\n",
            "  Downloading sphinx_rtd_theme-0.5.2-py2.py3-none-any.whl (9.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.1 MB 40.0 MB/s \n",
            "\u001b[?25hCollecting morfessor\n",
            "  Downloading Morfessor-2.0.6-py3-none-any.whl (35 kB)\n",
            "Requirement already satisfied: sphinx>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx-argparse->indic-nlp-library) (1.8.5)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.0)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.1.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.7.12)\n",
            "Requirement already satisfied: docutils>=0.11 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.17.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (21.0)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (57.2.0)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.9.1)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.6.1)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.11.3)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.23.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.0.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.10)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.4.7)\n",
            "Collecting docutils>=0.11\n",
            "  Downloading docutils-0.16-py2.py3-none-any.whl (548 kB)\n",
            "\u001b[K     |████████████████████████████████| 548 kB 39.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.1.5)\n",
            "Building wheels for collected packages: sphinx-argparse\n",
            "  Building wheel for sphinx-argparse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sphinx-argparse: filename=sphinx_argparse-0.2.5-py3-none-any.whl size=11549 sha256=a647f161147f5ce8fef36bccf61c2a96c21f313d8d0f0908f7762581f97442eb\n",
            "  Stored in directory: /root/.cache/pip/wheels/4a/4c/12/83c88bdc1bc352d7036de2891b5d6729adcf1bda3bab015560\n",
            "Successfully built sphinx-argparse\n",
            "Installing collected packages: docutils, sphinx-rtd-theme, sphinx-argparse, portalocker, morfessor, colorama, tensorboardX, sacremoses, sacrebleu, mock, indic-nlp-library\n",
            "  Attempting uninstall: docutils\n",
            "    Found existing installation: docutils 0.17.1\n",
            "    Uninstalling docutils-0.17.1:\n",
            "      Successfully uninstalled docutils-0.17.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed colorama-0.4.4 docutils-0.16 indic-nlp-library-0.81 mock-4.0.3 morfessor-2.0.6 portalocker-2.3.0 sacrebleu-2.0.0 sacremoses-0.0.45 sphinx-argparse-0.2.5 sphinx-rtd-theme-0.5.2 tensorboardX-2.4\n",
            "Collecting mosestokenizer\n",
            "  Downloading mosestokenizer-1.1.0.tar.gz (37 kB)\n",
            "Collecting subword-nmt\n",
            "  Downloading subword_nmt-0.3.7-py2.py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from mosestokenizer) (0.6.2)\n",
            "Collecting openfile\n",
            "  Downloading openfile-0.0.7-py3-none-any.whl (2.4 kB)\n",
            "Collecting uctools\n",
            "  Downloading uctools-1.3.0.tar.gz (4.6 kB)\n",
            "Collecting toolwrapper\n",
            "  Downloading toolwrapper-2.1.0.tar.gz (3.2 kB)\n",
            "Building wheels for collected packages: mosestokenizer, toolwrapper, uctools\n",
            "  Building wheel for mosestokenizer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mosestokenizer: filename=mosestokenizer-1.1.0-py3-none-any.whl size=49119 sha256=77fc44a4f8d3133812b27e17c2b5181f2bd8032fee3cadadad8548f048c3ed67\n",
            "  Stored in directory: /root/.cache/pip/wheels/a7/31/94/fef279382208e85a65c1a7f5c4d0020115477b0af74f296b57\n",
            "  Building wheel for toolwrapper (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for toolwrapper: filename=toolwrapper-2.1.0-py3-none-any.whl size=3353 sha256=cfe530800c0ba0d7c14accd928c3b89a25374ceb23d40f376d134359d373552b\n",
            "  Stored in directory: /root/.cache/pip/wheels/c5/4f/33/54741ffe08e38ececb1d28068a153729b4fe820bafa0a0691f\n",
            "  Building wheel for uctools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for uctools: filename=uctools-1.3.0-py3-none-any.whl size=6161 sha256=2bbf6aa1eb10b973c8dc99f0196081393ebfb9c0fdf57197ef90da28c718f336\n",
            "  Stored in directory: /root/.cache/pip/wheels/fb/44/e9/914cf8fa71f0141f9314f862538d1218fcf2b94542a0fb7d35\n",
            "Successfully built mosestokenizer toolwrapper uctools\n",
            "Installing collected packages: uctools, toolwrapper, openfile, subword-nmt, mosestokenizer\n",
            "Successfully installed mosestokenizer-1.1.0 openfile-0.0.7 subword-nmt-0.3.7 toolwrapper-2.1.0 uctools-1.3.0\n",
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 28932, done.\u001b[K\n",
            "remote: Counting objects: 100% (8/8), done.\u001b[K\n",
            "remote: Compressing objects: 100% (8/8), done.\u001b[K\n",
            "remote: Total 28932 (delta 0), reused 6 (delta 0), pack-reused 28924\u001b[K\n",
            "Receiving objects: 100% (28932/28932), 12.16 MiB | 14.31 MiB/s, done.\n",
            "Resolving deltas: 100% (21733/21733), done.\n",
            "/content/fairseq\n",
            "Obtaining file:///content/fairseq\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting omegaconf<2.1\n",
            "  Downloading omegaconf-2.0.6-py3-none-any.whl (36 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (2019.12.20)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (4.41.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (1.9.0+cu102)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (0.29.23)\n",
            "Requirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (2.0.0)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (1.14.6)\n",
            "Collecting hydra-core<1.1,>=1.0.7\n",
            "  Downloading hydra_core-1.0.7-py3-none-any.whl (123 kB)\n",
            "\u001b[K     |████████████████████████████████| 123 kB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9825786) (1.19.5)\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "  Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 17.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1,>=1.0.7->fairseq==1.0.0a0+9825786) (5.2.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+9825786) (3.7.4.3)\n",
            "Collecting PyYAML>=5.1.*\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 18.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+9825786) (0.8.9)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+9825786) (2.3.0)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+9825786) (0.4.4)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+9825786) (2.20)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources->hydra-core<1.1,>=1.0.7->fairseq==1.0.0a0+9825786) (3.5.0)\n",
            "Building wheels for collected packages: antlr4-python3-runtime\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141229 sha256=ee24bd95238827760a62c12d5a906b8373fa1c9d646d96c4ce70647edddf3237\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c\n",
            "Successfully built antlr4-python3-runtime\n",
            "Installing collected packages: PyYAML, omegaconf, antlr4-python3-runtime, hydra-core, fairseq\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Running setup.py develop for fairseq\n",
            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 fairseq-1.0.0a0+9825786 hydra-core-1.0.7 omegaconf-2.0.6\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TktUu9NW_PLq"
      },
      "source": [
        "# this step is only required if you are running the code on colab\n",
        "# restart the runtime after running prev cell (to update). See this -> https://stackoverflow.com/questions/57838013/modulenotfounderror-after-successful-pip-install-in-google-colaboratory\n",
        "\n",
        "# this import will not work without restarting runtime\n",
        "from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E_4JxNdRlPQB",
        "outputId": "c2a4a35b-2550-41f3-93bb-de3037bd4f0a"
      },
      "source": [
        "# download the indictrans model\n",
        "\n",
        "\n",
        "# downloading the indic-en model\n",
        "!wget https://storage.googleapis.com/samanantar-public/V0.2/models/indic-en.zip\n",
        "!unzip indic-en.zip\n",
        "\n",
        "# downloading the en-indic model\n",
        "# !wget https://storage.googleapis.com/samanantar-public/V0.2/models/en-indic.zip\n",
        "# !unzip en-indic.zip\n",
        "\n",
        "# # downloading the indic-indic model\n",
        "# !wget https://storage.googleapis.com/samanantar-public/V0.3/models/m2m.zip\n",
        "# !unzip m2m.zip\n",
        "\n",
        "%cd indicTrans"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-08-10 04:46:40--  https://storage.googleapis.com/samanantar-public/V0.2/models/indic-en.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 142.250.157.128, 74.125.23.128, 74.125.203.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|142.250.157.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4551079075 (4.2G) [application/zip]\n",
            "Saving to: ‘indic-en.zip’\n",
            "\n",
            "indic-en.zip         82%[===============>    ]   3.51G  48.2MB/s    eta 14s    "
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yTnWbHqY01-B",
        "outputId": "0d075f51-097b-46ad-aade-407a4437aa62"
      },
      "source": [
        "from indicTrans.inference.engine import Model\n",
        "\n",
        "indic2en_model = Model(expdir='../indic-en')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Initializing vocab and bpe\n",
            "Initializing model for translation\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QTp2NOgQ__sB",
        "outputId": "e015a71e-8206-4e1d-cb3e-11ecb4d44f76"
      },
      "source": [
        "ta_sents = ['அவனுக்கு நம்மைப் தெரியும் என்று தோன்றுகிறது',\n",
        "            \"இது எங்கே இருக்கு என்று என்னால் கண்டுபிடிக்க முடியவில்லை.\",\n",
        "            'உங்களுக்கு உங்கள் அருகில் இருக்கும் ஒருவருக்கோ இத்தகைய அறிகுறிகள் தென்பட்டால், வீட்டிலேயே இருப்பது, கொரோனா வைரஸ் தொற்று பிறருக்கு வராமல் தடுக்க உதவும்.']\n",
        "\n",
        "\n",
        "indic2en_model.batch_translate(ta_sents, 'ta', 'en')\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 3/3 [00:00<00:00, 1225.21it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /pytorch/aten/src/ATen/native/BinaryOps.cpp:467.)\n",
            "  return torch.floor_divide(self, other)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['He seems to know us.',\n",
              " 'I couldnt find it anywhere.',\n",
              " 'If someone in your neighbourhood develops these symptoms, staying at home can help prevent the spread of the coronavirus infection.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "id": "VFXrCNZGEN7Z",
        "outputId": "f72aad17-1cc0-4774-a7ee-5b3a5d954de3"
      },
      "source": [
        "\n",
        "ta_paragraph = \"\"\"இத்தொற்றுநோய் உலகளாவிய சமூக மற்றும் பொருளாதார சீர்குலைவை ஏற்படுத்தியுள்ளது.இதனால் பெரும் பொருளாதார மந்தநிலைக்குப் பின்னர் உலகளவில் மிகப்பெரிய மந்தநிலை ஏற்பட்டுள்ளது. இது விளையாட்டு,மத, அரசியல் மற்றும் கலாச்சார நிகழ்வுகளை ஒத்திவைக்க அல்லது ரத்து செய்ய வழிவகுத்தது.\n",
        "அச்சம் காரணமாக முகக்கவசம், கிருமிநாசினி உள்ளிட்ட பொருட்களை அதிக நபர்கள் வாங்கியதால் விநியோகப் பற்றாக்குறை ஏற்பட்டது.\"\"\"\n",
        "\n",
        "indic2en_model.translate_paragraph(ta_paragraph, 'ta', 'en')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4/4 [00:00<00:00, 1496.76it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'The pandemic has resulted in worldwide social and economic disruption. The world is facing the worst recession since the global financial crisis. This led to the postponement or cancellation of sporting, religious, political and cultural events. Due to the fear, there was shortage of supply as more people purchased items like masks, sanitizers etc.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hi_D7s_VIjis"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}