preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import unicodedata\n",
    "import os\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import time\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prettify Monolingual Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "SOS_token = 0\n",
    "EOS_token = 1\n",
    "\n",
    "class Lang:\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.word2index = {}\n",
    "        self.word2count = {}\n",
    "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
    "        self.n_words = 2 # Count SOS and EOS\n",
    "      \n",
    "    def index_words(self, sentence):\n",
    "        for word in sentence.split(' '):\n",
    "            self.index_word(word)\n",
    "\n",
    "    def index_word(self, word):\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.n_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.n_words] = word\n",
    "            self.n_words += 1\n",
    "        else:\n",
    "            self.word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Turn a Unicode string to plain ASCII, thanks to http://stackoverflow.com/a/518232/2809427\n",
    "def unicode_to_ascii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "    )\n",
    "\n",
    "# Lowercase, trim, and remove non-letter characters\n",
    "def normalize_string(s):\n",
    "    s = unicode_to_ascii(s.strip())\n",
    "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
    "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def read_langs(lang1, lang2, reverse=False):\n",
    "    print(\"Reading lines...\")\n",
    "\n",
    "    # Read the file and split into lines\n",
    "    lines = open('./data/%s-%s.txt' % (lang1, lang2)).read().strip().split('\\n')\n",
    "    \n",
    "    # Split every line into pairs and normalize\n",
    "    pairs = [[normalize_string(s) for s in l.split('\\t')] for l in lines]\n",
    "    \n",
    "    # Reverse pairs, make Lang instances\n",
    "    if reverse:\n",
    "        pairs = [list(reversed(p)) for p in pairs]\n",
    "        input_lang = Lang(lang2)\n",
    "        output_lang = Lang(lang1)\n",
    "    else:\n",
    "        input_lang = Lang(lang1)\n",
    "        output_lang = Lang(lang2)\n",
    "        \n",
    "    return input_lang, output_lang, pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "MIN_LENGTH = 0\n",
    "MAX_LENGTH = 50\n",
    "\n",
    "def filter_pair(p):\n",
    "    return (MIN_LENGTH <= len(p[0].split(' ')) < MAX_LENGTH) and (MIN_LENGTH <= len(p[1].split(' ')) < MAX_LENGTH)\n",
    "\n",
    "def filter_pairs(pairs):\n",
    "    return [pair for pair in pairs if filter_pair(pair)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Read 142224 sentence pairs\n",
      "Trimmed to 142219 sentence pairs\n",
      "Indexing words...\n",
      "['Tom ne savait pas ou Marie etait allee skier .', 'Tom didn t know where Mary had gone skiing .']\n"
     ]
    }
   ],
   "source": [
    "def prepare_data(lang1_name, lang2_name, reverse=False):\n",
    "    input_lang, output_lang, pairs = read_langs(lang1_name, lang2_name, reverse)\n",
    "    print(\"Read %s sentence pairs\" % len(pairs))\n",
    "    \n",
    "    pairs = filter_pairs(pairs)\n",
    "    print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
    "    \n",
    "    print(\"Indexing words...\")\n",
    "    for pair in pairs:\n",
    "        input_lang.index_words(pair[0])\n",
    "        output_lang.index_words(pair[1])\n",
    "\n",
    "    return input_lang, output_lang, pairs\n",
    "\n",
    "input_lang, output_lang, pairs = prepare_data('eng', 'fra', True)\n",
    "\n",
    "# Print an example pair\n",
    "print(random.choice(pairs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "eng = [x[1] for x in pairs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with open('./nmt_data/mono.en', 'w') as f:\n",
    "    for i in range(len(eng)):\n",
    "        f.write(eng[i] + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Sentence Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_avg_embeddings(word2vec_name, fpath_L, fpath_U):\n",
    "\n",
    "    model = gensim.models.KeyedVectors.load_word2vec_format('data/utils/%s.bin' % word2vec_name, binary=True)\n",
    "    \n",
    "    # the elements of both matrices below constitute the nodes of our graph\n",
    "    with open(fpath_L, 'r') as f, open(fpath_U, 'r') as g:\n",
    "        ss_L = f.readlines()\n",
    "        ss_L = [x[:-1] for x in ss_L]\n",
    "\n",
    "        ss_U = g.readlines()\n",
    "        ss_U = [x[:-1] for x in ss_U]\n",
    "\n",
    "    # matrix of labeled embeddings\n",
    "    L = np.zeros((len(ss_L), model.vector_size), dtype='float32')\n",
    "\n",
    "    # matrix of unlabeled embeddings\n",
    "    U = np.zeros((len(ss_U), model.vector_size), dtype='float32')\n",
    "\n",
    "    def word2vec(w):\n",
    "        \"\"\"\n",
    "        with this quick trick I can calculate the embeddings without normalizing the text (removing puctuaction, stop words etc...)\n",
    "        If I pass a word that is not in the word2vec_model, like a stopword or some weird symbol, it just returns a zero vector that\n",
    "        does not cotribute to the avg embedding\n",
    "        \"\"\"\n",
    "        out = np.zeros(model.vector_size)\n",
    "        try:\n",
    "            out = model.word_vec(w)\n",
    "        finally:\n",
    "            return out\n",
    "\n",
    "    i = 0\n",
    "    for s in ss_L:\n",
    "        words = word_tokenize(s)\n",
    "        \n",
    "        # embedding for review is calculated as average of the embeddings of all words\n",
    "        # this is not ideal but is shown to work reasonably well in literature\n",
    "        # if you need something a bit more sophisticated, look into Doc2Vec algorithms\n",
    "        tmp = [word2vec(w) for w in words]\n",
    "        if(len(tmp)==0):\n",
    "            val = 0\n",
    "        else:\n",
    "            val = np.mean([word2vec(w) for w in words], axis=0)\n",
    "        L[i] = val\n",
    "        print(str(i), end='\\r')\n",
    "        i = i+1\n",
    "        \n",
    "    print()\n",
    "    \n",
    "    with open('./data/graph/labeled.pickle', 'wb') as f:\n",
    "        pickle.dump(L, f)\n",
    "\n",
    "\n",
    "    j=0\n",
    "    for s in ss_U:\n",
    "        words = word_tokenize(s)\n",
    "\n",
    "        U[j] = np.mean([word2vec(w) for w in words], axis=0)\n",
    "        print(str(j), end='\\r')\n",
    "        j = j+1\n",
    "\n",
    "    with open('./data/graph/unlabeled.pickle', 'wb') as f:\n",
    "        pickle.dump(U, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131140\r"
     ]
    }
   ],
   "source": [
    "create_avg_embeddings('GoogleNews-vectors-negative300', './nmt_data/train.en', './nmt_data/mono.en')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}