Syntactic Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from src.data_processing import load_data\n",
    "from src.pronunciations import get_closest_sounding_words\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize, pos_tag\n",
    "from src.ngrams import get_three_gram_wildcard\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/ngram_frequencies.json\") as f:\n",
    "    ngram_frequencies = json.load(f)\n",
    "    \n",
    "task1, task2, task3, min_pairs = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'ADJ': 154,\n",
       "             'ADV': 200,\n",
       "             'CONJ': 1,\n",
       "             'NOUN': 699,\n",
       "             'NUM': 3,\n",
       "             'PRT': 1,\n",
       "             'VERB': 213})"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_counter = defaultdict(int)\n",
    "normal_pos = {'ADJ', 'ADV', 'NOUN', 'VERB'}\n",
    "for context in task2:\n",
    "    text = word_tokenize(' '.join(context['words']))\n",
    "    pos_text = pos_tag(text, tagset='universal')\n",
    "    target_pos = pos_text[text.index(context['target'])][1]\n",
    "    pos_counter[target_pos] += 1\n",
    "pos_counter\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'target': 'sedimental',\n",
       " 'words': ['I',\n",
       "  'just',\n",
       "  'had',\n",
       "  'some',\n",
       "  'coffee',\n",
       "  'that',\n",
       "  'was',\n",
       "  'good',\n",
       "  'only',\n",
       "  'for',\n",
       "  'its',\n",
       "  'sedimental',\n",
       "  'value',\n",
       "  '.']}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "task2[57]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngram_frequencies = json.load(open(\"data/ngram_frequencies.json\"))\n",
    "task2_frequencies = [n for i, n in enumerate(ngram_frequencies) if task1[i]['pun']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_freqencies = [{'frequencies': t2} for t2 in task2_frequencies]\n",
    "for i, t in enumerate(task2):\n",
    "    test_freqencies[i]['text'] = t['words']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "accepted_pos = {'ADV', 'ADJ', 'VERB', 'NOUN'}\n",
    "def ngram_search(index):\n",
    "    context = test_freqencies[index]\n",
    "    threshold = 0\n",
    "    res = {}\n",
    "    print(index)\n",
    "    pos_text = pos_tag(context['text'], tagset='universal')\n",
    "    for i, (trigram, freq) in enumerate(context['frequencies']):\n",
    "        print(i, trigram, pos_text[i+1][1], freq)\n",
    "        if freq <= threshold and pos_text[i+1][1] in accepted_pos:\n",
    "#             print(trigram[1], i+1, freq)\n",
    "            res[(trigram[1], i+1, freq)] = get_three_gram_wildcard(trigram[0], trigram[2])\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57\n",
      "0 ['I', 'just', 'had'] ADV 699084\n",
      "1 ['just', 'had', 'some'] VERB 24423\n",
      "2 ['had', 'some', 'coffee'] DET 4128\n",
      "3 ['some', 'coffee', 'that'] NOUN 496\n",
      "4 ['coffee', 'that', 'was'] DET 2799\n",
      "5 ['that', 'was', 'good'] VERB 123362\n",
      "6 ['was', 'good', 'only'] ADJ 2062\n",
      "7 ['good', 'only', 'for'] ADV 18613\n",
      "8 ['only', 'for', 'its'] ADP 75294\n",
      "9 ['for', 'its', 'sedimental'] PRON 0\n",
      "10 ['its', 'sedimental', 'value'] ADJ 0\n",
      "11 ['sedimental', 'value', '.'] NOUN 0\n"
     ]
    }
   ],
   "source": [
    "ngram_search(57)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}