hw3-Multitask-InputSenses.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "import numpy as np\n",
    "from math import ceil\n",
    "import re\n",
    "from pandas_ml import ConfusionMatrix\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import classification_report\n",
    "import time\n",
    "from util import *\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the embeddings matrix and an embedding dictionary\n",
    "word_sense_embeddings,embedding_dict, hidden_size = load_embeddings('../babelfy_vectors_slim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_train = \"../SRLData/EN/CoNLL2009-ST-English-train.txt\"\n",
    "path_dev = \"../SRLData/EN/CoNLL2009-ST-English-development.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read training data\n",
    "tr_raw_sentences = read_conll(path_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding semantic info\n",
    "add_semantic_info_conll(tr_raw_sentences,'../disambiguated_words_conll_train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate encoders for POS and ARGS\n",
    "pos_tags,pos_tag_encoder = list_pos_tags(tr_raw_sentences)\n",
    "args, args_encoder = list_args(tr_raw_sentences)\n",
    "\n",
    "args_classes = len(args)\n",
    "pos_tags_classes = len(pos_tags)\n",
    "null_code = args_encoder.transform(['_'])[0] #No classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicate-centered sentences / Windowing\n",
    "left_words = 20\n",
    "right_words = 10\n",
    "window_span = [left_words,right_words]\n",
    "sentence_length = left_words + right_words + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate inputs for the network \n",
    "tr_sentences,tr_pred_inds = generate_inputs(tr_raw_sentences,embedding_dict,pos_tag_encoder,window_span)\n",
    "tr_labels,tr_missed = generate_labels_pos(tr_raw_sentences,args_encoder,pos_tag_encoder,window_span)\n",
    "\n",
    "# Padding\n",
    "tr_sentences,tr_lens = pad(tr_sentences,max_length = sentence_length)\n",
    "tr_labels,_ = pad(tr_labels,max_length = sentence_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Development data\n",
    "dev_raw_sentences = read_conll(path_dev)\n",
    "add_semantic_info_conll(dev_raw_sentences,'../disambiguated_words_conll_dev')\n",
    "\n",
    "dev_sentences,dev_pred_inds = generate_inputs(dev_raw_sentences,embedding_dict,pos_tag_encoder,window_span)\n",
    "dev_labels,dev_missed = generate_labels_pos(dev_raw_sentences,args_encoder,pos_tag_encoder,window_span)\n",
    "\n",
    "dev_sentences, dev_lens = pad(dev_sentences,max_length = sentence_length)\n",
    "dev_labels,_ = pad(dev_labels,max_length = sentence_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "del dev_raw_sentences,tr_raw_sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network\n",
    "\n",
    "    in_sentences: Input sentences. Shape is (batch,sentence_length,2).\n",
    "    in_labels: Input labels. Shape is (batch,sentence_length,2)\n",
    "    in_lens: Input sentences' lengths. Shape is (batch)\n",
    "    in_pred_inds: Input predicate indexes. Shape is (batch)\n",
    "    in_prob_dropout: Input probability for dropout.\n",
    "    in_learn_rate: Input learning rate\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_units = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_sentences = tf.placeholder(tf.int32, shape=[None,sentence_length,2],name='in_sentences')\n",
    "in_labels = tf.placeholder(tf.int32, shape=[None,sentence_length,2],name='in_labels')\n",
    "in_lens = tf.placeholder(tf.int32, shape=[None],name='in_lens')\n",
    "in_pred_inds = tf.placeholder(tf.int32,shape=[None],name='in_pred_inds')\n",
    "in_prob_dropout = tf.placeholder(tf.float32,name='in_prob_dropout')\n",
    "in_learn_rate = tf.placeholder(tf.float32,name='in_learn_rate')\n",
    "\n",
    "t_batch_size = tf.shape(in_sentences)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input assembling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Slicing the sentences vector in order to place the embeddings\n",
    "# Shape is (batch,sentence_length)\n",
    "t_sentences_word_indexes = tf.squeeze(tf.slice(in_sentences,(0,0,0),(t_batch_size,sentence_length,1)),-1)\n",
    "\n",
    "embeddings = tf.Variable(word_sense_embeddings,dtype=tf.float32,trainable=False)\n",
    "\n",
    "# Replacing the embedding values with the embeddings\n",
    "# Shape is (batch,sentence_length,hidden_size)\n",
    "t_sentences_embeddings = tf.nn.embedding_lookup(embeddings,t_sentences_word_indexes)\n",
    "\n",
    "# Slicing the sentences vector in order to get the pos values\n",
    "# Shape is (batch,sentence_length,1)\n",
    "t_sentences_word_pos = tf.cast(tf.slice(in_sentences,(0,0,1),(t_batch_size,sentence_length,1)),tf.float32)\n",
    "\n",
    "# Concatenating in order to generate the vector\n",
    "# Shape is (batch,sentence_length,hidden_size + 1)\n",
    "t_sentences = tf.concat([t_sentences_embeddings,t_sentences_word_pos],-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating the binary vectors to append to the sentences\n",
    "t_bin_vects = tf.one_hot(in_pred_inds,sentence_length)\n",
    "t_bin_vects = tf.expand_dims(t_bin_vects,-1)\n",
    "\n",
    "t_sentences = tf.concat([t_sentences,t_bin_vects],axis=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cell_fw = tf.contrib.rnn.LSTMCell(lstm_units)\n",
    "cell_bw = tf.contrib.rnn.LSTMCell(lstm_units)\n",
    "\n",
    "# Dropout\n",
    "cell_fw = tf.nn.rnn_cell.DropoutWrapper(cell_fw, output_keep_prob=in_prob_dropout)\n",
    "cell_bw = tf.nn.rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=in_prob_dropout)\n",
    "\n",
    "t_bilstm,_ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, t_sentences,sequence_length=in_lens, dtype=tf.float32)\n",
    "\n",
    "# Concatenating the two hidden states of the BiLSTM output\n",
    "# Shape is (batch,sentence_length,lstm_units*2)\n",
    "t_bilstm = tf.concat([t_bilstm[0],t_bilstm[1]],axis=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input assembling for the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenating the batch_index to each predicate index (needed for gather)\n",
    "# Shape is (batch,2)\n",
    "t_pred_inds = tf.stack([tf.range(t_batch_size),in_pred_inds],axis=1)\n",
    "\n",
    "# Selecting the predicates with the right indexes. More precisely, we pick one vector from the sentence_length \n",
    "# dimension for every sentence\n",
    "# Shape is (batch,lstm_units*2)\n",
    "t_preds = tf.gather_nd(t_bilstm,t_pred_inds)\n",
    "\n",
    "# Adding the lenght dimension in order to tile\n",
    "# Shape is (batch,1,lstm_units*2)\n",
    "t_preds = tf.expand_dims(t_preds,1)\n",
    "\n",
    "# Tiling the vector predicate along the length dimension in order to concatenate\n",
    "# Shape is (batch,sentence_length,lstm_units*2)\n",
    "t_preds = tf.tile(t_preds,[1,sentence_length,1])\n",
    "\n",
    "# Concatenating along the last dimension word and predicate features\n",
    "# Shape is (batch,sentence_length,lstm_units*4)\n",
    "t_pairs = tf.concat([t_bilstm,t_preds],axis=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Padding removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating a mask for the padding using the input lengths\n",
    "# Shape is (batch,sentence_length)\n",
    "t_mask = tf.sequence_mask(in_lens)\n",
    "\n",
    "# Applying the mask on the word-predicate pairs, removing spurious pairs. T is the number of times 'True' appears\n",
    "# in the mask or the sum of all lengths in in_lens.\n",
    "# Shape is (T,lstm_units*4)\n",
    "t_pairs = tf.boolean_mask(t_pairs,t_mask)\n",
    "\n",
    "# Shape of the vectors feed to the classifier\n",
    "vect_shape = t_pairs.get_shape().as_list()[1]\n",
    "\n",
    "# Applying the mask on the labels\n",
    "# Shape is (T,2)\n",
    "t_labels = tf.boolean_mask(in_labels,t_mask)\n",
    "\n",
    "t_T = tf.shape(t_labels)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Labels extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Slicing in order to get the labels for the args\n",
    "# Shape is (T)\n",
    "t_labels_args = tf.squeeze(tf.slice(t_labels,(0,0),(t_T,1)),-1)\n",
    "\n",
    "# Slicing in order to get the labels for the pos tags\n",
    "# Shape is (T)\n",
    "t_labels_pos = tf.squeeze(tf.slice(t_labels,(0,1),(t_T,1)),-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logits and loss for the word-predicate labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_args = tf.Variable(tf.truncated_normal([vect_shape,args_classes],stddev=0.1))\n",
    "b_args = tf.Variable(tf.constant(0., shape=[args_classes]))\n",
    "\n",
    "t_logits_args = tf.matmul(t_pairs, W_args) + b_args\n",
    "t_loss_args = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=t_labels_args,logits=t_logits_args,name='sparse_softmax_args')\n",
    "t_loss_args = tf.reduce_mean(t_loss_args,name='t_loss_args')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logits and loss for the pos tag lables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_pos = tf.Variable(tf.truncated_normal([vect_shape,pos_tags_classes],stddev=0.1))\n",
    "b_pos = tf.Variable(tf.constant(0., shape=[pos_tags_classes]))\n",
    "\n",
    "t_logits_pos = tf.matmul(t_pairs, W_pos) + b_pos\n",
    "t_loss_pos = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=t_labels_pos,logits=t_logits_pos,name='sparse_softmax_pos')\n",
    "t_loss_pos = tf.reduce_mean(t_loss_pos,name='t_loss_pos')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:98: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "t_optimizer = tf.train.AdamOptimizer(in_learn_rate).minimize(t_loss_args + t_loss_pos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions and Accuracy for Roles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "t_args = tf.nn.softmax(t_logits_args)\n",
    "t_args = tf.argmax(t_args,1,output_type=tf.int32)\n",
    "\n",
    "t_accuracy_args = tf.equal(t_args,t_labels_args)\n",
    "t_accuracy_args = tf.reduce_mean(tf.cast(t_accuracy_args,tf.float32),name='t_accuracy_args')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions and Accuracy for POS tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_pos_tags = tf.nn.softmax(t_logits_pos)\n",
    "t_pos_tags = tf.argmax(t_pos_tags,1,output_type=tf.int32)\n",
    "\n",
    "t_accuracy_pos = tf.equal(t_pos_tags,t_labels_pos)\n",
    "t_accuracy_pos = tf.reduce_mean(tf.cast(t_accuracy_pos,tf.float32),name='t_accuracy_pos')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "summary_writer = tf.summary.FileWriter('./summary', sess.graph)\n",
    "summary_writer.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model-Multitask-InputSenses/model.ckpt\n",
      "Previous model restored.\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "\n",
    "if not os.path.exists(\"model-Multitask-InputSenses\"):\n",
    "    os.makedirs(\"model-Multitask-InputSenses\")\n",
    "    \n",
    "if tf.train.checkpoint_exists('./model-Multitask-InputSenses/model.ckpt'):\n",
    "    saver.restore(sess, './model-Multitask-InputSenses/model.ckpt')\n",
    "    print(\"Previous model restored.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration # 0 time: 3.747049000000004\n",
      "Train data)  Precision: 0 Recall: 0.0 F1 score: 0\n",
      "Train data)  Accuracy on args: 0.899899 Accuracy on pos tags: 0.0716848\n",
      "Dev data) Precision: 0 Recall: 0.0 F1 score: 0\n",
      "Dev data) Accuracy on args: 0.900079 Accuracy on pos tags: 0.0870422\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-af4e493e95da>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0min_sentences\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mbt_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_labels\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mbt_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_pred_inds\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbt_pred_inds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0min_lens\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbt_lens\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0min_learn_rate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlearn_rate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0min_prob_dropout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mkeep_prob\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_optimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1359\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1338\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "epochs = 30\n",
    "batch_size = 100\n",
    "keep_prob = 0.6\n",
    "learn_rate = 0.005\n",
    "\n",
    "batch_index = 0\n",
    "num_batches_per_epoch = ceil(len(tr_labels)/batch_size)\n",
    "n_iterations = num_batches_per_epoch*epochs\n",
    "\n",
    "# Part of training data in order to check overfitting\n",
    "\n",
    "tr_over_sentences = tr_sentences[:1000]\n",
    "tr_over_labels = tr_labels[:1000]\n",
    "tr_over_pred_inds = tr_pred_inds[:1000]\n",
    "tr_over_lens = tr_lens[:1000]\n",
    "tr_over_missed = tr_missed[:1000]\n",
    "\n",
    "# Computing the initial F1 score\n",
    "\n",
    "feed_dict = {in_sentences : dev_sentences, in_labels :dev_labels, in_pred_inds: dev_pred_inds,in_lens:dev_lens,in_prob_dropout:1}\n",
    "o_args,o_labels_args = sess.run([t_args,t_labels_args],feed_dict=feed_dict)\n",
    "_,_,max_f1 = compute_scores(o_args,o_labels_args,null_code,dev_missed)\n",
    "\n",
    "for ite in range(n_iterations):\n",
    "    start = time.clock()\n",
    "    \n",
    "    # Batch\n",
    "    bt_sentences = tr_sentences[batch_index*batch_size:(batch_index+1)*batch_size]\n",
    "    bt_labels = tr_labels[batch_index*batch_size:(batch_index+1)*batch_size]\n",
    "    bt_pred_inds = tr_pred_inds[batch_index*batch_size:(batch_index+1)*batch_size]\n",
    "    bt_lens = tr_lens[batch_index*batch_size:(batch_index+1)*batch_size]\n",
    "    \n",
    "    batch_index = (batch_index + 1 ) % num_batches_per_epoch\n",
    "    \n",
    "    feed_dict = {in_sentences : bt_sentences, in_labels :bt_labels, in_pred_inds: bt_pred_inds,in_lens:bt_lens,in_learn_rate:learn_rate,in_prob_dropout:keep_prob}\n",
    "    sess.run(t_optimizer,feed_dict=feed_dict)\n",
    "    \n",
    "    end = time.clock()\n",
    "    \n",
    "    if ite % 10 == 0:\n",
    "        print('Iteration # ' + str(ite) + ' time: ' + str(end-start))\n",
    "        \n",
    "    if ite % 100 == 0:\n",
    "        \n",
    "        # Training data\n",
    "        feed_dict = {in_sentences : tr_over_sentences, in_labels :tr_over_labels, in_pred_inds: tr_over_pred_inds,in_lens:tr_over_lens,in_prob_dropout:1}\n",
    "        o_args,o_labels_args,o_accuracy_args,o_accuracy_pos = sess.run([t_args,t_labels_args,t_accuracy_args,t_accuracy_pos],feed_dict=feed_dict)\n",
    "        precision,recall,f1_score = compute_scores(o_args,o_labels_args,null_code,tr_over_missed)\n",
    "        print('Train data)  Precision: ' + str(precision) + ' Recall: ' +str(recall)+ ' F1 score: '+str(f1_score) )\n",
    "        print('Train data)  Accuracy on args: '+  str(o_accuracy_args) +  ' Accuracy on pos tags: ' + str(o_accuracy_pos))\n",
    "        \n",
    "        # Development data\n",
    "        feed_dict = {in_sentences : dev_sentences, in_labels :dev_labels, in_pred_inds: dev_pred_inds,in_lens:dev_lens,in_prob_dropout:1}\n",
    "        o_args,o_labels_args,o_accuracy_args,o_accuracy_pos = sess.run([t_args,t_labels_args,t_accuracy_args,t_accuracy_pos],feed_dict=feed_dict)\n",
    "        precision,recall,f1_score = compute_scores(o_args,o_labels_args,null_code,dev_missed)\n",
    "        print('Dev data) Precision: ' + str(precision) + ' Recall: ' +str(recall)+ ' F1 score: '+str(f1_score),flush=True )\n",
    "        print('Dev data) Accuracy on args: '+  str(o_accuracy_args) +  ' Accuracy on pos tags: ' + str(o_accuracy_pos))\n",
    "        \n",
    "        # Save weights only if the score improved\n",
    "        if f1_score >= max_f1:\n",
    "            max_f1 = f1_score\n",
    "            saver.save(sess, './model-Multitask-InputSenses/model.ckpt')\n",
    "            print('---Weights have been saved---')        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model-Multitask-InputSenses/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "# Loading the weights with the highest F1 score\n",
    "saver.restore(sess, './model-Multitask-InputSenses/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dev data) Precision: 0.890807651434644 Recall: 0.7254958528669311 F1 score: 0.7996978972055492\n",
      "Dev data) Accuracy on args: 0.965482 Accuracy on pos tags: 0.989537\n"
     ]
    }
   ],
   "source": [
    "feed_dict = {in_sentences : dev_sentences, in_labels :dev_labels, in_pred_inds: dev_pred_inds,in_lens:dev_lens,in_prob_dropout:1}\n",
    "o_args,o_labels_args,o_accuracy_args,o_accuracy_pos = sess.run([t_args,t_labels_args,t_accuracy_args,t_accuracy_pos],feed_dict=feed_dict)\n",
    "precision,recall,f1_score = compute_scores(o_args,o_labels_args,null_code,dev_missed)\n",
    "print('Dev data) Precision: ' + str(precision) + ' Recall: ' +str(recall)+ ' F1 score: '+str(f1_score) )\n",
    "print('Dev data) Accuracy on args: '+  str(o_accuracy_args) +  ' Accuracy on pos tags: ' + str(o_accuracy_pos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "o_labels_args = args_encoder.inverse_transform(o_labels_args)\n",
    "o_args = args_encoder.inverse_transform(o_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted    A0    A1    A2   A3  A4  A5  AA  AM-ADV  AM-CAU  AM-DIR   ...     \\\n",
      "Actual                                                                 ...      \n",
      "A0         2611   147    36    3   1   0   0       0       0       0   ...      \n",
      "A1          116  4189    69    8   0   0   0       2       0       1   ...      \n",
      "A2           49   176   970    5   2   0   0       2       0       4   ...      \n",
      "A3           15    26    24  147   0   0   0       0       0       0   ...      \n",
      "A4            0     7    14    1  44   0   0       0       0       1   ...      \n",
      "A5            0     1     1    0   0   1   0       0       0       0   ...      \n",
      "AA            0     0     0    0   0   0   0       0       0       0   ...      \n",
      "AM-ADV        0     1     3    1   0   0   0     108       0       0   ...      \n",
      "AM-CAU        0     1     2    0   0   0   0       1      25       0   ...      \n",
      "AM-DIR        0     3     8    0   1   0   0       0       0      10   ...      \n",
      "AM-DIS        0     0     0    0   0   0   0      11       0       0   ...      \n",
      "AM-EXT        0     5     1    0   0   0   0       1       0       0   ...      \n",
      "AM-LOC       10    19    16    1   0   0   0       1       0       1   ...      \n",
      "AM-MNR        4    19    19    3   1   0   0      15       0       0   ...      \n",
      "AM-MOD        0     1     0    0   0   0   0       0       0       0   ...      \n",
      "AM-NEG        0     0     0    0   0   0   0       0       0       0   ...      \n",
      "AM-PNC        0     5     6    0   0   0   0       1       1       0   ...      \n",
      "AM-PRD        0     0     0    0   0   0   0       0       0       0   ...      \n",
      "AM-TMP        2    16     1    1   1   0   0      11       0       0   ...      \n",
      "C-A1          0    10     1    0   0   0   0       0       0       0   ...      \n",
      "C-A2          0     0     0    0   0   0   0       1       0       0   ...      \n",
      "C-AM-CAU      0     0     0    0   0   0   0       0       1       0   ...      \n",
      "C-AM-DIR      0     0     0    0   0   0   0       0       0       1   ...      \n",
      "C-AM-MNR      0     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-A0          2     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-A1          0     2     0    0   0   0   0       0       0       0   ...      \n",
      "R-A2          0     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-AM-CAU      0     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-AM-EXT      0     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-AM-LOC      0     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-AM-MNR      0     0     0    0   0   0   0       0       0       0   ...      \n",
      "R-AM-TMP      0     0     0    0   0   0   0       0       0       0   ...      \n",
      "_           338   419   137   13   6   0   0      26       3       4   ...      \n",
      "__all__    3147  5047  1308  183  56   1   0     180      30      22   ...      \n",
      "\n",
      "Predicted  R-A0  R-A1  R-A2  R-AM-CAU  R-AM-EXT  R-AM-LOC  R-AM-MNR  R-AM-TMP  \\\n",
      "Actual                                                                          \n",
      "A0            0     0     0         0         0         0         0         0   \n",
      "A1            0     1     0         0         0         0         0         0   \n",
      "A2            0     0     0         0         0         0         0         0   \n",
      "A3            0     0     0         0         0         0         0         0   \n",
      "A4            0     0     0         0         0         0         0         0   \n",
      "A5            0     0     0         0         0         0         0         0   \n",
      "AA            0     0     0         0         0         0         0         0   \n",
      "AM-ADV        0     0     0         0         0         0         0         0   \n",
      "AM-CAU        0     0     0         0         0         0         0         0   \n",
      "AM-DIR        0     0     0         0         0         0         0         0   \n",
      "AM-DIS        0     0     0         0         0         0         0         0   \n",
      "AM-EXT        0     0     0         0         0         0         0         0   \n",
      "AM-LOC        0     0     0         0         0         1         0         0   \n",
      "AM-MNR        0     0     0         0         0         0         0         0   \n",
      "AM-MOD        0     0     0         0         0         0         0         0   \n",
      "AM-NEG        0     0     0         0         0         0         0         0   \n",
      "AM-PNC        0     0     0         0         0         0         0         0   \n",
      "AM-PRD        0     0     0         0         0         0         0         0   \n",
      "AM-TMP        0     0     0         0         0         0         0         0   \n",
      "C-A1          0     0     0         0         0         0         0         0   \n",
      "C-A2          0     0     0         0         0         0         0         0   \n",
      "C-AM-CAU      0     0     0         0         0         0         0         0   \n",
      "C-AM-DIR      0     0     0         0         0         0         0         0   \n",
      "C-AM-MNR      0     0     0         0         0         0         0         0   \n",
      "R-A0        125     5     0         0         0         0         0         0   \n",
      "R-A1          2    67     1         0         0         0         0         0   \n",
      "R-A2          1     1     2         0         0         0         0         0   \n",
      "R-AM-CAU      0     0     0         0         0         0         0         0   \n",
      "R-AM-EXT      0     0     0         0         0         1         0         0   \n",
      "R-AM-LOC      0     0     0         0         0         6         0         0   \n",
      "R-AM-MNR      0     0     0         0         0         1         3         0   \n",
      "R-AM-TMP      0     0     0         0         0         0         0        21   \n",
      "_             6    10     0         1         0         6         0        11   \n",
      "__all__     134    84     3         1         0        15         3        32   \n",
      "\n",
      "Predicted       _  __all__  \n",
      "Actual                      \n",
      "A0            551     3363  \n",
      "A1            687     5112  \n",
      "A2            311     1566  \n",
      "A3             70      297  \n",
      "A4             12       82  \n",
      "A5              0        3  \n",
      "AA              1        1  \n",
      "AM-ADV         94      235  \n",
      "AM-CAU         10       41  \n",
      "AM-DIR         11       34  \n",
      "AM-DIS         50      195  \n",
      "AM-EXT          9       47  \n",
      "AM-LOC         82      330  \n",
      "AM-MNR        114      420  \n",
      "AM-MOD          9      314  \n",
      "AM-NEG          8      123  \n",
      "AM-PNC         28       71  \n",
      "AM-PRD          0        2  \n",
      "AM-TMP        163      828  \n",
      "C-A1           27      133  \n",
      "C-A2            1        2  \n",
      "C-AM-CAU        0        1  \n",
      "C-AM-DIR        0        1  \n",
      "C-AM-MNR        1        2  \n",
      "R-A0           12      144  \n",
      "R-A1           10       82  \n",
      "R-A2            1        5  \n",
      "R-AM-CAU        3        3  \n",
      "R-AM-EXT        0        1  \n",
      "R-AM-LOC        3        9  \n",
      "R-AM-MNR        2        6  \n",
      "R-AM-TMP       10       31  \n",
      "_          120229   121462  \n",
      "__all__    122509   134946  \n",
      "\n",
      "[34 rows x 34 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas_ml/confusion_matrix/abstract.py:66: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  df = df.loc[idx, idx.copy()].fillna(0)  # if some columns or rows are missing\n"
     ]
    }
   ],
   "source": [
    "conf_matr = ConfusionMatrix(o_labels_args,o_args)\n",
    "print(conf_matr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         A0       0.83      0.78      0.80      3363\n",
      "         A1       0.83      0.82      0.82      5112\n",
      "         A2       0.74      0.62      0.68      1566\n",
      "         A3       0.80      0.49      0.61       297\n",
      "         A4       0.79      0.54      0.64        82\n",
      "         A5       1.00      0.33      0.50         3\n",
      "         AA       0.00      0.00      0.00         1\n",
      "     AM-ADV       0.60      0.46      0.52       235\n",
      "     AM-CAU       0.83      0.61      0.70        41\n",
      "     AM-DIR       0.45      0.29      0.36        34\n",
      "     AM-DIS       0.80      0.64      0.71       195\n",
      "     AM-EXT       0.70      0.55      0.62        47\n",
      "     AM-LOC       0.61      0.59      0.60       330\n",
      "     AM-MNR       0.71      0.53      0.61       420\n",
      "     AM-MOD       0.97      0.97      0.97       314\n",
      "     AM-NEG       0.94      0.93      0.93       123\n",
      "     AM-PNC       0.54      0.41      0.46        71\n",
      "     AM-PRD       0.00      0.00      0.00         2\n",
      "     AM-TMP       0.80      0.75      0.78       828\n",
      "       C-A1       0.92      0.70      0.79       133\n",
      "       C-A2       0.00      0.00      0.00         2\n",
      "   C-AM-CAU       0.00      0.00      0.00         1\n",
      "   C-AM-DIR       0.00      0.00      0.00         1\n",
      "   C-AM-MNR       0.00      0.00      0.00         2\n",
      "       R-A0       0.93      0.87      0.90       144\n",
      "       R-A1       0.80      0.82      0.81        82\n",
      "       R-A2       0.67      0.40      0.50         5\n",
      "   R-AM-CAU       0.00      0.00      0.00         3\n",
      "   R-AM-EXT       0.00      0.00      0.00         1\n",
      "   R-AM-LOC       0.40      0.67      0.50         9\n",
      "   R-AM-MNR       1.00      0.50      0.67         6\n",
      "   R-AM-TMP       0.66      0.68      0.67        31\n",
      "          _       0.98      0.99      0.99    121462\n",
      "\n",
      "avg / total       0.96      0.97      0.96    134946\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/karapost/.local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "class_rep = classification_report(o_labels_args,o_args)\n",
    "print(class_rep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_test = '../TestData/test.csv'\n",
    "path_output_test = '../TestData/test_with_args.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_raw_sentences = read_conll(path_test)\n",
    "add_semantic_info_conll(test_raw_sentences,'../disambiguated_words_conll_test')\n",
    "test_sentences,test_pred_inds = generate_inputs(test_raw_sentences,embedding_dict,pos_tag_encoder,window_span)\n",
    "test_sentences,test_lens = pad(test_sentences,max_length=sentence_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prediction\n",
    "feed_dict = {in_sentences : test_sentences, in_pred_inds: test_pred_inds,in_lens:test_lens,in_prob_dropout:1}\n",
    "o_roles = sess.run(t_args,feed_dict=feed_dict)\n",
    "\n",
    "# Adding labels\n",
    "add_labels(test_raw_sentences,o_roles,window_span,args_encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Writing to a file\n",
    "write_labels_conll('../TestData/test.csv','../TestData/test_with_args.csv',test_raw_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}