steam-recommendation-wenning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis through Bi-LSTMs with Attention Layer using Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, I'll be looking at how to apply recurrent neural networks to the task of sentiment analysis by using Tensorflow.\n",
    "\n",
    "The review data is crawled from STEAM. https://store.steampowered.com/.\n",
    "\n",
    "The model structure is inspired by \"Hierarchical Attention Networks for Document Classification\", Zichao Yang et al. (http://www.aclweb.org/anthology/N16-1174). \n",
    "\n",
    "Specifically I used LSTM cell instead of GRU cell and can better deal with long text. And use pretrained word embedding vector to accelerate training stage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import shutil\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "from random import randint\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.python.ops.rnn import bidirectional_dynamic_rnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load pretrained GloVe word2vector model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For convenience, I used pretrained GloVe word2vector model as word embedding model. \n",
    "\n",
    "The model can be download here: https://nlp.stanford.edu/projects/glove/. Because of lacking resourses, I chose the model glove.6B.100d.txt. The matrix will contain 400,000 word vectors, each with a dimensionality of 100.\n",
    "\n",
    "If some word is not in the word2vector model, I'll use zero vector to represent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadGloveModel(gloveFilePath):\n",
    "     \n",
    "    print(\"Loading Glove Word2Vector Model...\")\n",
    "    f = open(gloveFilePath, 'r', encoding='utf-8')\n",
    "\n",
    "    word_to_index = dict()\n",
    "    index_to_vector = []\n",
    "    for idx, line in enumerate(f):\n",
    "        splitLine = line.split()\n",
    "        word = splitLine[0]\n",
    "        embedding = np.array([float(val) for val in splitLine[1:]])\n",
    "\n",
    "        word_to_index[word] = idx\n",
    "        index_to_vector.append(embedding)\n",
    "    \n",
    "    f.close()\n",
    "    \n",
    "    # deal with word out of dictionary\n",
    "    _WORD_NOT_FOUND = [0.0]* len(embedding)\n",
    "    _LAST_INDEX = idx + 1\n",
    "    \n",
    "    word_to_index = defaultdict(lambda: _LAST_INDEX, word_to_index)\n",
    "    index_to_vector += [_WORD_NOT_FOUND]\n",
    "    \n",
    "    print(\"{} words loaded!\".format(len(index_to_vector)))\n",
    "    return word_to_index, np.array(index_to_vector)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Glove Word2Vector Model...\n",
      "400001 words loaded!\n"
     ]
    }
   ],
   "source": [
    "word_to_index, index_to_vector = loadGloveModel(\"glove.6B.100d.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load crawled STEAM review data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load crawled STEAM review data and split data into training set and test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# load train & test data\n",
    "\n",
    "def load_reviews(data_file_path, max_seq_len):\n",
    "    r0_reviews = []\n",
    "    r1_reviews = []\n",
    "    r2_reviews = []\n",
    "    r3_reviews = []\n",
    "\n",
    "    with open(data_file_path, 'r', encoding='utf-8') as f:\n",
    "        for line in f.readlines():\n",
    "            attitude = line.split('\\t')[0]\n",
    "            review = line.split('\\t')[1]\n",
    "            if len(review) > max_seq_len:\n",
    "                review = review[:max_seq_len]\n",
    "            if int(attitude) == 0:\n",
    "                r0_reviews.append(review)\n",
    "            elif int(attitude) == 1:\n",
    "                r1_reviews.append(review)\n",
    "            elif int(attitude) == 2:\n",
    "                r2_reviews.append(review)\n",
    "            else:\n",
    "                r3_reviews.append(review)\n",
    "\n",
    "    all_reviews = np.array(r0_reviews + r1_reviews + r2_reviews + r3_reviews)\n",
    "    l0, l1, l2, l3 = len(r0_reviews), len(r1_reviews), len(r2_reviews), len(r3_reviews)\n",
    "    first_col = np.array([1] * l0 + [0] * l1 + [0] * l2 + [0] * l3).reshape(-1, 1)\n",
    "    second_col = np.array([0] * l0 + [1] * l1 + [0] * l2 + [0] * l3).reshape(-1, 1)\n",
    "    third_col = np.array([0] * l0 + [0] * l1 + [1] * l2 + [0] * l3).reshape(-1, 1)\n",
    "    fourth_col = np.array([0] * l0 + [0] * l1 + [0] * l2 + [1] * l3).reshape(-1, 1)\n",
    "    labels = np.concatenate([first_col, second_col, third_col, fourth_col], 1)\n",
    "\n",
    "    print(\"Numbers of rating 0 reviews: {}\".format(len(r0_reviews)))\n",
    "    print(\"Numbers of rating 1 reviews: {}\".format(len(r1_reviews)))\n",
    "    print(\"Numbers of rating 2 reviews: {}\".format(len(r2_reviews)))\n",
    "    print(\"Numbers of rating 3 reviews: {}\".format(len(r3_reviews)))\n",
    "    \n",
    "    return all_reviews, labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of rating 0 reviews: 3040\n",
      "Numbers of rating 1 reviews: 987\n",
      "Numbers of rating 2 reviews: 1451\n",
      "Numbers of rating 3 reviews: 4522\n"
     ]
    }
   ],
   "source": [
    "MAX_SEQUENCE_LEN = 400\n",
    "data_file_path = \"data/labeled_data2\"\n",
    "\n",
    "all_reviews, labels = load_reviews(data_file_path, MAX_SEQUENCE_LEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split training set and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split train & test\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    all_reviews, labels, test_size=0.2, random_state=42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 50\n",
    "VECTOR_DIM = 100\n",
    "LSTM_UNITS = 128\n",
    "ATTENTION_SIZE = 128\n",
    "NUM_CLASSES = 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bi-LSTM with Attention Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RNN(data, seq_len, lstm_units=100, attention_size=100):\n",
    "    \n",
    "    # bi-lstm\n",
    "    lstm_cell_1 = tf.contrib.rnn.BasicLSTMCell(lstm_units)  # lstm_units: output dimension of each cell\n",
    "    lstm_cell_2 = tf.contrib.rnn.BasicLSTMCell(lstm_units)\n",
    "    \n",
    "#     lstm_cell_1 = tf.contrib.rnn.GRUCell(lstm_units)\n",
    "#     lstm_cell_2 = tf.contrib.rnn.GRUCell(lstm_units)\n",
    "    \n",
    "    # lstm_cell_1 = tf.contrib.rnn.DropoutWrapper(cell=lstm_cell, output_keep_prob=0.75)\n",
    "    lstm_output, state = bidirectional_dynamic_rnn(lstm_cell_1, lstm_cell_2,\n",
    "                                                   inputs=data, sequence_length=seq_len,\n",
    "                                                   dtype=tf.float32)\n",
    "    # In case of bi-lstm, output is a tuple with forward and backward output\n",
    "    # output size:[batch_size, max_time, cell_fw.output_size]\n",
    "    \n",
    "    # attention layer\n",
    "    attention_input = tf.concat(lstm_output, 2)   # concat tuple on third dimension\n",
    "    \n",
    "    hidden_size = attention_input.shape[2].value  # should be 2 * lstm_units\n",
    "    \n",
    "    # Trainable parameters\n",
    "    ww = tf.Variable(tf.random_normal([hidden_size, attention_size], stddev=0.1))\n",
    "    bb = tf.Variable(tf.random_normal([attention_size], stddev=0.1))\n",
    "    uu = tf.Variable(tf.random_normal([attention_size], stddev=0.1))\n",
    "    \n",
    "    # convert B*T*D -> B*T*A\n",
    "    v = tf.tanh(tf.tensordot(attention_input, ww, axes=1) + bb)\n",
    "    \n",
    "    # convet B*T*A -> B*T\n",
    "    vu = tf.tensordot(v, uu, axes=1, name='vu')\n",
    "    alphas = tf.nn.softmax(vu, name='alphas')        # attention weights\n",
    "    \n",
    "    attention_output = tf.reduce_sum(attention_input * tf.expand_dims(alphas, -1), 1)\n",
    "    \n",
    "    return attention_output\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialized embedding layer with GloVe word embedding model. This can accelerate training process and get a higher result at the begining of training. Experiment result shows that set trainable be True can get better result.\n",
    "\n",
    "Experiment shows that use bi-lstm model can get a little better result than bi-gru model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/hangdu9595/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py:417: calling reverse_sequence (from tensorflow.python.ops.array_ops) with seq_dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "seq_dim is deprecated, use seq_axis instead\n",
      "WARNING:tensorflow:From /home/hangdu9595/anaconda3/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py:432: calling reverse_sequence (from tensorflow.python.ops.array_ops) with batch_dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "batch_dim is deprecated, use batch_axis instead\n"
     ]
    }
   ],
   "source": [
    "# model\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.name_scope('load_embedding_model'):\n",
    "    tf_embedding = tf.Variable(\n",
    "        tf.constant(0.0, shape=index_to_vector.shape),\n",
    "        trainable=True,   # ?\n",
    "        name=\"Embedding\"\n",
    "    )\n",
    "\n",
    "    embedding_ph = tf.placeholder(tf.float32, shape=index_to_vector.shape)\n",
    "    embedding_init = tf_embedding.assign(embedding_ph)\n",
    "\n",
    "# placeholders\n",
    "labels_ph = tf.placeholder(tf.float32, [None, NUM_CLASSES])\n",
    "batch_ph = tf.placeholder(tf.int32, [None, MAX_SEQUENCE_LEN])\n",
    "seq_len_ph = tf.placeholder(tf.int32, [None])\n",
    "keep_prob_ph = tf.placeholder(tf.float32)\n",
    "\n",
    "# embedding layer\n",
    "with tf.name_scope('embedding_layer'):\n",
    "    embedding_output = tf.nn.embedding_lookup(params=tf_embedding, ids=batch_ph)\n",
    "\n",
    "\n",
    "# bi-lstm with attention\n",
    "with tf.name_scope('rnn'):\n",
    "    rnn_output = RNN(embedding_output, seq_len_ph, lstm_units=LSTM_UNITS, attention_size=ATTENTION_SIZE)\n",
    "    tf.summary.histogram('rnn_outputs', rnn_output)\n",
    "\n",
    "# dropout layer\n",
    "with tf.name_scope('dropout_layer'):\n",
    "    dropout = tf.nn.dropout(rnn_output, keep_prob_ph)\n",
    "    dropout = dropout / keep_prob_ph\n",
    "\n",
    "# fully connected layer\n",
    "with tf.name_scope('fully_connected_layer'):\n",
    "    weight = tf.Variable(tf.truncated_normal([LSTM_UNITS * 2, NUM_CLASSES]))\n",
    "    bias = tf.Variable(tf.constant(0.1, shape=[NUM_CLASSES]))\n",
    "    pred = tf.nn.xw_plus_b(dropout, weight, bias)\n",
    "    #pred = tf.squeeze(pred)\n",
    "    pred = tf.nn.softmax(pred, name='pred')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-13-a8ad5b308746>:6: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# metrics\n",
    "learning_rate_ph = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('metrics'):\n",
    "    # cross entropy loss\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=labels_ph))\n",
    "    tf.summary.scalar('entropy_loss', loss)\n",
    "    # optimizer = tf.train.AdamOptimizer().minimize(loss)\n",
    "    optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate_ph).minimize(loss)\n",
    "\n",
    "    # accuracy\n",
    "    correctPred = tf.equal(tf.argmax(pred,1), tf.argmax(labels_ph,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32))\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    \n",
    "merged = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_generator(X, y, batch_size):\n",
    "    \"\"\"Primitive batch generator \n",
    "       copy from https://github.com/ilivans/tf-rnn-attention/blob/master/utils.py\n",
    "    \"\"\"\n",
    "    size = X.shape[0]\n",
    "    X_copy = X.copy()\n",
    "    y_copy = y.copy()\n",
    "    indices = np.arange(size)\n",
    "    np.random.shuffle(indices)\n",
    "    X_copy = X_copy[indices]\n",
    "    y_copy = y_copy[indices]\n",
    "    i = 0\n",
    "    while True:\n",
    "        if i + batch_size <= size:\n",
    "            yield X_copy[i:i + batch_size], y_copy[i:i + batch_size]\n",
    "            i += batch_size\n",
    "        else:\n",
    "            i = 0\n",
    "            indices = np.arange(size)\n",
    "            np.random.shuffle(indices)\n",
    "            X_copy = X_copy[indices]\n",
    "            y_copy = y_copy[indices]\n",
    "            continue\n",
    "            \n",
    "            \n",
    "def zero_pad(X, seq_len):\n",
    "    return np.array([x[:seq_len - 1] + [0] * max(seq_len - len(x), 1) for x in X])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "training parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training\n",
    "ITERATION = 50\n",
    "EARLY_STOP = 8\n",
    "PLATEAU = 3\n",
    "LEARNING_RATE = 1e-3\n",
    "REDUCED_LEARNING_RATE = 1e-4\n",
    "DELTA = 0.5\n",
    "KEEP_PROB = 1\n",
    "MODEL_PATH = './labeled_model/model.ckpt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/160 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training begins...\n",
      "epoch: 0\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:27<00:00,  6.07it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 12.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 1.250, val_loss: 1.267, acc: 0.449, val_acc: 0.464\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:27,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 1\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:27<00:00,  6.05it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 1.071, val_loss: 1.047, acc: 0.599, val_acc: 0.685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:28,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 2\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  6.12it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 1.070, val_loss: 1.024, acc: 0.683, val_acc: 0.718\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:27,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 3\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  6.33it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 1.017, val_loss: 1.001, acc: 0.717, val_acc: 0.744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:28,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 4\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  6.17it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 1.046, val_loss: 0.991, acc: 0.744, val_acc: 0.754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:28,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 5\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  6.13it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.991, val_loss: 0.970, acc: 0.764, val_acc: 0.776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:26,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 6\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.21it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.04it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.993, val_loss: 0.978, acc: 0.779, val_acc: 0.767\n",
      "model is not improved in current epoch!\n",
      "epoch: 7\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.29it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.36it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.989, val_loss: 0.982, acc: 0.794, val_acc: 0.757\n",
      "model is not improved in current epoch!\n",
      "epoch: 8\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.22it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.41it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 1.002, val_loss: 0.974, acc: 0.804, val_acc: 0.767\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 3 epoch, reduced learning rate!\n",
      "epoch: 9\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.12it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.883, val_loss: 0.954, acc: 0.834, val_acc: 0.788\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:25,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 10\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.31it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.36it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.880, val_loss: 0.952, acc: 0.842, val_acc: 0.788\n",
      "model is not improved in current epoch!\n",
      "epoch: 11\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  5.70it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.34it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.867, val_loss: 0.954, acc: 0.847, val_acc: 0.788\n",
      "model is not improved in current epoch!\n",
      "epoch: 12\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.13it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.874, val_loss: 0.951, acc: 0.851, val_acc: 0.790\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:26,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 13\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.44it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.900, val_loss: 0.951, acc: 0.859, val_acc: 0.791\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:26,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 14\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.33it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.62it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.863, val_loss: 0.949, acc: 0.864, val_acc: 0.791\n",
      "model is not improved in current epoch!\n",
      "epoch: 15\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.52it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.876, val_loss: 0.949, acc: 0.872, val_acc: 0.793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:27,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 16\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.23it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.35it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.887, val_loss: 0.951, acc: 0.876, val_acc: 0.789\n",
      "model is not improved in current epoch!\n",
      "epoch: 17\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.28it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.65it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.881, val_loss: 0.948, acc: 0.882, val_acc: 0.792\n",
      "model is not improved in current epoch!\n",
      "epoch: 18\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  5.70it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.88it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.829, val_loss: 0.950, acc: 0.887, val_acc: 0.792\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 3 epoch, reduced learning rate!\n",
      "epoch: 19\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.43it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.40it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.872, val_loss: 0.949, acc: 0.891, val_acc: 0.788\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 4 epoch, reduced learning rate!\n",
      "epoch: 20\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:24<00:00,  6.40it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.90it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.854, val_loss: 0.948, acc: 0.893, val_acc: 0.792\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 5 epoch, reduced learning rate!\n",
      "epoch: 21\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  5.92it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 12.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.867, val_loss: 0.947, acc: 0.897, val_acc: 0.795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:27,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 22\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  6.27it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.01it/s]\n",
      "  1%|          | 1/160 [00:00<00:27,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.852, val_loss: 0.947, acc: 0.901, val_acc: 0.790\n",
      "model is not improved in current epoch!\n",
      "epoch: 23\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.08it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.05it/s]\n",
      "  1%|          | 1/160 [00:00<00:28,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.842, val_loss: 0.946, acc: 0.903, val_acc: 0.793\n",
      "model is not improved in current epoch!\n",
      "epoch: 24\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.39it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.84it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.852, val_loss: 0.948, acc: 0.907, val_acc: 0.790\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 3 epoch, reduced learning rate!\n",
      "epoch: 25\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.33it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.65it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.811, val_loss: 0.947, acc: 0.909, val_acc: 0.794\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 4 epoch, reduced learning rate!\n",
      "epoch: 26\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.22it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.834, val_loss: 0.946, acc: 0.913, val_acc: 0.795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:25,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 27\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.38it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.71it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.824, val_loss: 0.945, acc: 0.914, val_acc: 0.793\n",
      "model is not improved in current epoch!\n",
      "epoch: 28\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.14it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.58it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.852, val_loss: 0.945, acc: 0.917, val_acc: 0.795\n",
      "model is not improved in current epoch!\n",
      "epoch: 29\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.38it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.62it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.874, val_loss: 0.946, acc: 0.920, val_acc: 0.793\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 3 epoch, reduced learning rate!\n",
      "epoch: 30\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.10it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 13.44it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.805, val_loss: 0.945, acc: 0.922, val_acc: 0.791\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 4 epoch, reduced learning rate!\n",
      "epoch: 31\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.32it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.818, val_loss: 0.941, acc: 0.923, val_acc: 0.800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/160 [00:00<00:27,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save model to./labeled_model/model.ckpt\n",
      "epoch: 32\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  5.93it/s]\n",
      "100%|██████████| 40/40 [00:03<00:00, 12.32it/s]\n",
      "  1%|          | 1/160 [00:00<00:28,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.862, val_loss: 0.942, acc: 0.924, val_acc: 0.796\n",
      "model is not improved in current epoch!\n",
      "epoch: 33\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:26<00:00,  6.32it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.48it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.805, val_loss: 0.943, acc: 0.925, val_acc: 0.795\n",
      "model is not improved in current epoch!\n",
      "epoch: 34\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:25<00:00,  6.10it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.87it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.837, val_loss: 0.941, acc: 0.927, val_acc: 0.796\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 3 epoch, reduced learning rate!\n",
      "epoch: 35\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:24<00:00,  6.59it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.94it/s]\n",
      "  1%|          | 1/160 [00:00<00:23,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.819, val_loss: 0.941, acc: 0.928, val_acc: 0.794\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 4 epoch, reduced learning rate!\n",
      "epoch: 36\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:24<00:00,  6.63it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 14.40it/s]\n",
      "  1%|          | 1/160 [00:00<00:24,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.814, val_loss: 0.941, acc: 0.929, val_acc: 0.796\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 5 epoch, reduced learning rate!\n",
      "epoch: 37\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:24<00:00,  6.69it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 14.35it/s]\n",
      "  1%|          | 1/160 [00:00<00:26,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.872, val_loss: 0.943, acc: 0.930, val_acc: 0.794\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 6 epoch, reduced learning rate!\n",
      "epoch: 38\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:24<00:00,  6.58it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.98it/s]\n",
      "  1%|          | 1/160 [00:00<00:25,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.794, val_loss: 0.943, acc: 0.931, val_acc: 0.792\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 7 epoch, reduced learning rate!\n",
      "epoch: 39\t"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160/160 [00:24<00:00,  6.67it/s]\n",
      "100%|██████████| 40/40 [00:02<00:00, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.813, val_loss: 0.942, acc: 0.932, val_acc: 0.795\n",
      "model is not improved in current epoch!\n",
      "model is not improved in the last 8 epoch, early stopped!\n",
      "training ends...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# batch generator\n",
    "train_batch_generator = batch_generator(X_train, y_train, BATCH_SIZE)\n",
    "test_batch_generator = batch_generator(X_test, y_test, BATCH_SIZE)\n",
    "\n",
    "# summary_writer for tensorboard\n",
    "if os.path.exists('labeled_logdir/test'):\n",
    "    shutil.rmtree('labeled_logdir/test')\n",
    "if os.path.exists('labeled_logdir/train'):\n",
    "    shutil.rmtree('labeled_logdir/train')\n",
    "    \n",
    "train_writer = tf.summary.FileWriter('./labeled_logdir/train', accuracy.graph)\n",
    "test_writer = tf.summary.FileWriter('./labeled_logdir/test', accuracy.graph)\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # load word2vec into tensorflow\n",
    "    _ = sess.run(embedding_init, \n",
    "                 feed_dict={embedding_ph: index_to_vector})\n",
    "    \n",
    "    del index_to_vector\n",
    "    \n",
    "    # used for early stopping\n",
    "    best_accuracy_test = 0\n",
    "    no_increase_count = 0\n",
    "    lr = LEARNING_RATE\n",
    "    \n",
    "    print(\"training begins...\")\n",
    "    for epoch in range(ITERATION):\n",
    "        \n",
    "        loss_train = 0\n",
    "        loss_test = 0\n",
    "        accuracy_train = 0\n",
    "        accuracy_test = 0\n",
    "        \n",
    "        print(\"epoch: {}\\t\".format(epoch), end=\"\")\n",
    "        \n",
    "        # training\n",
    "        num_batches = X_train.shape[0] // BATCH_SIZE\n",
    "        \n",
    "        for b in tqdm(range(num_batches)):\n",
    "            x_batch_raw, y_batch = next(train_batch_generator)\n",
    "            seq_len = np.array([len(s.split(' ')) for s in x_batch_raw])\n",
    "            x_batch_raw = np.array([s.split(' ') for s in x_batch_raw])      # string -> word array\n",
    "\n",
    "            x_batch = [[word_to_index[w.lower()] for w in sentense] for sentense in x_batch_raw]\n",
    "            x_batch = zero_pad(x_batch, MAX_SEQUENCE_LEN)\n",
    "\n",
    "            loss_tr, acc, _, summary = sess.run([loss, accuracy, optimizer, merged],\n",
    "                                                feed_dict={batch_ph: x_batch,\n",
    "                                                           labels_ph: y_batch,\n",
    "                                                           seq_len_ph: seq_len,\n",
    "                                                           learning_rate_ph: lr,\n",
    "                                                           keep_prob_ph: KEEP_PROB})\n",
    "            accuracy_train += acc\n",
    "            loss_train = loss_tr * DELTA + loss_train * (1 - DELTA)\n",
    "            train_writer.add_summary(summary, b + num_batches * epoch)\n",
    "\n",
    "        \n",
    "        accuracy_train /= num_batches\n",
    "\n",
    "        # testing\n",
    "        num_batches = X_test.shape[0] // BATCH_SIZE\n",
    "        # num_batches = 1\n",
    "        \n",
    "        for b in tqdm(range(num_batches)):\n",
    "            x_batch_raw, y_batch = next(test_batch_generator)\n",
    "            #x_batch_raw, y_batch = X_test, y_test\n",
    "            \n",
    "            seq_len = np.array([len(s.split(' ')) for s in x_batch_raw])\n",
    "            x_batch_raw = np.array([s.split(' ') for s in x_batch_raw])      # string -> word array\n",
    "\n",
    "            x_batch = [[word_to_index[w.lower()] for w in sentense] for sentense in x_batch_raw]\n",
    "            x_batch = zero_pad(x_batch, MAX_SEQUENCE_LEN)\n",
    "            \n",
    "            loss_test_batch, acc, summary = sess.run([loss, accuracy, merged],\n",
    "                                                     feed_dict={batch_ph: x_batch,\n",
    "                                                                labels_ph: y_batch,\n",
    "                                                                seq_len_ph: seq_len,\n",
    "                                                                keep_prob_ph: 1.0})\n",
    "            accuracy_test += acc\n",
    "            loss_test += loss_test_batch\n",
    "            test_writer.add_summary(summary, b + num_batches * epoch)\n",
    "\n",
    "        accuracy_test /= num_batches\n",
    "        loss_test /= num_batches\n",
    "\n",
    "        print(\"loss: {:.3f}, val_loss: {:.3f}, acc: {:.3f}, val_acc: {:.3f}\".format(\n",
    "            loss_train, loss_test, accuracy_train, accuracy_test\n",
    "        ))\n",
    "        \n",
    "        if best_accuracy_test < accuracy_test:\n",
    "            best_accuracy_test = accuracy_test\n",
    "            saver.save(sess, MODEL_PATH)\n",
    "            print(\"save model to\" + MODEL_PATH)\n",
    "            no_increase_count = 0\n",
    "        else:\n",
    "            print(\"model is not improved in current epoch!\")\n",
    "            no_increase_count += 1\n",
    "        \n",
    "        if no_increase_count == EARLY_STOP:\n",
    "            print(\"model is not improved in the last {} epoch, early stopped!\".format(EARLY_STOP))\n",
    "            break\n",
    "            \n",
    "        if no_increase_count >= PLATEAU:\n",
    "            print(\"model is not improved in the last {} epoch, reduced learning rate!\".format(no_increase_count))\n",
    "            lr = REDUCED_LEARNING_RATE\n",
    "#         else:\n",
    "#             lr = LEARNING_RATE\n",
    "        \n",
    "        \n",
    "    train_writer.close()\n",
    "    test_writer.close()\n",
    "    # saver.save(sess, MODEL_PATH)\n",
    "    \n",
    "    print(\"training ends...\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Predict Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./labeled_model/model.ckpt\n",
      "(2000, 4)\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, MODEL_PATH)\n",
    "\n",
    "    all_data, labels = X_test, y_test\n",
    "    \n",
    "    seq_len = np.array([len(s.split(' ')) for s in all_data])\n",
    "    input_data = np.array([s.split(' ') for s in all_data])      # string -> word array\n",
    "    \n",
    "    input_data = [[word_to_index[w.lower()] for w in sentense] for sentense in input_data]\n",
    "    input_data = zero_pad(input_data, MAX_SEQUENCE_LEN)\n",
    "\n",
    "    result = sess.run(pred, feed_dict={batch_ph: input_data,\n",
    "                                        seq_len_ph: seq_len,\n",
    "                                        keep_prob_ph: 1.0})\n",
    "\n",
    "\n",
    "#result = np.concatenate([result1, result2])\n",
    "\n",
    "print(result.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy is: 0.8\n"
     ]
    }
   ],
   "source": [
    "count_wrong = 0\n",
    "\n",
    "for i in range(len(result)):\n",
    "    if np.argmax(result[i], 0) != np.argmax(labels[i], 0):\n",
    "        count_wrong += 1\n",
    "\n",
    "print(\"accuracy is: {}\".format(1 - count_wrong / len(result)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = []\n",
    "for i in result:\n",
    "    output.append(np.argmax(i))\n",
    "\n",
    "with open('text_pred_res', 'w') as f:\n",
    "    f.write(str(output))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}