south-park-char-rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import tensorflow as tf\n",
    "from preeminence_utils import tf_utils\n",
    "import numpy as np\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_latest_epoch():\n",
    "    models = os.listdir(\"./model/model_southpark/\")\n",
    "    all_epochs = [int(model[model.find(\"-\")+1:].replace(\".meta\",\"\")) for model in models if \"meta\" in model]\n",
    "    return max(all_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     17
    ]
   },
   "outputs": [],
   "source": [
    "# initialise text variables\n",
    "data_file = \"./data/south_park/all_scripts.txt\"\n",
    "text = open(data_file).read().strip()\n",
    "print(len(text))\n",
    "vocab = sorted(list(set(text)))\n",
    "vocab_length = len(vocab)\n",
    "characters2id = dict((c, i) for i, c in enumerate(vocab))\n",
    "id2characters = dict((i, c) for i, c in enumerate(vocab))\n",
    "section_length = 50\n",
    "step = 10\n",
    "sections = []\n",
    "section_labels = []\n",
    "for i in range(0,len(text)-section_length,step):\n",
    "    sections.append(text[i:i+section_length])\n",
    "    section_labels.append(text[i+section_length])\n",
    "\n",
    "X_data = np.zeros((len(sections),section_length,vocab_length))\n",
    "Y_data = np.zeros((len(sections),vocab_length))\n",
    "for i,section in enumerate(sections):\n",
    "    for j,letter in enumerate(section):\n",
    "        X_data[i,j,characters2id[letter]] = 1\n",
    "    Y_data[i,characters2id[section_labels[i]]] = 1\n",
    "\n",
    "print(X_data.shape,Y_data.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = tf_utils.Model()\n",
    "model_graph = model.init().as_default()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "total_epochs = 500\n",
    "batch_size = 128\n",
    "log_every = 100\n",
    "save_every = 10\n",
    "hidden_nodes = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32,[None,section_length,vocab_length],name=\"X_train\")\n",
    "Y = tf.placeholder(tf.float32,[None,vocab_length],name=\"Y_train\")\n",
    "\n",
    "W = tf.Variable(tf.random_normal([hidden_nodes,vocab_length]),name=\"Output_weights\")\n",
    "b = tf.Variable(tf.random_normal([vocab_length]),name=\"Output_bias\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm(x,weights,bias,name_scope=\"lstm\"):\n",
    "    with tf.name_scope(name_scope):\n",
    "        x = tf.unstack(x,section_length,1)\n",
    "        lstm_cell = tf.contrib.rnn.BasicLSTMCell(hidden_nodes,forget_bias=1.0)\n",
    "        outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x, dtype=tf.float32)\n",
    "        return tf.matmul(outputs[-1],W)+b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logits = lstm(X,W,b)\n",
    "prediction = tf.nn.softmax(logits)\n",
    "loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "train_op = optimizer.minimize(loss_op)\n",
    "\n",
    "correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "# saver = tf.train.Saver(max_to_keep=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training\n",
    "epoch_start = get_latest_epoch()\n",
    "print(\"Resuming training from epoch: {}\".format(epoch_start))\n",
    "batch_size = 1024\n",
    "with model.session() as sess:\n",
    "#     sess.run(init)\n",
    "    model.restore_weights(\"./model/model_southpark/\")\n",
    "    for i in range(1):\n",
    "        ops = model.train([train_op,loss_op],X,Y,X_data,Y_data,num_epochs=1,batch_size=batch_size)\n",
    "        model.save_weights(checkpoint_path=\"./model/model_southpark/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample2(preds, temperature=1.0):\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.log(preds) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return probas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weights(weights_path):\n",
    "    if os.path.exists(weights_path):\n",
    "        weight_files = os.listdir(weights_path)\n",
    "    else:\n",
    "        return None\n",
    "#     print(weight_files)\n",
    "    weights = []\n",
    "    for weight_file in weight_files:\n",
    "        if \"meta\" in weight_file:\n",
    "#             print(weight_file)\n",
    "            weights.append(weight_file.split(\"-\")[1][:-5])\n",
    "    return sorted([int(weight) for weight in weights])\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing\n",
    "# test_start = \"STAN I said 'We're not getting on, you fat ugly bitch'.\"[:50]\n",
    "# section = [test_start]\n",
    "prediction_length = 500\n",
    "epoch_test = get_latest_epoch()\n",
    "# print(\"Testing epoch: {}\".format(epoch_test))\n",
    "#\n",
    "for weight in get_weights(\"./model/model_southpark_bk/\"):\n",
    "    print(\"\\n\\nEpoch number: {}\".format(weight))\n",
    "    with model.session() as sess:\n",
    "    #     model.restore_weights(\"./model/model_southpark_bk/\")\n",
    "        latest_checkpoint = \"./model/model_southpark_bk/model.ckpt-\"+str(weight)\n",
    "        saver = tf.train.Saver()\n",
    "\n",
    "        saver.restore(sess, latest_checkpoint)\n",
    "        start_index = random.randint(0, len(text) - section_length - 1)\n",
    "        test_start = text[start_index: start_index + section_length]\n",
    "\n",
    "        X_test = np.zeros((1,section_length,vocab_length))\n",
    "        for i,c in enumerate(test_start):\n",
    "            X_test[0,i,characters2id[c]]=1\n",
    "\n",
    "        for temperature in [0.2]:\n",
    "#             print(\"Temperature = {}\".format(temperature))\n",
    "\n",
    "            print(test_start,end=\"\")\n",
    "            for _ in range(prediction_length): \n",
    "                pred = sess.run(prediction,feed_dict={X:X_test})\n",
    "                pred = pred.reshape(-1)\n",
    "                pred = sample2(pred,temperature)\n",
    "                next_char = id2characters[np.argmax(pred)]\n",
    "    #             print(np.argmax(pred),max(pred))\n",
    "    #             print(pred)\n",
    "    #             break\n",
    "                test_start += next_char\n",
    "                print(next_char,end=\"\")\n",
    "\n",
    "                test_start = test_start[-section_length:]\n",
    "                X_test = np.zeros((1,section_length,vocab_length))\n",
    "                for i,c in enumerate(test_start):\n",
    "                    X_test[0,i,characters2id[c]]=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}