notebooks/char_rnn_sample_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages Imported\n"
     ]
    }
   ],
   "source": [
    "# Import Packages\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.models.rnn import rnn_cell\n",
    "from tensorflow.models.rnn import seq2seq\n",
    "\n",
    "import collections\n",
    "import argparse\n",
    "import time\n",
    "import os\n",
    "from six.moves import cPickle\n",
    "print (\"Packages Imported\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'vocab_size' is 99\n"
     ]
    }
   ],
   "source": [
    "# Load chars and vocab\n",
    "load_dir = \"/tmp/tf_logs/char_rnn_tutorial\"\n",
    "with open(os.path.join(load_dir, 'chars_vocab.pkl'), 'rb') as f:\n",
    "    chars, vocab = cPickle.load(f)\n",
    "vocab_size = len(vocab) \n",
    "print (\"'vocab_size' is %d\" % (vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now, we are ready to make our RNN model with seq2seq\n",
    "### This network is for sampling, so we don't need batches for sequenes nor optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network Ready\n"
     ]
    }
   ],
   "source": [
    "# Important RNN parameters \n",
    "rnn_size   = 128\n",
    "num_layers = 2\n",
    "\n",
    "batch_size = 1 # <= In the training phase, these were both 50\n",
    "seq_length = 1\n",
    "\n",
    "# Construct RNN model \n",
    "unitcell   = rnn_cell.BasicLSTMCell(rnn_size)\n",
    "cell       = rnn_cell.MultiRNNCell([unitcell] * num_layers)\n",
    "input_data = tf.placeholder(tf.int32, [batch_size, seq_length])\n",
    "istate     = cell.zero_state(batch_size, tf.float32)\n",
    "# Weigths \n",
    "with tf.variable_scope('rnnlm'):\n",
    "    softmax_w = tf.get_variable(\"softmax_w\", [rnn_size, vocab_size])\n",
    "    softmax_b = tf.get_variable(\"softmax_b\", [vocab_size])\n",
    "    with tf.device(\"/cpu:0\"):\n",
    "        embedding = tf.get_variable(\"embedding\", [vocab_size, rnn_size])\n",
    "        inputs = tf.split(1, seq_length, tf.nn.embedding_lookup(embedding, input_data))\n",
    "        inputs = [tf.squeeze(_input, [1]) for _input in inputs]\n",
    "# Output\n",
    "def loop(prev, _):\n",
    "    prev = tf.nn.xw_plus_b(prev, softmax_w, softmax_b)\n",
    "    prev_symbol = tf.stop_gradient(tf.argmax(prev, 1))\n",
    "    return tf.nn.embedding_lookup(embedding, prev_symbol)\n",
    "outputs, final_state = seq2seq.rnn_decoder(inputs, istate, cell\n",
    "                                          , loop_function=None, scope='rnnlm')\n",
    "output = tf.reshape(tf.concat(1, outputs), [-1, rnn_size])\n",
    "\n",
    "logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)\n",
    "probs  = tf.nn.softmax(logits)\n",
    "\n",
    "print (\"Network Ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/tf_logs/char_rnn_tutorial/model.ckpt-8000\n"
     ]
    }
   ],
   "source": [
    "# Restore RNN\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "saver = tf.train.Saver(tf.all_variables())\n",
    "ckpt  = tf.train.get_checkpoint_state(load_dir)\n",
    "\n",
    "print (ckpt.model_checkpoint_path)\n",
    "saver.restore(sess, ckpt.model_checkpoint_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finally, show what RNN has generated! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampling Done. \n",
      "___________________________________________\n",
      "\n",
      "/* *syscblanim types when the formathing)\n",
      "\n",
      " */\n",
      "\n",
      "void console_dir(tmp_cgroup_shorts);\n",
      "\n",
      "/*\n",
      " * This already (proces6or\n",
      " * @pid_struct(__kthread_lock\n",
      "\tunlatelf if (subskick_map shoulds (the softwach then as up is\n",
      "   boot fields up posix are HRPS5/\n",
      "\n",
      "int set_futex_intend_head_unintvel_ap,\n",
      "\t.max_acquire(void, test_cpu(&rq->load, sizeof(struct file *g),\n",
      "\t\t(singlec_ns);\n",
      "\n",
      "int EAVECLERPE\n",
      "}6\n",
      "\trlim_polic_entires;\n",
      "}\n",
      "\n",
      "static void audit_unlock_irq(struct helds, void cpu_stattr_namespace(__kprobess) ISusyoffreings void)\n",
      "{\n",
      "\tif (!res->panicalings)\n",
      "\t\tgoto root = ATOST_MEM:\n",
      "\t\tp >= d = __delayed_poffset;\n",
      "\t\t*strt' = delta_event_fs_allowed;\n",
      "\t\tBUG_ON(!oops->list, new_audit_lock);\n",
      "\t}\n",
      "\n",
      "\tfreepstart;\n",
      "\tlong d;\n",
      "\n",
      "\tatomic_expid(table, flags);\n",
      "\tsubbufs_pid_t now;\n",
      "\tidle = &pi_state->flags; maxRetect_call(aggm_natch);\n",
      "\n",
      "\treturn printk(*fa->tv_softirqs_entry) {\n",
      "\t\taddress,\n",
      "\t\t.cquient(i)\n",
      "\t\tgoto ops_type = seq_rese,\n",
      "};\n",
      "\n",
      "static struct head_stats {\n",
      "\tstruct trans_ctly *next;\n",
      "\n",
      "\t/* Ax or with unbless and just even notifier restart= r\n"
     ]
    }
   ],
   "source": [
    "# Sampling function\n",
    "def weighted_pick(weights):\n",
    "    t = np.cumsum(weights)\n",
    "    s = np.sum(weights)\n",
    "    return(int(np.searchsorted(t, np.random.rand(1)*s)))\n",
    "\n",
    "# Sample using RNN and prime characters\n",
    "prime = \"/* \"\n",
    "state = sess.run(cell.zero_state(1, tf.float32))\n",
    "for char in prime[:-1]:\n",
    "    x = np.zeros((1, 1))\n",
    "    x[0, 0] = vocab[char]\n",
    "    state = sess.run(final_state, feed_dict={input_data: x, istate:state})\n",
    "\n",
    "# Sample 'num' characters\n",
    "ret  = prime\n",
    "char = prime[-1] # <= This goes IN! \n",
    "num  = 1000\n",
    "for n in range(num):\n",
    "    x = np.zeros((1, 1))\n",
    "    x[0, 0] = vocab[char]\n",
    "    [probsval, state] = sess.run([probs, final_state]\n",
    "        , feed_dict={input_data: x, istate:state})\n",
    "    p      = probsval[0] \n",
    "    \n",
    "    sample = weighted_pick(p)\n",
    "    # sample = np.argmax(p)\n",
    "    \n",
    "    pred   = chars[sample]\n",
    "    ret    = ret + pred\n",
    "    char   = pred\n",
    "    \n",
    "print (\"Sampling Done. \\n___________________________________________\\n\")\n",
    "\n",
    "print (ret)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hope, it was good. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}