added BiRNN notebook

notebooks/3 - Neural Networks/bidirectional_rnn.ipynb

@@ -0,0 +1,350 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "'''\n",
+    "A Bidirectional Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\n",
+    "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n",
+    "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n",
+    "\n",
+    "Author: Aymeric Damien\n",
+    "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
+      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
+      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
+      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import MINST data\n",
+    "import input_data\n",
+    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)\n",
+    "\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.python.ops.constant_op import constant\n",
+    "from tensorflow.models.rnn import rnn, rnn_cell\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "'''\n",
+    "To classify images using a bidirectional reccurent neural network, we consider every image row as a sequence of pixels.\n",
+    "Because MNIST image shape is 28*28px, we will then handle 28 sequences of 28 steps for every sample.\n",
+    "'''\n",
+    "\n",
+    "# Parameters\n",
+    "learning_rate = 0.001\n",
+    "training_iters = 100000\n",
+    "batch_size = 128\n",
+    "display_step = 10\n",
+    "\n",
+    "# Network Parameters\n",
+    "n_input = 28 # MNIST data input (img shape: 28*28)\n",
+    "n_steps = 28 # timesteps\n",
+    "n_hidden = 128 # hidden layer num of features\n",
+    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
+    "\n",
+    "# tf Graph input\n",
+    "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n",
+    "# Tensorflow LSTM cell requires 2x n_hidden length (state & cell)\n",
+    "istate_fw = tf.placeholder(\"float\", [None, 2*n_hidden])\n",
+    "istate_bw = tf.placeholder(\"float\", [None, 2*n_hidden])\n",
+    "y = tf.placeholder(\"float\", [None, n_classes])\n",
+    "\n",
+    "# Define weights\n",
+    "weights = {\n",
+    "    # Hidden layer weights => 2*n_hidden because of foward + backward cells\n",
+    "    'hidden': tf.Variable(tf.random_normal([n_input, 2*n_hidden])),\n",
+    "    'out': tf.Variable(tf.random_normal([2*n_hidden, n_classes]))\n",
+    "}\n",
+    "biases = {\n",
+    "    'hidden': tf.Variable(tf.random_normal([2*n_hidden])),\n",
+    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def BiRNN(_X, _istate_fw, _istate_bw, _weights, _biases, _batch_size, _seq_len):\n",
+    "\n",
+    "    # BiRNN requires to supply sequence_length as [batch_size, int64]\n",
+    "    # Note: Tensorflow 0.6.0 requires BiRNN sequence_length parameter to be set\n",
+    "    # For a better implementation with latest version of tensorflow, check below\n",
+    "    _seq_len = tf.fill([_batch_size], constant(_seq_len, dtype=tf.int64))\n",
+    "\n",
+    "    # input shape: (batch_size, n_steps, n_input)\n",
+    "    _X = tf.transpose(_X, [1, 0, 2])  # permute n_steps and batch_size\n",
+    "    # Reshape to prepare input to hidden activation\n",
+    "    _X = tf.reshape(_X, [-1, n_input]) # (n_steps*batch_size, n_input)\n",
+    "    # Linear activation\n",
+    "    _X = tf.matmul(_X, _weights['hidden']) + _biases['hidden']\n",
+    "\n",
+    "    # Define lstm cells with tensorflow\n",
+    "    # Forward direction cell\n",
+    "    lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
+    "    # Backward direction cell\n",
+    "    lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
+    "    # Split data because rnn cell needs a list of inputs for the RNN inner loop\n",
+    "    _X = tf.split(0, n_steps, _X) # n_steps * (batch_size, n_hidden)\n",
+    "\n",
+    "    # Get lstm cell output\n",
+    "    outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, _X,\n",
+    "                                            initial_state_fw=_istate_fw,\n",
+    "                                            initial_state_bw=_istate_bw,\n",
+    "                                            sequence_length=_seq_len)\n",
+    "\n",
+    "    # Linear activation\n",
+    "    # Get inner loop last output\n",
+    "    return tf.matmul(outputs[-1], _weights['out']) + _biases['out']\n",
+    "\n",
+    "pred = BiRNN(x, istate_fw, istate_bw, weights, biases, batch_size, n_steps)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# NOTE: The following code is working with current master version of tensorflow\n",
+    "#       BiRNN sequence_length parameter isn't required, so we don't define it\n",
+    "#\n",
+    "# def BiRNN(_X, _istate_fw, _istate_bw, _weights, _biases):\n",
+    "#\n",
+    "#     # input shape: (batch_size, n_steps, n_input)\n",
+    "#     _X = tf.transpose(_X, [1, 0, 2])  # permute n_steps and batch_size\n",
+    "#     # Reshape to prepare input to hidden activation\n",
+    "#     _X = tf.reshape(_X, [-1, n_input]) # (n_steps*batch_size, n_input)\n",
+    "#     # Linear activation\n",
+    "#     _X = tf.matmul(_X, _weights['hidden']) + _biases['hidden']\n",
+    "#\n",
+    "#     # Define lstm cells with tensorflow\n",
+    "#     # Forward direction cell\n",
+    "#     lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
+    "#     # Backward direction cell\n",
+    "#     lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
+    "#     # Split data because rnn cell needs a list of inputs for the RNN inner loop\n",
+    "#     _X = tf.split(0, n_steps, _X) # n_steps * (batch_size, n_hidden)\n",
+    "#\n",
+    "#     # Get lstm cell output\n",
+    "#     outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, _X,\n",
+    "#                                             initial_state_fw=_istate_fw,\n",
+    "#                                             initial_state_bw=_istate_bw)\n",
+    "#\n",
+    "#     # Linear activation\n",
+    "#     # Get inner loop last output\n",
+    "#     return tf.matmul(outputs[-1], _weights['out']) + _biases['out']\n",
+    "#\n",
+    "# pred = BiRNN(x, istate_fw, istate_bw, weights, biases)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Define loss and optimizer\n",
+    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) # Softmax loss\n",
+    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer\n",
+    "\n",
+    "# Evaluate model\n",
+    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
+    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
+    "\n",
+    "# Initializing the variables\n",
+    "init = tf.initialize_all_variables()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Iter 1280, Minibatch Loss= 4.548751, Training Accuracy= 0.25781\n",
+      "Iter 2560, Minibatch Loss= 1.881705, Training Accuracy= 0.36719\n",
+      "Iter 3840, Minibatch Loss= 1.791362, Training Accuracy= 0.34375\n",
+      "Iter 5120, Minibatch Loss= 1.186327, Training Accuracy= 0.63281\n",
+      "Iter 6400, Minibatch Loss= 0.933242, Training Accuracy= 0.66406\n",
+      "Iter 7680, Minibatch Loss= 1.210745, Training Accuracy= 0.59375\n",
+      "Iter 8960, Minibatch Loss= 0.893051, Training Accuracy= 0.63281\n",
+      "Iter 10240, Minibatch Loss= 0.752483, Training Accuracy= 0.77344\n",
+      "Iter 11520, Minibatch Loss= 0.599419, Training Accuracy= 0.77344\n",
+      "Iter 12800, Minibatch Loss= 0.931269, Training Accuracy= 0.67969\n",
+      "Iter 14080, Minibatch Loss= 0.521487, Training Accuracy= 0.82031\n",
+      "Iter 15360, Minibatch Loss= 0.593033, Training Accuracy= 0.78906\n",
+      "Iter 16640, Minibatch Loss= 0.554892, Training Accuracy= 0.78906\n",
+      "Iter 17920, Minibatch Loss= 0.495159, Training Accuracy= 0.86719\n",
+      "Iter 19200, Minibatch Loss= 0.477557, Training Accuracy= 0.82812\n",
+      "Iter 20480, Minibatch Loss= 0.345205, Training Accuracy= 0.89844\n",
+      "Iter 21760, Minibatch Loss= 0.764044, Training Accuracy= 0.76562\n",
+      "Iter 23040, Minibatch Loss= 0.360194, Training Accuracy= 0.86719\n",
+      "Iter 24320, Minibatch Loss= 0.563836, Training Accuracy= 0.79688\n",
+      "Iter 25600, Minibatch Loss= 0.619804, Training Accuracy= 0.78906\n",
+      "Iter 26880, Minibatch Loss= 0.489240, Training Accuracy= 0.81250\n",
+      "Iter 28160, Minibatch Loss= 0.386111, Training Accuracy= 0.89844\n",
+      "Iter 29440, Minibatch Loss= 0.443906, Training Accuracy= 0.88281\n",
+      "Iter 30720, Minibatch Loss= 0.363123, Training Accuracy= 0.86719\n",
+      "Iter 32000, Minibatch Loss= 0.447942, Training Accuracy= 0.85938\n",
+      "Iter 33280, Minibatch Loss= 0.375448, Training Accuracy= 0.88281\n",
+      "Iter 34560, Minibatch Loss= 0.605834, Training Accuracy= 0.81250\n",
+      "Iter 35840, Minibatch Loss= 0.235447, Training Accuracy= 0.90625\n",
+      "Iter 37120, Minibatch Loss= 0.485220, Training Accuracy= 0.86719\n",
+      "Iter 38400, Minibatch Loss= 0.327258, Training Accuracy= 0.92969\n",
+      "Iter 39680, Minibatch Loss= 0.216945, Training Accuracy= 0.91406\n",
+      "Iter 40960, Minibatch Loss= 0.554652, Training Accuracy= 0.82812\n",
+      "Iter 42240, Minibatch Loss= 0.409230, Training Accuracy= 0.87500\n",
+      "Iter 43520, Minibatch Loss= 0.204563, Training Accuracy= 0.92188\n",
+      "Iter 44800, Minibatch Loss= 0.359138, Training Accuracy= 0.87500\n",
+      "Iter 46080, Minibatch Loss= 0.306512, Training Accuracy= 0.89844\n",
+      "Iter 47360, Minibatch Loss= 0.356531, Training Accuracy= 0.86719\n",
+      "Iter 48640, Minibatch Loss= 0.319080, Training Accuracy= 0.87500\n",
+      "Iter 49920, Minibatch Loss= 0.326718, Training Accuracy= 0.89844\n",
+      "Iter 51200, Minibatch Loss= 0.346867, Training Accuracy= 0.88281\n",
+      "Iter 52480, Minibatch Loss= 0.248568, Training Accuracy= 0.92969\n",
+      "Iter 53760, Minibatch Loss= 0.127805, Training Accuracy= 0.94531\n",
+      "Iter 55040, Minibatch Loss= 0.386457, Training Accuracy= 0.88281\n",
+      "Iter 56320, Minibatch Loss= 0.384653, Training Accuracy= 0.84375\n",
+      "Iter 57600, Minibatch Loss= 0.384377, Training Accuracy= 0.85938\n",
+      "Iter 58880, Minibatch Loss= 0.378528, Training Accuracy= 0.83594\n",
+      "Iter 60160, Minibatch Loss= 0.183152, Training Accuracy= 0.94531\n",
+      "Iter 61440, Minibatch Loss= 0.211561, Training Accuracy= 0.92969\n",
+      "Iter 62720, Minibatch Loss= 0.194529, Training Accuracy= 0.94531\n",
+      "Iter 64000, Minibatch Loss= 0.175247, Training Accuracy= 0.93750\n",
+      "Iter 65280, Minibatch Loss= 0.270519, Training Accuracy= 0.89844\n",
+      "Iter 66560, Minibatch Loss= 0.225893, Training Accuracy= 0.94531\n",
+      "Iter 67840, Minibatch Loss= 0.391300, Training Accuracy= 0.91406\n",
+      "Iter 69120, Minibatch Loss= 0.259621, Training Accuracy= 0.87500\n",
+      "Iter 70400, Minibatch Loss= 0.255645, Training Accuracy= 0.92969\n",
+      "Iter 71680, Minibatch Loss= 0.217164, Training Accuracy= 0.91406\n",
+      "Iter 72960, Minibatch Loss= 0.235931, Training Accuracy= 0.92188\n",
+      "Iter 74240, Minibatch Loss= 0.193127, Training Accuracy= 0.92188\n",
+      "Iter 75520, Minibatch Loss= 0.246558, Training Accuracy= 0.92969\n",
+      "Iter 76800, Minibatch Loss= 0.167383, Training Accuracy= 0.92969\n",
+      "Iter 78080, Minibatch Loss= 0.130506, Training Accuracy= 0.96875\n",
+      "Iter 79360, Minibatch Loss= 0.168879, Training Accuracy= 0.96875\n",
+      "Iter 80640, Minibatch Loss= 0.245589, Training Accuracy= 0.93750\n",
+      "Iter 81920, Minibatch Loss= 0.136840, Training Accuracy= 0.94531\n",
+      "Iter 83200, Minibatch Loss= 0.133286, Training Accuracy= 0.96875\n",
+      "Iter 84480, Minibatch Loss= 0.221121, Training Accuracy= 0.95312\n",
+      "Iter 85760, Minibatch Loss= 0.257268, Training Accuracy= 0.91406\n",
+      "Iter 87040, Minibatch Loss= 0.227299, Training Accuracy= 0.92969\n",
+      "Iter 88320, Minibatch Loss= 0.170016, Training Accuracy= 0.96094\n",
+      "Iter 89600, Minibatch Loss= 0.350118, Training Accuracy= 0.89844\n",
+      "Iter 90880, Minibatch Loss= 0.149303, Training Accuracy= 0.95312\n",
+      "Iter 92160, Minibatch Loss= 0.200295, Training Accuracy= 0.94531\n",
+      "Iter 93440, Minibatch Loss= 0.274823, Training Accuracy= 0.89844\n",
+      "Iter 94720, Minibatch Loss= 0.162888, Training Accuracy= 0.96875\n",
+      "Iter 96000, Minibatch Loss= 0.164938, Training Accuracy= 0.93750\n",
+      "Iter 97280, Minibatch Loss= 0.257220, Training Accuracy= 0.92969\n",
+      "Iter 98560, Minibatch Loss= 0.208767, Training Accuracy= 0.92188\n",
+      "Iter 99840, Minibatch Loss= 0.101323, Training Accuracy= 0.97656\n",
+      "Optimization Finished!\n",
+      "Testing Accuracy: 0.945312\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Launch the graph\n",
+    "with tf.Session() as sess:\n",
+    "    sess.run(init)\n",
+    "    step = 1\n",
+    "    # Keep training until reach max iterations\n",
+    "    while step * batch_size < training_iters:\n",
+    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
+    "        # Reshape data to get 28 seq of 28 elements\n",
+    "        batch_xs = batch_xs.reshape((batch_size, n_steps, n_input))\n",
+    "        # Fit training using batch data\n",
+    "        sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys,\n",
+    "                                       istate_fw: np.zeros((batch_size, 2*n_hidden)),\n",
+    "                                       istate_bw: np.zeros((batch_size, 2*n_hidden))})\n",
+    "        if step % display_step == 0:\n",
+    "            # Calculate batch accuracy\n",
+    "            acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys,\n",
+    "                                                istate_fw: np.zeros((batch_size, 2*n_hidden)),\n",
+    "                                                istate_bw: np.zeros((batch_size, 2*n_hidden))})\n",
+    "            # Calculate batch loss\n",
+    "            loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys,\n",
+    "                                             istate_fw: np.zeros((batch_size, 2*n_hidden)),\n",
+    "                                             istate_bw: np.zeros((batch_size, 2*n_hidden))})\n",
+    "            print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \"{:.6f}\".format(loss) + \\\n",
+    "                  \", Training Accuracy= \" + \"{:.5f}\".format(acc)\n",
+    "        step += 1\n",
+    "    print \"Optimization Finished!\"\n",
+    "    # Calculate accuracy for 128 mnist test images\n",
+    "    test_len = 128\n",
+    "    test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n",
+    "    test_label = mnist.test.labels[:test_len]\n",
+    "    print \"Testing Accuracy:\", sess.run(accuracy, feed_dict={x: test_data, y: test_label,\n",
+    "                                                             istate_fw: np.zeros((test_len, 2*n_hidden)),\n",
+    "                                                             istate_bw: np.zeros((test_len, 2*n_hidden))})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "IPython (Python 2.7)",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}