{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "'''\n",
    "A Bidirectional Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\n",
    "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n",
    "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n",
    "\n",
    "Author: Aymeric Damien\n",
    "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "import numpy as np\n",
    "\n",
    "# Import MINST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "'''\n",
    "To classify images using a bidirectional reccurent neural network, we consider\n",
    "every image row as a sequence of pixels. Because MNIST image shape is 28*28px,\n",
    "we will then handle 28 sequences of 28 steps for every sample.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 100000\n",
    "batch_size = 128\n",
    "display_step = 10\n",
    "\n",
    "# Network Parameters\n",
    "n_input = 28 # MNIST data input (img shape: 28*28)\n",
    "n_steps = 28 # timesteps\n",
    "n_hidden = 128 # hidden layer num of features\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "# Define weights\n",
    "weights = {\n",
    "    # Hidden layer weights => 2*n_hidden because of foward + backward cells\n",
    "    'out': tf.Variable(tf.random_normal([2*n_hidden, n_classes]))\n",
    "}\n",
    "biases = {\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def BiRNN(x, weights, biases):\n",
    "\n",
    "    # Prepare data shape to match `bidirectional_rnn` function requirements\n",
    "    # Current data input shape: (batch_size, n_steps, n_input)\n",
    "    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)\n",
    "    \n",
    "    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)\n",
    "    x = tf.unstack(x, n_steps, 1)\n",
    "\n",
    "    # Define lstm cells with tensorflow\n",
    "    # Forward direction cell\n",
    "    lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
    "    # Backward direction cell\n",
    "    lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
    "\n",
    "    # Get lstm cell output\n",
    "    try:\n",
    "        outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n",
    "                                              dtype=tf.float32)\n",
    "    except Exception: # Old TensorFlow version only returns outputs not states\n",
    "        outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n",
    "                                        dtype=tf.float32)\n",
    "\n",
    "    # Linear activation, using rnn inner loop last output\n",
    "    return tf.matmul(outputs[-1], weights['out']) + biases['out']\n",
    "\n",
    "pred = BiRNN(x, weights, biases)\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 1280, Minibatch Loss= 1.557283, Training Accuracy= 0.49219\n",
      "Iter 2560, Minibatch Loss= 1.358445, Training Accuracy= 0.56250\n",
      "Iter 3840, Minibatch Loss= 1.043732, Training Accuracy= 0.64062\n",
      "Iter 5120, Minibatch Loss= 0.796770, Training Accuracy= 0.72656\n",
      "Iter 6400, Minibatch Loss= 0.626206, Training Accuracy= 0.72656\n",
      "Iter 7680, Minibatch Loss= 1.025919, Training Accuracy= 0.65625\n",
      "Iter 8960, Minibatch Loss= 0.744850, Training Accuracy= 0.76562\n",
      "Iter 10240, Minibatch Loss= 0.530111, Training Accuracy= 0.84375\n",
      "Iter 11520, Minibatch Loss= 0.383806, Training Accuracy= 0.86719\n",
      "Iter 12800, Minibatch Loss= 0.607816, Training Accuracy= 0.82812\n",
      "Iter 14080, Minibatch Loss= 0.410879, Training Accuracy= 0.89062\n",
      "Iter 15360, Minibatch Loss= 0.335351, Training Accuracy= 0.89844\n",
      "Iter 16640, Minibatch Loss= 0.428004, Training Accuracy= 0.91406\n",
      "Iter 17920, Minibatch Loss= 0.307468, Training Accuracy= 0.91406\n",
      "Iter 19200, Minibatch Loss= 0.249527, Training Accuracy= 0.92188\n",
      "Iter 20480, Minibatch Loss= 0.148163, Training Accuracy= 0.96094\n",
      "Iter 21760, Minibatch Loss= 0.445275, Training Accuracy= 0.83594\n",
      "Iter 23040, Minibatch Loss= 0.173083, Training Accuracy= 0.93750\n",
      "Iter 24320, Minibatch Loss= 0.373696, Training Accuracy= 0.87500\n",
      "Iter 25600, Minibatch Loss= 0.509869, Training Accuracy= 0.85938\n",
      "Iter 26880, Minibatch Loss= 0.198096, Training Accuracy= 0.92969\n",
      "Iter 28160, Minibatch Loss= 0.228221, Training Accuracy= 0.92188\n",
      "Iter 29440, Minibatch Loss= 0.280088, Training Accuracy= 0.89844\n",
      "Iter 30720, Minibatch Loss= 0.300495, Training Accuracy= 0.91406\n",
      "Iter 32000, Minibatch Loss= 0.171746, Training Accuracy= 0.95312\n",
      "Iter 33280, Minibatch Loss= 0.263745, Training Accuracy= 0.89844\n",
      "Iter 34560, Minibatch Loss= 0.177300, Training Accuracy= 0.93750\n",
      "Iter 35840, Minibatch Loss= 0.160621, Training Accuracy= 0.95312\n",
      "Iter 37120, Minibatch Loss= 0.321745, Training Accuracy= 0.91406\n",
      "Iter 38400, Minibatch Loss= 0.188322, Training Accuracy= 0.93750\n",
      "Iter 39680, Minibatch Loss= 0.104025, Training Accuracy= 0.96875\n",
      "Iter 40960, Minibatch Loss= 0.291053, Training Accuracy= 0.89062\n",
      "Iter 42240, Minibatch Loss= 0.131189, Training Accuracy= 0.95312\n",
      "Iter 43520, Minibatch Loss= 0.154949, Training Accuracy= 0.92969\n",
      "Iter 44800, Minibatch Loss= 0.150411, Training Accuracy= 0.93750\n",
      "Iter 46080, Minibatch Loss= 0.117008, Training Accuracy= 0.96094\n",
      "Iter 47360, Minibatch Loss= 0.181344, Training Accuracy= 0.96094\n",
      "Iter 48640, Minibatch Loss= 0.209197, Training Accuracy= 0.94531\n",
      "Iter 49920, Minibatch Loss= 0.159350, Training Accuracy= 0.96094\n",
      "Iter 51200, Minibatch Loss= 0.124001, Training Accuracy= 0.95312\n",
      "Iter 52480, Minibatch Loss= 0.165183, Training Accuracy= 0.94531\n",
      "Iter 53760, Minibatch Loss= 0.046438, Training Accuracy= 0.97656\n",
      "Iter 55040, Minibatch Loss= 0.199995, Training Accuracy= 0.91406\n",
      "Iter 56320, Minibatch Loss= 0.057071, Training Accuracy= 0.97656\n",
      "Iter 57600, Minibatch Loss= 0.177065, Training Accuracy= 0.92188\n",
      "Iter 58880, Minibatch Loss= 0.091666, Training Accuracy= 0.96094\n",
      "Iter 60160, Minibatch Loss= 0.069232, Training Accuracy= 0.96875\n",
      "Iter 61440, Minibatch Loss= 0.127353, Training Accuracy= 0.94531\n",
      "Iter 62720, Minibatch Loss= 0.095795, Training Accuracy= 0.96094\n",
      "Iter 64000, Minibatch Loss= 0.202651, Training Accuracy= 0.96875\n",
      "Iter 65280, Minibatch Loss= 0.118779, Training Accuracy= 0.95312\n",
      "Iter 66560, Minibatch Loss= 0.043173, Training Accuracy= 0.98438\n",
      "Iter 67840, Minibatch Loss= 0.152280, Training Accuracy= 0.95312\n",
      "Iter 69120, Minibatch Loss= 0.085301, Training Accuracy= 0.96875\n",
      "Iter 70400, Minibatch Loss= 0.093421, Training Accuracy= 0.96094\n",
      "Iter 71680, Minibatch Loss= 0.096358, Training Accuracy= 0.96875\n",
      "Iter 72960, Minibatch Loss= 0.053386, Training Accuracy= 0.98438\n",
      "Iter 74240, Minibatch Loss= 0.065237, Training Accuracy= 0.97656\n",
      "Iter 75520, Minibatch Loss= 0.228090, Training Accuracy= 0.92188\n",
      "Iter 76800, Minibatch Loss= 0.106751, Training Accuracy= 0.95312\n",
      "Iter 78080, Minibatch Loss= 0.187795, Training Accuracy= 0.94531\n",
      "Iter 79360, Minibatch Loss= 0.092611, Training Accuracy= 0.96094\n",
      "Iter 80640, Minibatch Loss= 0.137386, Training Accuracy= 0.96875\n",
      "Iter 81920, Minibatch Loss= 0.106634, Training Accuracy= 0.98438\n",
      "Iter 83200, Minibatch Loss= 0.111749, Training Accuracy= 0.94531\n",
      "Iter 84480, Minibatch Loss= 0.191184, Training Accuracy= 0.94531\n",
      "Iter 85760, Minibatch Loss= 0.063982, Training Accuracy= 0.96094\n",
      "Iter 87040, Minibatch Loss= 0.092380, Training Accuracy= 0.96875\n",
      "Iter 88320, Minibatch Loss= 0.089899, Training Accuracy= 0.97656\n",
      "Iter 89600, Minibatch Loss= 0.141107, Training Accuracy= 0.94531\n",
      "Iter 90880, Minibatch Loss= 0.075549, Training Accuracy= 0.96094\n",
      "Iter 92160, Minibatch Loss= 0.186539, Training Accuracy= 0.94531\n",
      "Iter 93440, Minibatch Loss= 0.079639, Training Accuracy= 0.97656\n",
      "Iter 94720, Minibatch Loss= 0.156895, Training Accuracy= 0.95312\n",
      "Iter 96000, Minibatch Loss= 0.088042, Training Accuracy= 0.97656\n",
      "Iter 97280, Minibatch Loss= 0.076670, Training Accuracy= 0.96875\n",
      "Iter 98560, Minibatch Loss= 0.051336, Training Accuracy= 0.97656\n",
      "Iter 99840, Minibatch Loss= 0.086923, Training Accuracy= 0.98438\n",
      "Optimization Finished!\n",
      "Testing Accuracy: 0.960938\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
    "        # Reshape data to get 28 seq of 28 elements\n",
    "        batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n",
    "        # Run optimization op (backprop)\n",
    "        sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n",
    "        if step % display_step == 0:\n",
    "            # Calculate batch accuracy\n",
    "            acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n",
    "            # Calculate batch loss\n",
    "            loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n",
    "            print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.5f}\".format(acc)\n",
    "        step += 1\n",
    "    print \"Optimization Finished!\"\n",
    "\n",
    "    # Calculate accuracy for 128 mnist test images\n",
    "    test_len = 128\n",
    "    test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n",
    "    test_label = mnist.test.labels[:test_len]\n",
    "    print \"Testing Accuracy:\", \\\n",
    "        sess.run(accuracy, feed_dict={x: test_data, y: test_label})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}