add eager API notebooks

examples/2_BasicModels/linear_regression_eager_api.py

@@ -1,7 +1,6 @@
-'''
-A logistic regression learning algorithm example using TensorFlow library.
-This example is using the MNIST database of handwritten digits
-(http://yann.lecun.com/exdb/mnist/)
+''' Linear Regression with Eager API.
+
+A linear regression learning algorithm example using TensorFlow's Eager API.
 
 Author: Aymeric Damien
 Project: https://github.com/aymericdamien/TensorFlow-Examples/

examples/2_BasicModels/logistic_regression_eager_api.py

@@ -1,5 +1,6 @@
-'''
-A logistic regression learning algorithm example using TensorFlow library.
+''' Logistic Regression with Eager API.
+
+A logistic regression learning algorithm example using TensorFlow's Eager API.
 This example is using the MNIST database of handwritten digits
 (http://yann.lecun.com/exdb/mnist/)
 

examples/3_NeuralNetworks/neural_network_eager_api.py

@@ -1,7 +1,7 @@
-""" Neural Network.
+""" Neural Network with Eager API.
 
 A 2-Hidden Layers Fully Connected Neural Network (a.k.a Multilayer Perceptron)
-implementation with TensorFlow. This example is using the MNIST database
+implementation with TensorFlow's Eager API. This example is using the MNIST database
 of handwritten digits (http://yann.lecun.com/exdb/mnist/).
 
 This example is using TensorFlow layers, see 'neural_network_raw' example for

notebooks/1_Introduction/basic_eager_api.ipynb

@@ -0,0 +1,238 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Basic introduction to TensorFlow's Eager API\n",
+    "\n",
+    "A simple introduction to get started with TensorFlow's Eager API.\n",
+    "\n",
+    "- Author: Aymeric Damien\n",
+    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### What is TensorFlow's Eager API ?\n",
+    "\n",
+    "*Eager execution is an imperative, define-by-run interface where operations are\n",
+    "executed immediately as they are called from Python. This makes it easier to\n",
+    "get started with TensorFlow, and can make research and development more\n",
+    "intuitive. A vast majority of the TensorFlow API remains the same whether eager\n",
+    "execution is enabled or not. As a result, the exact same code that constructs\n",
+    "TensorFlow graphs (e.g. using the layers API) can be executed imperatively\n",
+    "by using eager execution. Conversely, most models written with Eager enabled\n",
+    "can be converted to a graph that can be further optimized and/or extracted\n",
+    "for deployment in production without changing code. - Rajat Monga*\n",
+    "\n",
+    "More info: https://research.googleblog.com/2017/10/eager-execution-imperative-define-by.html"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from __future__ import absolute_import, division, print_function\n",
+    "\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "import tensorflow.contrib.eager as tfe"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Setting Eager mode...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Set Eager API\n",
+    "print(\"Setting Eager mode...\")\n",
+    "tfe.enable_eager_execution()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Define constant tensors\n",
+      "a = 2\n",
+      "b = 3\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define constant tensors\n",
+    "print(\"Define constant tensors\")\n",
+    "a = tf.constant(2)\n",
+    "print(\"a = %i\" % a)\n",
+    "b = tf.constant(3)\n",
+    "print(\"b = %i\" % b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running operations, without tf.Session\n",
+      "a + b = 5\n",
+      "a * b = 6\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run the operation without the need for tf.Session\n",
+    "print(\"Running operations, without tf.Session\")\n",
+    "c = a + b\n",
+    "print(\"a + b = %i\" % c)\n",
+    "d = a * b\n",
+    "print(\"a * b = %i\" % d)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Mixing operations with Tensors and Numpy Arrays\n",
+      "Tensor:\n",
+      " a = tf.Tensor(\n",
+      "[[2. 1.]\n",
+      " [1. 0.]], shape=(2, 2), dtype=float32)\n",
+      "NumpyArray:\n",
+      " b = [[3. 0.]\n",
+      " [5. 1.]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Full compatibility with Numpy\n",
+    "print(\"Mixing operations with Tensors and Numpy Arrays\")\n",
+    "\n",
+    "# Define constant tensors\n",
+    "a = tf.constant([[2., 1.],\n",
+    "                 [1., 0.]], dtype=tf.float32)\n",
+    "print(\"Tensor:\\n a = %s\" % a)\n",
+    "b = np.array([[3., 0.],\n",
+    "              [5., 1.]], dtype=np.float32)\n",
+    "print(\"NumpyArray:\\n b = %s\" % b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running operations, without tf.Session\n",
+      "a + b = tf.Tensor(\n",
+      "[[5. 1.]\n",
+      " [6. 1.]], shape=(2, 2), dtype=float32)\n",
+      "a * b = tf.Tensor(\n",
+      "[[11.  1.]\n",
+      " [ 3.  0.]], shape=(2, 2), dtype=float32)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run the operation without the need for tf.Session\n",
+    "print(\"Running operations, without tf.Session\")\n",
+    "\n",
+    "c = a + b\n",
+    "print(\"a + b = %s\" % c)\n",
+    "\n",
+    "d = tf.matmul(a, b)\n",
+    "print(\"a * b = %s\" % d)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Iterate through Tensor 'a':\n",
+      "tf.Tensor(2.0, shape=(), dtype=float32)\n",
+      "tf.Tensor(1.0, shape=(), dtype=float32)\n",
+      "tf.Tensor(1.0, shape=(), dtype=float32)\n",
+      "tf.Tensor(0.0, shape=(), dtype=float32)\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Iterate through Tensor 'a':\")\n",
+    "for i in range(a.shape[0]):\n",
+    "    for j in range(a.shape[1]):\n",
+    "        print(a[i][j])"
+   ]
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python [default]",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

notebooks/2_BasicModels/logistic_regression.ipynb

@@ -9,13 +9,25 @@
     "# Logistic Regression Example\n",
     "\n",
     "A logistic regression learning algorithm example using TensorFlow library.\n",
-    "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n",
     "\n",
     "- Author: Aymeric Damien\n",
     "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
    ]
   },
   {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## MNIST Dataset Overview\n",
+    "\n",
+    "This example is using MNIST handwritten digits. The dataset contains 60,000 examples for training and 10,000 examples for testing. The digits have been size-normalized and centered in a fixed-size image (28x28 pixels) with values from 0 to 1. For simplicity, each image has been flattened and converted to a 1-D numpy array of 784 features (28*28).\n",
+    "\n",
+    "![MNIST Dataset](http://neuralnetworksanddeeplearning.com/images/mnist_100_digits.png)\n",
+    "\n",
+    "More info: http://yann.lecun.com/exdb/mnist/"
+   ]
+  },
+  {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {

notebooks/2_BasicModels/logistic_regression_eager_api.ipynb

@@ -0,0 +1,268 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Logistic Regression with Eager API\n",
+    "\n",
+    "A logistic regression implemented using TensorFlow's Eager API.\n",
+    "\n",
+    "- Author: Aymeric Damien\n",
+    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## MNIST Dataset Overview\n",
+    "\n",
+    "This example is using MNIST handwritten digits. The dataset contains 60,000 examples for training and 10,000 examples for testing. The digits have been size-normalized and centered in a fixed-size image (28x28 pixels) with values from 0 to 1. For simplicity, each image has been flattened and converted to a 1-D numpy array of 784 features (28*28).\n",
+    "\n",
+    "![MNIST Dataset](http://neuralnetworksanddeeplearning.com/images/mnist_100_digits.png)\n",
+    "\n",
+    "More info: http://yann.lecun.com/exdb/mnist/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from __future__ import absolute_import, division, print_function\n",
+    "\n",
+    "import tensorflow as tf\n",
+    "import tensorflow.contrib.eager as tfe"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Set Eager API\n",
+    "tfe.enable_eager_execution()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
+      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
+      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
+      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import MNIST data\n",
+    "from tensorflow.examples.tutorials.mnist import input_data\n",
+    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Parameters\n",
+    "learning_rate = 0.1\n",
+    "batch_size = 128\n",
+    "num_steps = 1000\n",
+    "display_step = 100"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Iterator for the dataset\n",
+    "dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "    (mnist.train.images, mnist.train.labels)).batch(batch_size)\n",
+    "dataset_iter = tfe.Iterator(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Variables\n",
+    "W = tfe.Variable(tf.zeros([784, 10]), name='weights')\n",
+    "b = tfe.Variable(tf.zeros([10]), name='bias')\n",
+    "\n",
+    "# Logistic regression (Wx + b)\n",
+    "def logistic_regression(inputs):\n",
+    "    return tf.matmul(inputs, W) + b\n",
+    "\n",
+    "# Cross-Entropy loss function\n",
+    "def loss_fn(inference_fn, inputs, labels):\n",
+    "    # Using sparse_softmax cross entropy\n",
+    "    return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
+    "        logits=inference_fn(inputs), labels=labels))\n",
+    "\n",
+    "# Calculate accuracy\n",
+    "def accuracy_fn(inference_fn, inputs, labels):\n",
+    "    prediction = tf.nn.softmax(inference_fn(inputs))\n",
+    "    correct_pred = tf.equal(tf.argmax(prediction, 1), labels)\n",
+    "    return tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# SGD Optimizer\n",
+    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
+    "\n",
+    "# Compute gradients\n",
+    "grad = tfe.implicit_gradients(loss_fn)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Initial loss= 2.302584887\n",
+      "Step: 0001  loss= 2.302584887  accuracy= 0.1172\n",
+      "Step: 0100  loss= 0.952338457  accuracy= 0.7955\n",
+      "Step: 0200  loss= 0.535867393  accuracy= 0.8712\n",
+      "Step: 0300  loss= 0.485415280  accuracy= 0.8757\n",
+      "Step: 0400  loss= 0.433947206  accuracy= 0.8843\n",
+      "Step: 0500  loss= 0.381990731  accuracy= 0.8971\n",
+      "Step: 0600  loss= 0.394154936  accuracy= 0.8947\n",
+      "Step: 0700  loss= 0.391497582  accuracy= 0.8905\n",
+      "Step: 0800  loss= 0.386373103  accuracy= 0.8945\n",
+      "Step: 0900  loss= 0.332039326  accuracy= 0.9096\n",
+      "Step: 1000  loss= 0.358993769  accuracy= 0.9002\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Training\n",
+    "average_loss = 0.\n",
+    "average_acc = 0.\n",
+    "for step in range(num_steps):\n",
+    "\n",
+    "    # Iterate through the dataset\n",
+    "    try:\n",
+    "        d = dataset_iter.next()\n",
+    "    except StopIteration:\n",
+    "        # Refill queue\n",
+    "        dataset_iter = tfe.Iterator(dataset)\n",
+    "        d = dataset_iter.next()\n",
+    "\n",
+    "    # Images\n",
+    "    x_batch = d[0]\n",
+    "    # Labels\n",
+    "    y_batch = tf.cast(d[1], dtype=tf.int64)\n",
+    "\n",
+    "    # Compute the batch loss\n",
+    "    batch_loss = loss_fn(logistic_regression, x_batch, y_batch)\n",
+    "    average_loss += batch_loss\n",
+    "    # Compute the batch accuracy\n",
+    "    batch_accuracy = accuracy_fn(logistic_regression, x_batch, y_batch)\n",
+    "    average_acc += batch_accuracy\n",
+    "\n",
+    "    if step == 0:\n",
+    "        # Display the initial cost, before optimizing\n",
+    "        print(\"Initial loss= {:.9f}\".format(average_loss))\n",
+    "\n",
+    "    # Update the variables following gradients info\n",
+    "    optimizer.apply_gradients(grad(logistic_regression, x_batch, y_batch))\n",
+    "\n",
+    "    # Display info\n",
+    "    if (step + 1) % display_step == 0 or step == 0:\n",
+    "        if step > 0:\n",
+    "            average_loss /= display_step\n",
+    "            average_acc /= display_step\n",
+    "        print(\"Step:\", '%04d' % (step + 1), \" loss=\",\n",
+    "              \"{:.9f}\".format(average_loss), \" accuracy=\",\n",
+    "              \"{:.4f}\".format(average_acc))\n",
+    "        average_loss = 0.\n",
+    "        average_acc = 0."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Testset Accuracy: 0.9083\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Evaluate model on the test image set\n",
+    "testX = mnist.test.images\n",
+    "testY = mnist.test.labels\n",
+    "\n",
+    "test_acc = accuracy_fn(logistic_regression, testX, testY)\n",
+    "print(\"Testset Accuracy: {:.4f}\".format(test_acc))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [default]",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

notebooks/3_NeuralNetworks/neural_network_eager_api.ipynb

@@ -0,0 +1,297 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Neural Network with Eager API\n",
+    "\n",
+    "Build a 2-hidden layers fully connected neural network (a.k.a multilayer perceptron) with TensorFlow's Eager API.\n",
+    "\n",
+    "This example is using some of TensorFlow higher-level wrappers (tf.estimators, tf.layers, tf.metrics, ...), you can check 'neural_network_raw' example for a raw, and more detailed TensorFlow implementation.\n",
+    "\n",
+    "- Author: Aymeric Damien\n",
+    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Neural Network Overview\n",
+    "\n",
+    "<img src=\"http://cs231n.github.io/assets/nn1/neural_net2.jpeg\" alt=\"nn\" style=\"width: 400px;\"/>\n",
+    "\n",
+    "## MNIST Dataset Overview\n",
+    "\n",
+    "This example is using MNIST handwritten digits. The dataset contains 60,000 examples for training and 10,000 examples for testing. The digits have been size-normalized and centered in a fixed-size image (28x28 pixels) with values from 0 to 1. For simplicity, each image has been flattened and converted to a 1-D numpy array of 784 features (28*28).\n",
+    "\n",
+    "![MNIST Dataset](http://neuralnetworksanddeeplearning.com/images/mnist_100_digits.png)\n",
+    "\n",
+    "More info: http://yann.lecun.com/exdb/mnist/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from __future__ import print_function\n",
+    "\n",
+    "import tensorflow as tf\n",
+    "import tensorflow.contrib.eager as tfe"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Set Eager API\n",
+    "tfe.enable_eager_execution()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
+      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
+      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
+      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import MNIST data\n",
+    "from tensorflow.examples.tutorials.mnist import input_data\n",
+    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Parameters\n",
+    "learning_rate = 0.001\n",
+    "num_steps = 1000\n",
+    "batch_size = 128\n",
+    "display_step = 100\n",
+    "\n",
+    "# Network Parameters\n",
+    "n_hidden_1 = 256 # 1st layer number of neurons\n",
+    "n_hidden_2 = 256 # 2nd layer number of neurons\n",
+    "num_input = 784 # MNIST data input (img shape: 28*28)\n",
+    "num_classes = 10 # MNIST total classes (0-9 digits)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Using TF Dataset to split data into batches\n",
+    "dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "    (mnist.train.images, mnist.train.labels)).batch(batch_size)\n",
+    "dataset_iter = tfe.Iterator(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Define the neural network. To use eager API and tf.layers API together,\n",
+    "# we must instantiate a tfe.Network class as follow:\n",
+    "class NeuralNet(tfe.Network):\n",
+    "    def __init__(self):\n",
+    "        # Define each layer\n",
+    "        super(NeuralNet, self).__init__()\n",
+    "        # Hidden fully connected layer with 256 neurons\n",
+    "        self.layer1 = self.track_layer(\n",
+    "            tf.layers.Dense(n_hidden_1, activation=tf.nn.relu))\n",
+    "        # Hidden fully connected layer with 256 neurons\n",
+    "        self.layer2 = self.track_layer(\n",
+    "            tf.layers.Dense(n_hidden_2, activation=tf.nn.relu))\n",
+    "        # Output fully connected layer with a neuron for each class\n",
+    "        self.out_layer = self.track_layer(tf.layers.Dense(num_classes))\n",
+    "\n",
+    "    def call(self, x):\n",
+    "        x = self.layer1(x)\n",
+    "        x = self.layer2(x)\n",
+    "        return self.out_layer(x)\n",
+    "\n",
+    "\n",
+    "neural_net = NeuralNet()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Cross-Entropy loss function\n",
+    "def loss_fn(inference_fn, inputs, labels):\n",
+    "    # Using sparse_softmax cross entropy\n",
+    "    return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
+    "        logits=inference_fn(inputs), labels=labels))\n",
+    "\n",
+    "\n",
+    "# Calculate accuracy\n",
+    "def accuracy_fn(inference_fn, inputs, labels):\n",
+    "    prediction = tf.nn.softmax(inference_fn(inputs))\n",
+    "    correct_pred = tf.equal(tf.argmax(prediction, 1), labels)\n",
+    "    return tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
+    "\n",
+    "\n",
+    "# SGD Optimizer\n",
+    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
+    "\n",
+    "# Compute gradients\n",
+    "grad = tfe.implicit_gradients(loss_fn)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Initial loss= 2.340397596\n",
+      "Step: 0001  loss= 2.340397596  accuracy= 0.0703\n",
+      "Step: 0100  loss= 0.586046159  accuracy= 0.8305\n",
+      "Step: 0200  loss= 0.253318846  accuracy= 0.9282\n",
+      "Step: 0300  loss= 0.214748293  accuracy= 0.9377\n",
+      "Step: 0400  loss= 0.180644721  accuracy= 0.9466\n",
+      "Step: 0500  loss= 0.137285724  accuracy= 0.9591\n",
+      "Step: 0600  loss= 0.119845696  accuracy= 0.9636\n",
+      "Step: 0700  loss= 0.113618039  accuracy= 0.9665\n",
+      "Step: 0800  loss= 0.109642141  accuracy= 0.9676\n",
+      "Step: 0900  loss= 0.085067607  accuracy= 0.9746\n",
+      "Step: 1000  loss= 0.079819344  accuracy= 0.9754\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Training\n",
+    "average_loss = 0.\n",
+    "average_acc = 0.\n",
+    "for step in range(num_steps):\n",
+    "\n",
+    "    # Iterate through the dataset\n",
+    "    try:\n",
+    "        d = dataset_iter.next()\n",
+    "    except StopIteration:\n",
+    "        # Refill queue\n",
+    "        dataset_iter = tfe.Iterator(dataset)\n",
+    "        d = dataset_iter.next()\n",
+    "\n",
+    "    # Images\n",
+    "    x_batch = d[0]\n",
+    "    # Labels\n",
+    "    y_batch = tf.cast(d[1], dtype=tf.int64)\n",
+    "\n",
+    "    # Compute the batch loss\n",
+    "    batch_loss = loss_fn(neural_net, x_batch, y_batch)\n",
+    "    average_loss += batch_loss\n",
+    "    # Compute the batch accuracy\n",
+    "    batch_accuracy = accuracy_fn(neural_net, x_batch, y_batch)\n",
+    "    average_acc += batch_accuracy\n",
+    "\n",
+    "    if step == 0:\n",
+    "        # Display the initial cost, before optimizing\n",
+    "        print(\"Initial loss= {:.9f}\".format(average_loss))\n",
+    "\n",
+    "    # Update the variables following gradients info\n",
+    "    optimizer.apply_gradients(grad(neural_net, x_batch, y_batch))\n",
+    "\n",
+    "    # Display info\n",
+    "    if (step + 1) % display_step == 0 or step == 0:\n",
+    "        if step > 0:\n",
+    "            average_loss /= display_step\n",
+    "            average_acc /= display_step\n",
+    "        print(\"Step:\", '%04d' % (step + 1), \" loss=\",\n",
+    "              \"{:.9f}\".format(average_loss), \" accuracy=\",\n",
+    "              \"{:.4f}\".format(average_acc))\n",
+    "        average_loss = 0.\n",
+    "        average_acc = 0."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Testset Accuracy: 0.9719\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Evaluate model on the test image set\n",
+    "testX = mnist.test.images\n",
+    "testY = mnist.test.labels\n",
+    "\n",
+    "test_acc = accuracy_fn(neural_net, testX, testY)\n",
+    "print(\"Testset Accuracy: {:.4f}\".format(test_acc))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [default]",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}