Update comments + minor changes

alexnet.py

@@ -1,3 +1,12 @@
+'''
+AlexNet implementation example using TensorFlow library.
+This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
+AlexNet Paper (http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)
+
+Author: Aymeric Damien
+Project: https://github.com/aymericdamien/TensorFlow-Examples/
+'''
+
 # Import MINST data
 import input_data
 mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
@@ -10,16 +19,17 @@ training_iters = 200000
 batch_size = 64
 display_step = 20
 
-#Network Parameters
-n_input = 784 #MNIST data input
-n_classes = 10 #MNIST total classes
-dropout = 0.8
+# Network Parameters
+n_input = 784 # MNIST data input (img shape: 28*28)
+n_classes = 10 # MNIST total classes (0-9 digits)
+dropout = 0.8 # Dropout, probability to keep units
 
-# Create model
+# tf Graph input
 x = tf.placeholder(tf.types.float32, [None, n_input])
 y = tf.placeholder(tf.types.float32, [None, n_classes])
-keep_prob = tf.placeholder(tf.types.float32) #dropout
+keep_prob = tf.placeholder(tf.types.float32) # dropout (keep probability)
 
+# Create AlexNet model
 def conv2d(name, l_input, w, b):
     return tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(l_input, w, strides=[1, 1, 1, 1], padding='SAME'),b), name=name)
 
@@ -29,32 +39,48 @@ def max_pool(name, l_input, k):
 def norm(name, l_input, lsize=4):
     return tf.nn.lrn(l_input, lsize, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name=name)
 
-def conv_net(_X, _weights, _biases, _dropout):
+def alex_net(_X, _weights, _biases, _dropout):
+    # Reshape input picture
     _X = tf.reshape(_X, shape=[-1, 28, 28, 1])
 
+    # Convolution Layer
     conv1 = conv2d('conv1', _X, _weights['wc1'], _biases['bc1'])
+    # Max Pooling (down-sampling)
     pool1 = max_pool('pool1', conv1, k=2)
+    # Apply Normalization
     norm1 = norm('norm1', pool1, lsize=4)
+    # Apply Dropout
     norm1 = tf.nn.dropout(norm1, _dropout)
 
+    # Convolution Layer
     conv2 = conv2d('conv2', norm1, _weights['wc2'], _biases['bc2'])
+    # Max Pooling (down-sampling)
     pool2 = max_pool('pool2', conv2, k=2)
+    # Apply Normalization
     norm2 = norm('norm2', pool2, lsize=4)
+    # Apply Dropout
     norm2 = tf.nn.dropout(norm2, _dropout)
 
+    # Convolution Layer
     conv3 = conv2d('conv3', norm2, _weights['wc3'], _biases['bc3'])
+    # Max Pooling (down-sampling)
     pool3 = max_pool('pool3', conv3, k=2)
+    # Apply Normalization
     norm3 = norm('norm3', pool3, lsize=4)
+    # Apply Dropout
     norm3 = tf.nn.dropout(norm3, _dropout)
 
-    dense1 = tf.reshape(norm3, [-1, _weights['wd1'].get_shape().as_list()[0]])
-    dense1 = tf.nn.relu(tf.matmul(dense1, _weights['wd1']) + _biases['bd1'], name='fc1')
+    # Fully connected layer
+    dense1 = tf.reshape(norm3, [-1, _weights['wd1'].get_shape().as_list()[0]]) # Reshape conv3 output to fit dense layer input
+    dense1 = tf.nn.relu(tf.matmul(dense1, _weights['wd1']) + _biases['bd1'], name='fc1') # Relu activation
 
-    dense2 = tf.nn.relu(tf.matmul(dense1, _weights['wd2']) + _biases['bd2'], name='fc2')
+    dense2 = tf.nn.relu(tf.matmul(dense1, _weights['wd2']) + _biases['bd2'], name='fc2') # Relu activation
 
+    # Output, class prediction
     out = tf.matmul(dense2, _weights['out']) + _biases['out']
     return out
 
+# Store layers weight & bias
 weights = {
     'wc1': tf.Variable(tf.random_normal([3, 3, 1, 64])),
     'wc2': tf.Variable(tf.random_normal([3, 3, 64, 128])),
@@ -63,7 +89,6 @@ weights = {
     'wd2': tf.Variable(tf.random_normal([1024, 1024])),
     'out': tf.Variable(tf.random_normal([1024, 10]))
 }
-
 biases = {
     'bc1': tf.Variable(tf.random_normal([64])),
     'bc2': tf.Variable(tf.random_normal([128])),
@@ -73,27 +98,36 @@ biases = {
     'out': tf.Variable(tf.random_normal([n_classes]))
 }
 
-pred = conv_net(x, weights, biases, keep_prob)
+# Construct model
+pred = alex_net(x, weights, biases, keep_prob)
+
+# Define loss and optimizer
 cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
 
-#Evaluate model
+# Evaluate model
 correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
 accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.types.float32))
 
-# Train
+# Initializing the variables
 init = tf.initialize_all_variables()
+
+# Launch the graph
 with tf.Session() as sess:
     sess.run(init)
     step = 1
+    # Keep training until reach max iterations
     while step * batch_size < training_iters:
         batch_xs, batch_ys = mnist.train.next_batch(batch_size)
+        # Fit training using batch data
         sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys, keep_prob: dropout})
         if step % display_step == 0:
+            # Calculate batch accuracy
             acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})
+            # Calculate batch loss
             loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})
             print "Iter " + str(step*batch_size) + ", Loss= " + "{:.6f}".format(loss) + ", Training Accuracy= " + "{:.5f}".format(acc)
         step += 1
     print "Optimization Finished!"
-    #Accuracy on 256 mnist test images
+    # Calculate accuracy for 256 mnist test images
     print "Testing Accuracy:", sess.run(accuracy, feed_dict={x: mnist.test.images[:256], y: mnist.test.labels[:256], keep_prob: 1.})

convolutional_network.py

@@ -1,3 +1,11 @@
+'''
+A Multilayer Perceptron implementation example using TensorFlow library.
+This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
+
+Author: Aymeric Damien
+Project: https://github.com/aymericdamien/TensorFlow-Examples/
+'''
+
 # Import MINST data
 import input_data
 mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
@@ -10,16 +18,17 @@ training_iters = 100000
 batch_size = 128
 display_step = 10
 
-#Network Parameters
-n_input = 784 #MNIST data input
-n_classes = 10 #MNIST total classes
-dropout = 0.75
+# Network Parameters
+n_input = 784 # MNIST data input (img shape: 28*28)
+n_classes = 10 # MNIST total classes (0-9 digits)
+dropout = 0.75 # Dropout, probability to keep units
 
-# Create model
+# tf Graph input
 x = tf.placeholder(tf.types.float32, [None, n_input])
 y = tf.placeholder(tf.types.float32, [None, n_classes])
-keep_prob = tf.placeholder(tf.types.float32) #dropout
+keep_prob = tf.placeholder(tf.types.float32) #dropout (keep probability)
 
+# Create model
 def conv2d(img, w, b):
     return tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(img, w, strides=[1, 1, 1, 1], padding='SAME'),b))
 
@@ -27,28 +36,38 @@ def max_pool(img, k):
     return tf.nn.max_pool(img, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')
 
 def conv_net(_X, _weights, _biases, _dropout):
+    # Reshape input picture
     _X = tf.reshape(_X, shape=[-1, 28, 28, 1])
 
+    # Convolution Layer
     conv1 = conv2d(_X, _weights['wc1'], _biases['bc1'])
+    # Max Pooling (down-sampling)
     conv1 = max_pool(conv1, k=2)
+    # Apply Dropout
     conv1 = tf.nn.dropout(conv1, _dropout)
 
+    # Convolution Layer
     conv2 = conv2d(conv1, _weights['wc2'], _biases['bc2'])
+    # Max Pooling (down-sampling)
     conv2 = max_pool(conv2, k=2)
+    # Apply Dropout
     conv2 = tf.nn.dropout(conv2, _dropout)
 
-    dense1 = tf.reshape(conv2, [-1, _weights['wd1'].get_shape().as_list()[0]])
-    dense1 = tf.nn.relu(tf.matmul(dense1, _weights['wd1']) + _biases['bd1'])
-    dense1 = tf.nn.dropout(dense1, _dropout)
+    # Fully connected layer
+    dense1 = tf.reshape(conv2, [-1, _weights['wd1'].get_shape().as_list()[0]]) # Reshape conv2 output to fit dense layer input
+    dense1 = tf.nn.relu(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1'])) # Relu activation
+    dense1 = tf.nn.dropout(dense1, _dropout) # Apply Dropout
 
-    out = tf.matmul(dense1, _weights['out']) + _biases['out']
+    # Output, class prediction
+    out = tf.add(tf.matmul(dense1, _weights['out']), _biases['out'])
     return out
 
+# Store layers weight & bias
 weights = {
-    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])),
-    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])),
-    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])),
-    'out': tf.Variable(tf.random_normal([1024, 10]))
+    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])), # 5x5 conv, 1 input, 32 outputs
+    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])), # 5x5 conv, 32 inputs, 64 outputs
+    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])), # fully connected, 7*7*64 inputs, 1024 outputs
+    'out': tf.Variable(tf.random_normal([1024, n_classes])) # 1024 inputs, 10 outputs (class prediction)
 }
 
 biases = {
@@ -58,27 +77,36 @@ biases = {
     'out': tf.Variable(tf.random_normal([n_classes]))
 }
 
+# Construct model
 pred = conv_net(x, weights, biases, keep_prob)
+
+# Define loss and optimizer
 cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
 
-#Evaluate model
+# Evaluate model
 correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
 accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.types.float32))
 
-# Train
+# Initializing the variables
 init = tf.initialize_all_variables()
+
+# Launch the graph
 with tf.Session() as sess:
     sess.run(init)
     step = 1
+    # Keep training until reach max iterations
     while step * batch_size < training_iters:
         batch_xs, batch_ys = mnist.train.next_batch(batch_size)
+        # Fit training using batch data
         sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys, keep_prob: dropout})
         if step % display_step == 0:
+            # Calculate batch accuracy
             acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})
+            # Calculate batch loss
             loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})
             print "Iter " + str(step*batch_size) + ", Loss= " + "{:.6f}".format(loss) + ", Training Accuracy= " + "{:.5f}".format(acc)
         step += 1
     print "Optimization Finished!"
-    #Accuracy on 256 mnist test images
-    print "Accuracy:", sess.run(accuracy, feed_dict={x: mnist.test.images[:256], y: mnist.test.labels[:256], keep_prob: 1.})
+    # Calculate accuracy for 256 mnist test images
+    print "Testing Accuracy:", sess.run(accuracy, feed_dict={x: mnist.test.images[:256], y: mnist.test.labels[:256], keep_prob: 1.})

multilayer_perceptron.py

@@ -1,3 +1,11 @@
+'''
+A Multilayer Perceptron implementation example using TensorFlow library.
+This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
+
+Author: Aymeric Damien
+Project: https://github.com/aymericdamien/TensorFlow-Examples/
+'''
+
 # Import MINST data
 import input_data
 mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
@@ -11,53 +19,66 @@ batch_size = 100
 display_step = 1
 
 #Network Parameters
-n_hidden_1 = 256
-n_hidden_2 = 256
-n_input = 784 #MNIST data input
-n_classes = 10 #MNIST total classes
+n_hidden_1 = 256 # 1st layer num features
+n_hidden_2 = 256 # 2nd layer num features
+n_input = 784 # MNIST data input (img shape: 28*28)
+n_classes = 10 # MNIST total classes (0-9 digits)
 
-# Create model
+# tf Graph input
 x = tf.placeholder("float", [None, n_input])
 y = tf.placeholder("float", [None, n_classes])
 
+# Create model
 def multilayer_perceptron(_X, _weights, _biases):
-    layer_1 = tf.nn.relu(tf.matmul(_X, _weights['h1']) + _biases['b1']) #Hidden layer with RELU activation
-    layer_2 = tf.nn.relu(tf.matmul(layer_1, _weights['h2']) + _biases['b2']) #Hidden layer with RELU activation
+    layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) #Hidden layer with RELU activation
+    layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) #Hidden layer with RELU activation
     return tf.matmul(layer_2, weights['out']) + biases['out']
 
+# Store layers weight & bias
 weights = {
     'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
     'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
     'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
 }
-
 biases = {
     'b1': tf.Variable(tf.random_normal([n_hidden_1])),
     'b2': tf.Variable(tf.random_normal([n_hidden_2])),
     'out': tf.Variable(tf.random_normal([n_classes]))
 }
 
+# Construct model
 pred = multilayer_perceptron(x, weights, biases)
-cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
-optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
 
-# Train
+# Define loss and optimizer
+cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) # Softmax loss
+optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer
+
+# Initializing the variables
 init = tf.initialize_all_variables()
+
+# Launch the graph
 with tf.Session() as sess:
     sess.run(init)
+
+    # Training cycle
     for epoch in range(training_epochs):
         avg_cost = 0.
         total_batch = int(mnist.train.num_examples/batch_size)
+        # Loop over all batches
         for i in range(total_batch):
             batch_xs, batch_ys = mnist.train.next_batch(batch_size)
+            # Fit training using batch data
             sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys})
+            # Compute average loss
             avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys})/total_batch
+        # Display logs per epoch step
         if epoch % display_step == 0:
             print "Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost)
 
     print "Optimization Finished!"
 
-    # Test trained model
-    correct_prediction = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
+    # Test model
+    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
+    # Calculate accuracy
     accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
     print "Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels})