Merge branch 'master' into master

README.md

@@ -9,15 +9,15 @@ To propose a model for inclusion please submit a pull request.
 
 ## Models
 - [autoencoder](autoencoder) -- various autoencoders
+- [differential_privacy](differential_privacy) -- privacy-preserving student models from multiple teachers
+- [im2txt](im2txt) -- image-to-text neural network for image captioning.
 - [inception](inception) -- deep convolutional networks for computer vision
 - [namignizer](namignizer) -- recognize and generate names
 - [neural_gpu](neural_gpu) -- highly parallel neural computer
-- [privacy](privacy) -- privacy-preserving student models from multiple teachers
+- [neural_programmer](neural_programmer) -- neural network augmented with logic and mathematic operations.
 - [resnet](resnet) -- deep and wide residual networks
 - [slim](slim) -- image classification models in TF-Slim
 - [swivel](swivel) -- the Swivel algorithm for generating word embeddings
 - [syntaxnet](syntaxnet) -- neural models of natural language syntax
 - [textsum](textsum) -- sequence-to-sequence with attention model for text summarization.
 - [transformer](transformer) -- spatial transformer network, which allows the spatial manipulation of data within the network
-- [im2txt](im2txt) -- image-to-text neural network for image captioning.
-- [neural_programmer](neural programmer) -- neural network augmented with logic and mathematic operations.

differential_privacy/multiple_teachers/deep_cnn.py

@@ -341,7 +341,7 @@ def loss_fun(logits, labels):
   # Calculate the cross entropy between labels and predictions
   labels = tf.cast(labels, tf.int64)
   cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
-      logits, labels, name='cross_entropy_per_example')
+      logits=logits, labels=labels, name='cross_entropy_per_example')
 
   # Calculate the average cross entropy loss across the batch.
   cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')

im2txt/README.md

@@ -119,7 +119,7 @@ approximately 10 times slower.
 First ensure that you have installed the following required packages:
 
 * **Bazel** ([instructions](http://bazel.io/docs/install.html)).
-* **TensorFlow** ([instructions](https://www.tensorflow.org/versions/master/get_started/os_setup.html)).
+* **TensorFlow** r0.12 or greater ([instructions](https://www.tensorflow.org/versions/master/get_started/os_setup.html)).
 * **NumPy** ([instructions](http://www.scipy.org/install.html)).
 * **Natural Language Toolkit (NLTK)**:
     * First install NLTK ([instructions](http://www.nltk.org/install.html)).

im2txt/im2txt/inference_utils/caption_generator.py

@@ -54,6 +54,16 @@ class Caption(object):
       return -1
     else:
       return 1
+  
+  # For Python 3 compatibility (__cmp__ is deprecated).
+  def __lt__(self, other):
+    assert isinstance(other, Caption)
+    return self.score < other.score
+  
+  # Also for Python 3 compatibility.
+  def __eq__(self, other):
+    assert isinstance(other, Caption)
+    return self.score == other.score
 
 
 class TopN(object):

im2txt/im2txt/ops/image_embedding_test.py

@@ -41,7 +41,7 @@ class InceptionV3Test(tf.test.TestCase):
   def _countInceptionParameters(self):
     """Counts the number of parameters in the inception model at top scope."""
     counter = {}
-    for v in tf.all_variables():
+    for v in tf.global_variables():
       name_tokens = v.op.name.split("/")
       if name_tokens[0] == "InceptionV3":
         name = "InceptionV3/" + name_tokens[1]
@@ -85,7 +85,7 @@ class InceptionV3Test(tf.test.TestCase):
     self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
 
     self._verifyParameterCounts()
-    self._assertCollectionSize(376, tf.GraphKeys.VARIABLES)
+    self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
     self._assertCollectionSize(188, tf.GraphKeys.TRAINABLE_VARIABLES)
     self._assertCollectionSize(188, tf.GraphKeys.UPDATE_OPS)
     self._assertCollectionSize(94, tf.GraphKeys.REGULARIZATION_LOSSES)
@@ -98,7 +98,7 @@ class InceptionV3Test(tf.test.TestCase):
     self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
 
     self._verifyParameterCounts()
-    self._assertCollectionSize(376, tf.GraphKeys.VARIABLES)
+    self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
     self._assertCollectionSize(188, tf.GraphKeys.TRAINABLE_VARIABLES)
     self._assertCollectionSize(0, tf.GraphKeys.UPDATE_OPS)
     self._assertCollectionSize(94, tf.GraphKeys.REGULARIZATION_LOSSES)
@@ -111,7 +111,7 @@ class InceptionV3Test(tf.test.TestCase):
     self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
 
     self._verifyParameterCounts()
-    self._assertCollectionSize(376, tf.GraphKeys.VARIABLES)
+    self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
     self._assertCollectionSize(0, tf.GraphKeys.TRAINABLE_VARIABLES)
     self._assertCollectionSize(0, tf.GraphKeys.UPDATE_OPS)
     self._assertCollectionSize(0, tf.GraphKeys.REGULARIZATION_LOSSES)
@@ -124,7 +124,7 @@ class InceptionV3Test(tf.test.TestCase):
     self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
 
     self._verifyParameterCounts()
-    self._assertCollectionSize(376, tf.GraphKeys.VARIABLES)
+    self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
     self._assertCollectionSize(0, tf.GraphKeys.TRAINABLE_VARIABLES)
     self._assertCollectionSize(0, tf.GraphKeys.UPDATE_OPS)
     self._assertCollectionSize(0, tf.GraphKeys.REGULARIZATION_LOSSES)

im2txt/im2txt/ops/image_processing.py

@@ -92,7 +92,7 @@ def process_image(encoded_image,
   # only logged in thread 0.
   def image_summary(name, image):
     if not thread_id:
-      tf.image_summary(name, tf.expand_dims(image, 0))
+      tf.summary.image(name, tf.expand_dims(image, 0))
 
   # Decode image into a float32 Tensor of shape [?, ?, 3] with values in [0, 1).
   with tf.name_scope("decode", values=[encoded_image]):
@@ -128,6 +128,6 @@ def process_image(encoded_image,
   image_summary("final_image", image)
 
   # Rescale to [-1,1] instead of [0, 1]
-  image = tf.sub(image, 0.5)
-  image = tf.mul(image, 2.0)
+  image = tf.subtract(image, 0.5)
+  image = tf.multiply(image, 2.0)
   return image

im2txt/im2txt/ops/inputs.py

@@ -116,7 +116,7 @@ def prefetch_input_data(reader,
     enqueue_ops.append(values_queue.enqueue([value]))
   tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(
       values_queue, enqueue_ops))
-  tf.scalar_summary(
+  tf.summary.scalar(
       "queue/%s/fraction_of_%d_full" % (values_queue.name, capacity),
       tf.cast(values_queue.size(), tf.float32) * (1. / capacity))
 
@@ -181,7 +181,7 @@ def batch_with_dynamic_pad(images_and_captions,
   enqueue_list = []
   for image, caption in images_and_captions:
     caption_length = tf.shape(caption)[0]
-    input_length = tf.expand_dims(tf.sub(caption_length, 1), 0)
+    input_length = tf.expand_dims(tf.subtract(caption_length, 1), 0)
 
     input_seq = tf.slice(caption, [0], input_length)
     target_seq = tf.slice(caption, [1], input_length)
@@ -197,8 +197,8 @@ def batch_with_dynamic_pad(images_and_captions,
 
   if add_summaries:
     lengths = tf.add(tf.reduce_sum(mask, 1), 1)
-    tf.scalar_summary("caption_length/batch_min", tf.reduce_min(lengths))
-    tf.scalar_summary("caption_length/batch_max", tf.reduce_max(lengths))
-    tf.scalar_summary("caption_length/batch_mean", tf.reduce_mean(lengths))
+    tf.summary.scalar("caption_length/batch_min", tf.reduce_min(lengths))
+    tf.summary.scalar("caption_length/batch_max", tf.reduce_max(lengths))
+    tf.summary.scalar("caption_length/batch_mean", tf.reduce_mean(lengths))
 
   return images, input_seqs, target_seqs, mask

im2txt/im2txt/show_and_tell_model.py

@@ -244,10 +244,10 @@ class ShowAndTellModel(object):
     # This LSTM cell has biases and outputs tanh(new_c) * sigmoid(o), but the
     # modified LSTM in the "Show and Tell" paper has no biases and outputs
     # new_c * sigmoid(o).
-    lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(
+    lstm_cell = tf.contrib.rnn.BasicLSTMCell(
         num_units=self.config.num_lstm_units, state_is_tuple=True)
     if self.mode == "train":
-      lstm_cell = tf.nn.rnn_cell.DropoutWrapper(
+      lstm_cell = tf.contrib.rnn.DropoutWrapper(
           lstm_cell,
           input_keep_prob=self.config.lstm_dropout_keep_prob,
           output_keep_prob=self.config.lstm_dropout_keep_prob)
@@ -264,13 +264,13 @@ class ShowAndTellModel(object):
       if self.mode == "inference":
         # In inference mode, use concatenated states for convenient feeding and
         # fetching.
-        tf.concat(1, initial_state, name="initial_state")
+        tf.concat_v2(initial_state, 1, name="initial_state")
 
         # Placeholder for feeding a batch of concatenated states.
         state_feed = tf.placeholder(dtype=tf.float32,
                                     shape=[None, sum(lstm_cell.state_size)],
                                     name="state_feed")
-        state_tuple = tf.split(1, 2, state_feed)
+        state_tuple = tf.split(value=state_feed, num_or_size_splits=2, axis=1)
 
         # Run a single LSTM step.
         lstm_outputs, state_tuple = lstm_cell(
@@ -278,7 +278,7 @@ class ShowAndTellModel(object):
             state=state_tuple)
 
         # Concatentate the resulting state.
-        tf.concat(1, state_tuple, name="state")
+        tf.concat_v2(state_tuple, 1, name="state")
       else:
         # Run the batch of sequence embeddings through the LSTM.
         sequence_length = tf.reduce_sum(self.input_mask, 1)
@@ -307,18 +307,19 @@ class ShowAndTellModel(object):
       weights = tf.to_float(tf.reshape(self.input_mask, [-1]))
 
       # Compute losses.
-      losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, targets)
-      batch_loss = tf.div(tf.reduce_sum(tf.mul(losses, weights)),
+      losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=targets,
+                                                              logits=logits)
+      batch_loss = tf.div(tf.reduce_sum(tf.multiply(losses, weights)),
                           tf.reduce_sum(weights),
                           name="batch_loss")
-      tf.contrib.losses.add_loss(batch_loss)
-      total_loss = tf.contrib.losses.get_total_loss()
+      tf.losses.add_loss(batch_loss)
+      total_loss = tf.losses.get_total_loss()
 
       # Add summaries.
-      tf.scalar_summary("batch_loss", batch_loss)
-      tf.scalar_summary("total_loss", total_loss)
+      tf.summary.scalar("losses/batch_loss", batch_loss)
+      tf.summary.scalar("losses/total_loss", total_loss)
       for var in tf.trainable_variables():
-        tf.histogram_summary(var.op.name, var)
+        tf.summary.histogram("parameters/" + var.op.name, var)
 
       self.total_loss = total_loss
       self.target_cross_entropy_losses = losses  # Used in evaluation.

im2txt/im2txt/show_and_tell_model_test.py

@@ -63,7 +63,7 @@ class ShowAndTellModelTest(tf.test.TestCase):
   def _countModelParameters(self):
     """Counts the number of parameters in the model at top level scope."""
     counter = {}
-    for v in tf.all_variables():
+    for v in tf.global_variables():
       name = v.op.name.split("/")[0]
       num_params = v.get_shape().num_elements()
       assert num_params
@@ -98,7 +98,7 @@ class ShowAndTellModelTest(tf.test.TestCase):
     fetches = expected_shapes.keys()
 
     with self.test_session() as sess:
-      sess.run(tf.initialize_all_variables())
+      sess.run(tf.global_variables_initializer())
       outputs = sess.run(fetches, feed_dict)
 
     for index, output in enumerate(outputs):

inception/inception/data/build_image_data.py

@@ -137,13 +137,13 @@ def _convert_to_example(filename, image_buffer, label, text, height, width):
   example = tf.train.Example(features=tf.train.Features(feature={
       'image/height': _int64_feature(height),
       'image/width': _int64_feature(width),
-      'image/colorspace': _bytes_feature(colorspace),
+      'image/colorspace': _bytes_feature(tf.compat.as_bytes(colorspace)),
       'image/channels': _int64_feature(channels),
       'image/class/label': _int64_feature(label),
-      'image/class/text': _bytes_feature(text),
-      'image/format': _bytes_feature(image_format),
-      'image/filename': _bytes_feature(os.path.basename(filename)),
-      'image/encoded': _bytes_feature(image_buffer)}))
+      'image/class/text': _bytes_feature(tf.compat.as_bytes(text)),
+      'image/format': _bytes_feature(tf.compat.as_bytes(image_format)),
+      'image/filename': _bytes_feature(tf.compat.as_bytes(os.path.basename(filename))),
+      'image/encoded': _bytes_feature(tf.compat.as_bytes(image_buffer))}))
   return example
 
 

inception/inception/inception_train.py

@@ -79,7 +79,7 @@ RMSPROP_MOMENTUM = 0.9             # Momentum in RMSProp.
 RMSPROP_EPSILON = 1.0              # Epsilon term for RMSProp.
 
 
-def _tower_loss(images, labels, num_classes, scope):
+def _tower_loss(images, labels, num_classes, scope, reuse_variables=None):
   """Calculate the total loss on a single tower running the ImageNet model.
 
   We perform 'batch splitting'. This means that we cut up a batch across
@@ -103,9 +103,10 @@ def _tower_loss(images, labels, num_classes, scope):
   restore_logits = not FLAGS.fine_tune
 
   # Build inference Graph.
-  logits = inception.inference(images, num_classes, for_training=True,
-                               restore_logits=restore_logits,
-                               scope=scope)
+  with tf.variable_scope(tf.get_variable_scope(), reuse=reuse_variables):
+    logits = inception.inference(images, num_classes, for_training=True,
+                                 restore_logits=restore_logits,
+                                 scope=scope)
 
   # Build the portion of the Graph calculating the losses. Note that we will
   # assemble the total_loss using a custom function below.
@@ -220,13 +221,14 @@ def train(dataset):
     # Number of classes in the Dataset label set plus 1.
     # Label 0 is reserved for an (unused) background class.
     num_classes = dataset.num_classes() + 1
-    
+
      # Split the batch of images and labels for towers.
     images_splits = tf.split(0, FLAGS.num_gpus, images)
     labels_splits = tf.split(0, FLAGS.num_gpus, labels)
 
     # Calculate the gradients for each model tower.
     tower_grads = []
+    reuse_variables = None
     for i in range(FLAGS.num_gpus):
       with tf.device('/gpu:%d' % i):
         with tf.name_scope('%s_%d' % (inception.TOWER_NAME, i)) as scope:
@@ -236,10 +238,10 @@ def train(dataset):
             # function constructs the entire ImageNet model but shares the
             # variables across all towers.
             loss = _tower_loss(images_splits[i], labels_splits[i], num_classes,
-                               scope)
+                               scope, reuse_variables)
 
           # Reuse variables for the next tower.
-          tf.get_variable_scope().reuse_variables()
+          reuse_variables = True
 
           # Retain the summaries from the final tower.
           summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)

inception/inception/slim/losses.py

@@ -163,8 +163,8 @@ def cross_entropy_loss(logits, one_hot_labels, label_smoothing=0,
       smooth_positives = 1.0 - label_smoothing
       smooth_negatives = label_smoothing / num_classes
       one_hot_labels = one_hot_labels * smooth_positives + smooth_negatives
-    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits,
-                                                            one_hot_labels,
+    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
+                                                            labels=one_hot_labels,
                                                             name='xentropy')
     weight = tf.convert_to_tensor(weight,
                                   dtype=logits.dtype.base_dtype,

inception/inception/slim/ops.py

@@ -91,7 +91,7 @@ def batch_norm(inputs,
     if scale:
       gamma = variables.variable('gamma',
                                  params_shape,
-                                 initializer=tf.ones_initializer,
+                                 initializer=tf.ones_initializer(),
                                  trainable=trainable,
                                  restore=restore)
     # Create moving_mean and moving_variance add them to
@@ -105,7 +105,7 @@ def batch_norm(inputs,
                                      collections=moving_collections)
     moving_variance = variables.variable('moving_variance',
                                          params_shape,
-                                         initializer=tf.ones_initializer,
+                                         initializer=tf.ones_initializer(),
                                          trainable=False,
                                          restore=restore,
                                          collections=moving_collections)

neural_programmer/parameters.py

@@ -72,7 +72,7 @@ class Parameters:
         self.RandomUniformInit([1, embedding_dims]))
     params["break_conditional"] = tf.Variable(
         self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
-    init = tf.initialize_all_variables()
+    init = tf.global_variables_initializer()
     return params, global_step, init
 
   def RandomUniformInit(self, shape):

next_frame_prediction/README.md

@@ -0,0 +1,91 @@
+<font size=4><b>Visual Dynamics: Probabilistic Future Frame Synthesis via Cross Convolutional Networks.</b></font>
+
+<b>Introduction</b>
+
+https://arxiv.org/pdf/1607.02586v1.pdf
+
+This is an implementation based on my understanding, with small
+variations. It doesn't necessarily represents the paper published
+by the original authors.
+
+Authors: Xin Pan (Github: panyx0718), Anelia Angelova
+
+<b>Results:</b>
+
+<left>
+![Sample1](g3doc/cross_conv.png)
+</left>
+<left>
+![Sample2](g3doc/cross_conv2.png)
+</left>
+
+<left>
+![Loss](g3doc/cross_conv3.png)
+</left>
+
+
+<b>Prerequisite:</b>
+
+1. Install TensorFlow (r0.12), Bazel.
+
+2. Download the Sprites dataset or generate moving object dataset.
+
+Sprites data is located here:
+
+http://www.scottreed.info/files/nips2015-analogy-data.tar.gz
+
+Convert .mat files into images and use sprites_gen.py to convert them
+to tf.SequenceExample.
+
+<b>How to run:</b>
+
+```shell
+ls -R
+.:
+data  next_frame_prediction  WORKSPACE
+
+./data:
+tfrecords  tfrecords_test
+
+./next_frame_prediction:
+cross_conv  g3doc  README.md
+
+./next_frame_prediction/cross_conv:
+BUILD  eval.py  objects_gen.py  model.py  reader.py  sprites_gen.py  train.py
+
+./next_frame_prediction/g3doc:
+cross_conv2.png  cross_conv3.png  cross_conv.png
+
+
+# Build everything.
+bazel build -c opt next_frame_prediction/...
+
+# The following example runs the generated 2d objects.
+# For Sprites dataset, image_size should be 60, norm_scale should be 255.0.
+# Batch size is normally 16~64, depending on your memory size.
+#
+# Run training.
+bazel-bin/next_frame_prediction/cross_conv/train \
+  --batch_size=1 \
+  --data_filepattern=data/tfrecords \
+  --image_size=64 \
+  --log_root=/tmp/predict
+
+step: 1, loss: 24.428671
+step: 2, loss: 19.211605
+step: 3, loss: 5.543143
+step: 4, loss: 3.035339
+step: 5, loss: 1.771392
+step: 6, loss: 2.099824
+step: 7, loss: 1.747665
+step: 8, loss: 1.572436
+step: 9, loss: 1.586816
+step: 10, loss: 1.434191
+#
+# Run eval.
+bazel-bin/next_frame_prediction/cross_conv/eval \
+  --batch_size=1 \
+  --data_filepattern=data/tfrecords_test \
+  --image_size=64 \
+  --log_root=/tmp/predict
+```

next_frame_prediction/cross_conv/BUILD

@@ -0,0 +1,48 @@
+licenses(["notice"])  # Apache 2.0
+
+package_group(
+    name = "internal",
+    packages = [
+        "//next_frame_prediction/...",
+    ],
+)
+
+package(default_visibility = [":internal"])
+
+py_library(
+    name = "model",
+    srcs = ["model.py"],
+)
+
+py_library(
+    name = "reader",
+    srcs = ["reader.py"],
+)
+
+py_binary(
+    name = "train",
+    srcs = ["train.py"],
+    deps = [
+        ":model",
+        ":reader",
+    ],
+)
+
+py_binary(
+    name = "eval",
+    srcs = ["eval.py"],
+    deps = [
+        ":model",
+        ":reader",
+    ],
+)
+
+py_binary(
+    name = "example_gen",
+    srcs = ["example_gen.py"],
+)
+
+py_binary(
+    name = "sprites_gen",
+    srcs = ["sprites_gen.py"],
+)

next_frame_prediction/cross_conv/eval.py

@@ -0,0 +1,118 @@
+# Copyright 2016 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Eval Cross Convolutional Model."""
+import io
+import os
+import sys
+import time
+
+import numpy as np
+import tensorflow as tf
+
+import model as cross_conv_model
+import reader
+
+FLAGS = tf.flags.FLAGS
+tf.flags.DEFINE_string('log_root', '/tmp/moving_obj', 'The root dir of output.')
+tf.flags.DEFINE_string('data_filepattern',
+                       'est',
+                       'training data file pattern.')
+tf.flags.DEFINE_integer('batch_size', 1, 'Batch size.')
+tf.flags.DEFINE_integer('image_size', 64, 'Image height and width.')
+tf.flags.DEFINE_float('norm_scale', 1.0, 'Normalize the original image')
+tf.flags.DEFINE_float('scale', 10.0,
+                      'Scale the image after norm_scale and move the diff '
+                      'to the positive realm.')
+tf.flags.DEFINE_integer('sequence_length', 2, 'tf.SequenceExample length.')
+tf.flags.DEFINE_integer('eval_batch_count', 100,
+                        'Average the result this number of examples.')
+tf.flags.DEFINE_bool('l2_loss', True, 'If true, include l2_loss.')
+tf.flags.DEFINE_bool('reconstr_loss', False, 'If true, include reconstr_loss.')
+tf.flags.DEFINE_bool('kl_loss', True, 'If true, include KL loss.')
+
+slim = tf.contrib.slim
+
+
+def _Eval():
+  params = dict()
+  params['batch_size'] = FLAGS.batch_size
+  params['seq_len'] = FLAGS.sequence_length
+  params['image_size'] = FLAGS.image_size
+  params['is_training'] = False
+  params['norm_scale'] = FLAGS.norm_scale
+  params['scale'] = FLAGS.scale
+  params['l2_loss'] = FLAGS.l2_loss
+  params['reconstr_loss'] = FLAGS.reconstr_loss
+  params['kl_loss'] = FLAGS.kl_loss
+
+  eval_dir = os.path.join(FLAGS.log_root, 'eval')
+
+  images = reader.ReadInput(
+      FLAGS.data_filepattern, shuffle=False, params=params)
+  images *= params['scale']
+  # Increase the value makes training much faster.
+  image_diff_list = reader.SequenceToImageAndDiff(images)
+  model = cross_conv_model.CrossConvModel(image_diff_list, params)
+  model.Build()
+
+  summary_writer = tf.summary.FileWriter(eval_dir)
+  saver = tf.train.Saver()
+  sess = tf.Session('', config=tf.ConfigProto(allow_soft_placement=True))
+  tf.train.start_queue_runners(sess)
+
+  while True:
+    time.sleep(60)
+    try:
+      ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
+    except tf.errors.OutOfRangeError as e:
+      sys.stderr.write('Cannot restore checkpoint: %s\n' % e)
+      continue
+    if not (ckpt_state and ckpt_state.model_checkpoint_path):
+      sys.stderr.write('No model to eval yet at %s\n' % FLAGS.log_root)
+      continue
+    sys.stderr.write('Loading checkpoint %s\n' %
+                     ckpt_state.model_checkpoint_path)
+    saver.restore(sess, ckpt_state.model_checkpoint_path)
+    # Use the empirical distribution of z from training set.
+    if not tf.gfile.Exists(os.path.join(FLAGS.log_root, 'z_mean.npy')):
+      sys.stderr.write('No z at %s\n' % FLAGS.log_root)
+      continue
+
+    with tf.gfile.Open(os.path.join(FLAGS.log_root, 'z_mean.npy')) as f:
+      sample_z_mean = np.load(io.BytesIO(f.read()))
+    with tf.gfile.Open(
+        os.path.join(FLAGS.log_root, 'z_stddev_log.npy')) as f:
+      sample_z_stddev_log = np.load(io.BytesIO(f.read()))
+
+    total_loss = 0.0
+    for _ in xrange(FLAGS.eval_batch_count):
+      loss_val, total_steps, summaries = sess.run(
+          [model.loss, model.global_step, model.summary_op],
+          feed_dict={model.z_mean: sample_z_mean,
+                     model.z_stddev_log: sample_z_stddev_log})
+      total_loss += loss_val
+
+    summary_writer.add_summary(summaries, total_steps)
+    sys.stderr.write('steps: %d, loss: %f\n' %
+                     (total_steps, total_loss / FLAGS.eval_batch_count))
+
+
+def main(_):
+  _Eval()
+
+
+if __name__ == '__main__':
+  tf.app.run()

next_frame_prediction/cross_conv/example_gen.py

@@ -0,0 +1,92 @@
+# Copyright 2016 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Generate examples of two objects moving in different directions."""
+import random
+import sys
+
+import numpy as np
+import tensorflow as tf
+
+
+tf.flags.DEFINE_string('out_file', '',
+                       'Output file for the tfrecords.')
+
+
+def _add_object(obj_type, image, image2, xpos, ypos):
+  """Add a moving obj to two consecutive images."""
+  obj_size = random.randint(8, 10)
+  channel = random.randint(0, 2)
+  move = random.randint(6, 10)
+
+  obj = np.zeros([obj_size, obj_size, 3])
+  if obj_type == 'rectangle':
+    xpos2 = xpos + move
+    ypos2 = ypos
+    for i in xrange(obj_size):
+      obj[i, 0:i+1, channel] = [1.0 for _ in xrange(i+1)]
+  elif obj_type == 'square':
+    xpos2 = xpos
+    ypos2 = ypos + move
+    obj[:, :, channel] = 1.0
+
+  for x in xrange(obj_size):
+    for y in xrange(obj_size):
+      if obj[x, y, channel] == 1.0:
+        image[xpos+x, ypos+y, channel] = 1.0
+        image2[xpos2+x, ypos2+y, channel] = 1.0
+
+
+def _images_to_example(image, image2):
+  """Convert two consecutive images to SequenceExample."""
+  example = tf.SequenceExample()
+  feature_list = example.feature_lists.feature_list['moving_objs']
+  feature = feature_list.feature.add()
+  feature.float_list.value.extend(np.reshape(image, [-1]).tolist())
+  feature = feature_list.feature.add()
+  feature.float_list.value.extend(np.reshape(image2, [-1]).tolist())
+  return example
+
+
+def generate_input():
+  """Generate tfrecords."""
+  writer = tf.python_io.TFRecordWriter(tf.flags.FLAGS.out_file)
+  writer2 = tf.python_io.TFRecordWriter(tf.flags.FLAGS.out_file + '_test')
+
+  examples = []
+  for xpos in xrange(0, 40, 3):
+    for ypos in xrange(0, 40, 3):
+      for xpos2 in xrange(0, 40, 3):
+        for ypos2 in xrange(0, 40, 3):
+          image = np.zeros([64, 64, 3])
+          image2 = np.zeros([64, 64, 3])
+          _add_object('rectangle', image, image2, xpos, ypos)
+          _add_object('square', image, image2, xpos2, ypos2)
+          examples.append(_images_to_example(image, image2))
+
+  sys.stderr.write('Finish generating examples.\n')
+  random.shuffle(examples)
+  for count, ex in enumerate(examples):
+    if count % 10 == 0:
+      writer2.write(ex.SerializeToString())
+    else:
+      writer.write(ex.SerializeToString())
+
+def main(_):
+  generate_input()
+
+
+if __name__ == '__main__':
+  tf.app.run()

next_frame_prediction/cross_conv/model.py

@@ -0,0 +1,232 @@
+# Copyright 2016 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Cross Convolutional Model.
+
+https://arxiv.org/pdf/1607.02586v1.pdf
+"""
+import math
+import sys
+
+import tensorflow as tf
+
+slim = tf.contrib.slim
+
+
+class CrossConvModel(object):
+
+  def __init__(self, image_diff_list, params):
+    """Constructor.
+
+    Args:
+      image_diff_list: A list of (image, diff) tuples, with shape
+          [batch_size, image_size, image_size, 3] and image_sizes as
+          [32, 64, 128, 256].
+      params: Dict of parameters.
+    """
+    self.images = [i for (i, _) in image_diff_list]
+    # Move the diff to the positive realm.
+    self.diffs = [(d + params['scale']) / 2 for (i, d) in image_diff_list]
+    self.params = params
+
+  def Build(self):
+    with tf.device('/gpu:0'):
+      with slim.arg_scope([slim.conv2d],
+                          activation_fn=tf.nn.relu,
+                          normalizer_fn=slim.batch_norm,
+                          normalizer_params={'is_training':
+                                             self.params['is_training']}):
+        self._BuildMotionKernel()
+        encoded_images = self._BuildImageEncoder()
+        cross_conved_images = self._CrossConv(encoded_images)
+        self._BuildImageDecoder(cross_conved_images)
+        self._BuildLoss()
+
+      image = self.images[1]
+      diff = self.diffs[1]
+
+      self.global_step = tf.Variable(0, name='global_step', trainable=False)
+
+      if self.params['is_training']:
+        self._BuildTrainOp()
+
+      diff = diff * 2.0 - self.params['scale']
+      diff_output = self.diff_output * 2.0 - self.params['scale']
+      concat_image = tf.concat(
+          1, [image, image + diff_output, image + diff, diff_output])
+      tf.summary.image('origin_predict_expect_predictdiff', concat_image)
+      self.summary_op = tf.summary.merge_all()
+      return self.loss
+
+  def _BuildTrainOp(self):
+    lrn_rate = tf.maximum(
+        0.01,  # min_lr_rate.
+        tf.train.exponential_decay(
+            self.params['learning_rate'], self.global_step, 10000, 0.5))
+    tf.summary.scalar('learning rate', lrn_rate)
+    optimizer = tf.train.GradientDescentOptimizer(lrn_rate)
+    self.train_op = slim.learning.create_train_op(
+        self.loss, optimizer, global_step=self.global_step)
+
+  def _BuildLoss(self):
+    # 1. reconstr_loss seems doesn't do better than l2 loss.
+    # 2. Only works when using reduce_mean. reduce_sum doesn't work.
+    # 3. It seems kl loss doesn't play an important role.
+    self.loss = 0
+    with tf.variable_scope('loss'):
+      if self.params['l2_loss']:
+        l2_loss = tf.reduce_mean(tf.square(self.diff_output - self.diffs[1]))
+        tf.summary.scalar('l2_loss', l2_loss)
+        self.loss += l2_loss
+      if self.params['reconstr_loss']:
+        reconstr_loss = (-tf.reduce_mean(
+            self.diffs[1] * (1e-10 + self.diff_output) +
+            (1-self.diffs[1]) * tf.log(1e-10 + 1 - self.diff_output)))
+        reconstr_loss = tf.check_numerics(reconstr_loss, 'reconstr_loss')
+        tf.summary.scalar('reconstr_loss', reconstr_loss)
+        self.loss += reconstr_loss
+      if self.params['kl_loss']:
+        kl_loss = (0.5 * tf.reduce_mean(
+            tf.square(self.z_mean) + tf.square(self.z_stddev) -
+            2 * self.z_stddev_log - 1))
+        tf.summary.scalar('kl_loss', kl_loss)
+        self.loss += kl_loss
+
+      tf.summary.scalar('loss', self.loss)
+
+  def _BuildMotionKernel(self):
+    image = self.images[-2]
+    diff = self.diffs[-2]
+    shape = image.get_shape().as_list()
+    assert shape[1] == shape[2] and shape[1] == 128
+    batch_size = shape[0]
+
+    net = tf.concat(3, [image, diff])
+    with tf.variable_scope('motion_encoder'):
+      with slim.arg_scope([slim.conv2d], padding='VALID'):
+        net = slim.conv2d(net, 96, [5, 5], stride=1)
+        net = slim.max_pool2d(net, [2, 2])
+        net = slim.conv2d(net, 96, [5, 5], stride=1)
+        net = slim.max_pool2d(net, [2, 2])
+        net = slim.conv2d(net, 128, [5, 5], stride=1)
+        net = slim.conv2d(net, 128, [5, 5], stride=1)
+        net = slim.max_pool2d(net, [2, 2])
+        net = slim.conv2d(net, 256, [4, 4], stride=1)
+        net = slim.conv2d(net, 256, [3, 3], stride=1)
+
+        z = tf.reshape(net, shape=[batch_size, -1])
+        self.z_mean, self.z_stddev_log = tf.split(
+            split_dim=1, num_split=2, value=z)
+        self.z_stddev = tf.exp(self.z_stddev_log)
+
+        epsilon = tf.random_normal(
+            self.z_mean.get_shape().as_list(), 0, 1, dtype=tf.float32)
+        kernel = self.z_mean + tf.multiply(self.z_stddev, epsilon)
+
+        width = int(math.sqrt(kernel.get_shape().as_list()[1] // 128))
+        kernel = tf.reshape(kernel, [batch_size, width, width, 128])
+    with tf.variable_scope('kernel_decoder'):
+      with slim.arg_scope([slim.conv2d], padding='SAME'):
+        kernel = slim.conv2d(kernel, 128, [5, 5], stride=1)
+        self.kernel = slim.conv2d(kernel, 128, [5, 5], stride=1)
+
+    sys.stderr.write('kernel shape: %s\n' % kernel.get_shape())
+
+  def _BuildImageEncoder(self):
+    feature_maps = []
+    for (i, image) in enumerate(self.images):
+      with tf.variable_scope('image_encoder_%d' % i):
+        with slim.arg_scope([slim.conv2d, slim.max_pool2d], padding='SAME'):
+          net = slim.conv2d(image, 64, [5, 5], stride=1)
+          net = slim.conv2d(net, 64, [5, 5], stride=1)
+          net = slim.max_pool2d(net, [5, 5])
+          net = slim.conv2d(net, 64, [5, 5], stride=1)
+          net = slim.conv2d(net, 32, [5, 5], stride=1)
+          net = slim.max_pool2d(net, [2, 2])
+      sys.stderr.write('image_conv shape: %s\n' % net.get_shape())
+      feature_maps.append(net)
+    return feature_maps
+
+  def _CrossConvHelper(self, encoded_image, kernel):
+    """Cross Convolution.
+
+      The encoded image and kernel are of the same shape. Namely
+      [batch_size, image_size, image_size, channels]. They are split
+      into [image_size, image_size] image squares [kernel_size, kernel_size]
+      kernel squares. kernel squares are used to convolute image squares.
+    """
+    images = tf.expand_dims(encoded_image, 0)
+    kernels = tf.expand_dims(kernel, 3)
+    return tf.nn.depthwise_conv2d(images, kernels, [1, 1, 1, 1], 'SAME')
+
+  def _CrossConv(self, encoded_images):
+    """Apply the motion kernel on the encoded_images."""
+    cross_conved_images = []
+    kernels = tf.split(split_dim=3, num_split=4, value=self.kernel)
+    for (i, encoded_image) in enumerate(encoded_images):
+      with tf.variable_scope('cross_conv_%d' % i):
+        kernel = kernels[i]
+
+        encoded_image = tf.unstack(encoded_image, axis=0)
+        kernel = tf.unstack(kernel, axis=0)
+        assert len(encoded_image) == len(kernel)
+        assert len(encoded_image) == self.params['batch_size']
+        conved_image = []
+        for j in xrange(len(encoded_image)):
+          conved_image.append(self._CrossConvHelper(
+              encoded_image[j], kernel[j]))
+        cross_conved_images.append(tf.concat(0, conved_image))
+        sys.stderr.write('cross_conved shape: %s\n' %
+                         cross_conved_images[-1].get_shape())
+    return cross_conved_images
+
+  def _Deconv(self, net, out_filters, kernel_size, stride):
+    shape = net.get_shape().as_list()
+    in_filters = shape[3]
+    kernel_shape = [kernel_size, kernel_size, out_filters, in_filters]
+
+    weights = tf.get_variable(
+        name='weights',
+        shape=kernel_shape,
+        dtype=tf.float32,
+        initializer=tf.truncated_normal_initializer(stddev=0.01))
+
+
+    out_height = shape[1] * stride
+    out_width = shape[2] * stride
+    batch_size = shape[0]
+
+    output_shape = [batch_size, out_height, out_width, out_filters]
+    net = tf.nn.conv2d_transpose(net, weights, output_shape,
+                                 [1, stride, stride, 1], padding='SAME')
+    slim.batch_norm(net)
+    return net
+
+  def _BuildImageDecoder(self, cross_conved_images):
+    """Decode the cross_conved feature maps into the predicted images."""
+    nets = []
+    for i, cross_conved_image in enumerate(cross_conved_images):
+      with tf.variable_scope('image_decoder_%d' % i):
+        stride = 64 / cross_conved_image.get_shape().as_list()[1]
+        # TODO(xpan): Alternative solution for upsampling?
+        nets.append(self._Deconv(
+            cross_conved_image, 64, kernel_size=3, stride=stride))
+
+    net = tf.concat(3, nets)
+    net = slim.conv2d(net, 128, [9, 9], padding='SAME', stride=1)
+    net = slim.conv2d(net, 128, [1, 1], padding='SAME', stride=1)
+    net = slim.conv2d(net, 3, [1, 1], padding='SAME', stride=1)
+    self.diff_output = net
+    sys.stderr.write('diff_output shape: %s\n' % self.diff_output.get_shape())

next_frame_prediction/cross_conv/reader.py

@@ -0,0 +1,85 @@
+# Copyright 2016 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Read image sequence."""
+
+import tensorflow as tf
+
+
+def SequenceToImageAndDiff(images):
+  """Convert image sequence batch into image and diff batch.
+
+    Each image pair is converted to the first image and their diff.
+    Batch size will increase if sequence length is larger than 2.
+
+  Args:
+    images: Image sequence with shape
+        [batch_size, seq_len, image_size, image_size, channel]
+
+  Returns:
+    the list of (image, diff) tuples with shape
+        [batch_size2, image_size, image_size, channel]. image_sizes are
+        [32, 64, 128, 256].
+  """
+  image_diff_list = []
+  image_seq = tf.unstack(images, axis=1)
+  for size in [32, 64, 128, 256]:
+    resized_images = [
+        tf.image.resize_images(i, [size, size]) for i in image_seq]
+    diffs = []
+    for i in xrange(0, len(resized_images)-1):
+      diffs.append(resized_images[i+1] - resized_images[i])
+    image_diff_list.append(
+        (tf.concat(0, resized_images[:-1]), tf.concat(0, diffs)))
+  return image_diff_list
+
+
+def ReadInput(data_filepattern, shuffle, params):
+  """Read the tf.SequenceExample tfrecord files.
+
+  Args:
+    data_filepattern: tf.SequenceExample tfrecord filepattern.
+    shuffle: Whether to shuffle the examples.
+    params: parameter dict.
+
+  Returns:
+    image sequence batch [batch_size, seq_len, image_size, image_size, channel].
+  """
+  image_size = params['image_size']
+  filenames = tf.gfile.Glob(data_filepattern)
+  filename_queue = tf.train.string_input_producer(filenames, shuffle=shuffle)
+  reader = tf.TFRecordReader()
+  _, example = reader.read(filename_queue)
+  feature_sepc = {
+      'moving_objs': tf.FixedLenSequenceFeature(
+          shape=[image_size * image_size * 3], dtype=tf.float32)}
+  _, features = tf.parse_single_sequence_example(
+      example, sequence_features=feature_sepc)
+  moving_objs = tf.reshape(
+      features['moving_objs'], [params['seq_len'], image_size, image_size, 3])
+  if shuffle:
+    examples = tf.train.shuffle_batch(
+        [moving_objs],
+        batch_size=params['batch_size'],
+        num_threads=64,
+        capacity=params['batch_size'] * 100,
+        min_after_dequeue=params['batch_size'] * 4)
+  else:
+    examples = tf.train.batch([moving_objs],
+                              batch_size=params['batch_size'],
+                              num_threads=16,
+                              capacity=params['batch_size'])
+  examples /= params['norm_scale']
+  return examples

next_frame_prediction/cross_conv/sprites_gen.py

@@ -0,0 +1,97 @@
+# Copyright 2016 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Generate the sprites tfrecords from raw_images."""
+import os
+import random
+import re
+import sys
+
+import numpy as np
+import scipy.misc
+import tensorflow as tf
+
+
+tf.flags.DEFINE_string('data_filepattern', '', 'The raw images.')
+tf.flags.DEFINE_string('out_file', '',
+                       'File name for the tfrecord output.')
+
+
+def _read_images():
+  """Read images from image files into data structure."""
+  sprites = dict()
+  files = tf.gfile.Glob(tf.flags.FLAGS.data_filepattern)
+  for f in files:
+    image = scipy.misc.imread(f)
+    m = re.search('image_([0-9]+)_([0-9]+)_([0-9]+).jpg', os.path.basename(f))
+    if m.group(1) not in sprites:
+      sprites[m.group(1)] = dict()
+    character = sprites[m.group(1)]
+    if m.group(2) not in character:
+      character[m.group(2)] = dict()
+    pose = character[m.group(2)]
+    pose[int(m.group(3))] = image
+  return sprites
+
+
+def _images_to_example(image, image2):
+  """Convert 2 consecutive image to a SequenceExample."""
+  example = tf.SequenceExample()
+  feature_list = example.feature_lists.feature_list['moving_objs']
+  feature = feature_list.feature.add()
+  feature.float_list.value.extend(np.reshape(image, [-1]).tolist())
+  feature = feature_list.feature.add()
+  feature.float_list.value.extend(np.reshape(image2, [-1]).tolist())
+  return example
+
+
+def generate_input():
+  """Generate tfrecords."""
+  sprites = _read_images()
+  sys.stderr.write('Finish reading images.\n')
+  train_writer = tf.python_io.TFRecordWriter(
+      tf.flags.FLAGS.out_file.replace('sprites', 'sprites_train'))
+  test_writer = tf.python_io.TFRecordWriter(
+      tf.flags.FLAGS.out_file.replace('sprites', 'sprites_test'))
+
+  train_examples = []
+  test_examples = []
+  for i in sprites:
+    if int(i) < 24:
+      examples = test_examples
+    else:
+      examples = train_examples
+
+    character = sprites[i]
+    for j in character.keys():
+      pose = character[j]
+      for k in xrange(1, len(pose), 1):
+        image = pose[k]
+        image2 = pose[k+1]
+        examples.append(_images_to_example(image, image2))
+
+  sys.stderr.write('Finish generating examples: %d, %d.\n' %
+                   (len(train_examples), len(test_examples)))
+  random.shuffle(train_examples)
+  _ = [train_writer.write(ex.SerializeToString()) for ex in train_examples]
+  _ = [test_writer.write(ex.SerializeToString()) for ex in test_examples]
+
+
+def main(_):
+  generate_input()
+
+
+if __name__ == '__main__':
+  tf.app.run()

next_frame_prediction/cross_conv/train.py

@@ -0,0 +1,122 @@
+# Copyright 2016 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Train the cross convolutional model."""
+import os
+import sys
+
+import numpy as np
+import tensorflow as tf
+
+import model as cross_conv_model
+import reader
+
+FLAGS = tf.flags.FLAGS
+tf.flags.DEFINE_string('master', '', 'Session address.')
+tf.flags.DEFINE_string('log_root', '/tmp/moving_obj', 'The root dir of output.')
+tf.flags.DEFINE_string('data_filepattern', '',
+                       'training data file pattern.')
+tf.flags.DEFINE_integer('image_size', 64, 'Image height and width.')
+tf.flags.DEFINE_integer('batch_size', 1, 'Batch size.')
+tf.flags.DEFINE_float('norm_scale', 1.0, 'Normalize the original image')
+tf.flags.DEFINE_float('scale', 10.0,
+                      'Scale the image after norm_scale and move the diff '
+                      'to the positive realm.')
+tf.flags.DEFINE_integer('sequence_length', 2, 'tf.SequenceExample length.')
+tf.flags.DEFINE_float('learning_rate', 0.8, 'Learning rate.')
+tf.flags.DEFINE_bool('l2_loss', True, 'If true, include l2_loss.')
+tf.flags.DEFINE_bool('reconstr_loss', False, 'If true, include reconstr_loss.')
+tf.flags.DEFINE_bool('kl_loss', True, 'If true, include KL loss.')
+
+slim = tf.contrib.slim
+
+
+def _Train():
+  params = dict()
+  params['batch_size'] = FLAGS.batch_size
+  params['seq_len'] = FLAGS.sequence_length
+  params['image_size'] = FLAGS.image_size
+  params['is_training'] = True
+  params['norm_scale'] = FLAGS.norm_scale
+  params['scale'] = FLAGS.scale
+  params['learning_rate'] = FLAGS.learning_rate
+  params['l2_loss'] = FLAGS.l2_loss
+  params['reconstr_loss'] = FLAGS.reconstr_loss
+  params['kl_loss'] = FLAGS.kl_loss
+
+  train_dir = os.path.join(FLAGS.log_root, 'train')
+
+  images = reader.ReadInput(FLAGS.data_filepattern, shuffle=True, params=params)
+  images *= params['scale']
+  # Increase the value makes training much faster.
+  image_diff_list = reader.SequenceToImageAndDiff(images)
+  model = cross_conv_model.CrossConvModel(image_diff_list, params)
+  model.Build()
+  tf.contrib.tfprof.model_analyzer.print_model_analysis(tf.get_default_graph())
+
+  summary_writer = tf.summary.FileWriter(train_dir)
+  sv = tf.train.Supervisor(logdir=FLAGS.log_root,
+                           summary_op=None,
+                           is_chief=True,
+                           save_model_secs=60,
+                           global_step=model.global_step)
+  sess = sv.prepare_or_wait_for_session(
+      FLAGS.master, config=tf.ConfigProto(allow_soft_placement=True))
+
+  total_loss = 0.0
+  step = 0
+  sample_z_mean = np.zeros(model.z_mean.get_shape().as_list())
+  sample_z_stddev_log = np.zeros(model.z_stddev_log.get_shape().as_list())
+  sample_step = 0
+
+  while True:
+    _, loss_val, total_steps, summaries, z_mean, z_stddev_log = sess.run(
+        [model.train_op, model.loss, model.global_step,
+         model.summary_op,
+         model.z_mean, model.z_stddev_log])
+
+    sample_z_mean += z_mean
+    sample_z_stddev_log += z_stddev_log
+    total_loss += loss_val
+    step += 1
+    sample_step += 1
+
+    if step % 100 == 0:
+      summary_writer.add_summary(summaries, total_steps)
+      sys.stderr.write('step: %d, loss: %f\n' %
+                       (total_steps, total_loss / step))
+      total_loss = 0.0
+      step = 0
+
+    # Sampled z is used for eval.
+    # It seems 10k is better than 1k. Maybe try 100k next?
+    if sample_step % 10000 == 0:
+      with tf.gfile.Open(os.path.join(FLAGS.log_root, 'z_mean.npy'), 'w') as f:
+        np.save(f, sample_z_mean / sample_step)
+      with tf.gfile.Open(
+          os.path.join(FLAGS.log_root, 'z_stddev_log.npy'), 'w') as f:
+        np.save(f, sample_z_stddev_log / sample_step)
+      sample_z_mean = np.zeros(model.z_mean.get_shape().as_list())
+      sample_z_stddev_log = np.zeros(
+          model.z_stddev_log.get_shape().as_list())
+      sample_step = 0
+
+
+def main(_):
+  _Train()
+
+
+if __name__ == '__main__':
+  tf.app.run()

real_nvp/README.md

@@ -0,0 +1,278 @@
+# Real NVP in TensorFlow
+
+*A Tensorflow implementation of the training procedure of*
+[*Density estimation using Real NVP*](https://arxiv.org/abs/1605.08803)*, by
+Laurent Dinh, Jascha Sohl-Dickstein and Samy Bengio, for Imagenet
+(32x32 and 64x64), CelebA and LSUN Including the scripts to
+put the datasets in `.tfrecords` format.*
+
+We are happy to open source the code for *Real NVP*, a novel approach to
+density estimation using deep neural networks that enables tractable density
+estimation and efficient one-pass inference and sampling. This model
+successfully decomposes images into hierarchical features ranging from
+high-level concepts to low-resolution details. Visualizations are available
+[here](http://goo.gl/yco14s).
+
+## Installation
+*   python 2.7:
+    * python 3 support is not available yet
+*   pip (python package manager)
+    * `apt-get install python-pip` on Ubuntu
+    * `brew` installs pip along with python on OSX
+*   Install the dependencies for [LSUN](https://github.com/fyu/lsun.git)
+    * Install [OpenCV](http://opencv.org/)
+    * `pip install numpy lmdb`
+*   Install the python dependencies
+    * `pip install scipy scikit-image Pillow`
+*   Install the
+[latest Tensorflow Pip package](https://www.tensorflow.org/get_started/os_setup.html#using-pip)
+for Python 2.7
+
+## Getting Started
+Once you have successfully installed the dependencies, you can start by
+downloading the repository:
+```shell
+git clone --recursive https://github.com/tensorflow/models.git
+```
+Afterward, you can use the utilities in this folder prepare the datasets.
+
+## Preparing datasets
+### CelebA
+For [*CelebA*](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html), download
+`img_align_celeba.zip` from the Dropbox link on this
+[page](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) under the
+link *Align&Cropped Images* in the *Img* directory and `list_eval_partition.txt`
+under the link *Train/Val/Test Partitions* in the *Eval* directory. Then do:
+
+```shell
+mkdir celeba
+cd celeba
+unzip img_align_celeba.zip
+```
+
+We'll format the training subset:
+```shell
+python2.7 ../models/real_nvp/celeba_formatting.py \
+    --partition_fn list_eval_partition.txt \
+    --file_out celeba_train \
+    --fn_root img_align_celeba \
+    --set 0
+```
+
+Then the validation subset:
+```shell
+python2.7 ../models/real_nvp/celeba_formatting.py \
+    --partition_fn list_eval_partition.txt \
+    --file_out celeba_valid \
+    --fn_root img_align_celeba \
+    --set 1
+```
+
+And finally the test subset:
+```shell
+python2.7 ../models/real_nvp/celeba_formatting.py \
+    --partition_fn list_eval_partition.txt \
+    --file_out celeba_test \
+    --fn_root img_align_celeba \
+    --set 2
+```
+
+Afterward:
+```shell
+cd ..
+```
+
+### Small Imagenet
+Downloading the [*small Imagenet*](http://image-net.org/small/download.php)
+dataset is more straightforward and can be done
+entirely in Shell:
+```shell
+mkdir small_imnet
+cd small_imnet
+for FILENAME in train_32x32.tar valid_32x32.tar train_64x64.tar valid_64x64.tar
+do
+    curl -O http://image-net.org/small/$FILENAME
+    tar -xvf $FILENAME
+done
+```
+
+Then, you can format the datasets as follow:
+```shell
+for DIRNAME in train_32x32 valid_32x32 train_64x64 valid_64x64
+do
+    python2.7 ../models/real_nvp/imnet_formatting.py \
+        --file_out $DIRNAME \
+        --fn_root $DIRNAME
+done
+cd ..
+```
+
+### LSUN
+To prepare the [*LSUN*](http://lsun.cs.princeton.edu/2016/) dataset, we will
+need to use the code associated:
+```shell
+git clone https://github.com/fyu/lsun.git
+cd lsun
+```
+Then we'll download the db files:
+```shell
+for CATEGORY in bedroom church_outdoor tower
+do
+    python2.7 download.py -c $CATEGORY
+    unzip "$CATEGORY"_train_lmdb.zip
+    unzip "$CATEGORY"_val_lmdb.zip
+    python2.7 data.py export "$CATEGORY"_train_lmdb \
+        --out_dir "$CATEGORY"_train --flat
+    python2.7 data.py export "$CATEGORY"_val_lmdb \
+        --out_dir "$CATEGORY"_val --flat
+done
+```
+
+Finally, we then format the dataset into `.tfrecords`:
+```shell
+for CATEGORY in bedroom church_outdoor tower
+do
+    python2.7 ../models/real_nvp/lsun_formatting.py \
+        --file_out "$CATEGORY"_train \
+        --fn_root "$CATEGORY"_train
+    python2.7 ../models/real_nvp/lsun_formatting.py \
+        --file_out "$CATEGORY"_val \
+        --fn_root "$CATEGORY"_val
+done
+cd ..
+```
+
+
+## Training
+We'll give an example on how to train a model on the small Imagenet
+dataset (32x32):
+```shell
+cd models/real_nvp/
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 32 \
+--hpconfig=n_scale=4,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset imnet \
+--traindir /tmp/real_nvp_imnet32/train \
+--logdir /tmp/real_nvp_imnet32/train \
+--data_path ../../small_imnet/train_32x32_?????.tfrecords
+```
+In parallel, you can run the script to generate visualization from the model:
+```shell
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 32 \
+--hpconfig=n_scale=4,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset imnet \
+--traindir /tmp/real_nvp_imnet32/train \
+--logdir /tmp/real_nvp_imnet32/sample \
+--data_path ../../small_imnet/valid_32x32_?????.tfrecords \
+--mode sample
+```
+Additionally, you can also run in the script to evaluate the model on the
+validation set:
+```shell
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 32 \
+--hpconfig=n_scale=4,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset imnet \
+--traindir /tmp/real_nvp_imnet32/train \
+--logdir /tmp/real_nvp_imnet32/eval \
+--data_path ../../small_imnet/valid_32x32_?????.tfrecords \
+--eval_set_size 50000
+--mode eval
+```
+The visualizations and validation set evaluation can be seen through
+[Tensorboard](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tensorboard/README.md).
+
+Another example would be how to run the model on LSUN (bedroom category):
+```shell
+# train the model
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset lsun \
+--traindir /tmp/real_nvp_church_outdoor/train \
+--logdir /tmp/real_nvp_church_outdoor/train \
+--data_path ../../lsun/church_outdoor_train_?????.tfrecords
+```
+
+```shell
+# sample from the model
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset lsun \
+--traindir /tmp/real_nvp_church_outdoor/train \
+--logdir /tmp/real_nvp_church_outdoor/sample \
+--data_path ../../lsun/church_outdoor_val_?????.tfrecords \
+--mode sample
+```
+
+```shell
+# evaluate the model
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset lsun \
+--traindir /tmp/real_nvp_church_outdoor/train \
+--logdir /tmp/real_nvp_church_outdoor/eval \
+--data_path ../../lsun/church_outdoor_val_?????.tfrecords \
+--eval_set_size 300
+--mode eval
+```
+
+Finally, we'll give the commands to run the model on the CelebA dataset:
+```shell
+# train the model
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset lsun \
+--traindir /tmp/real_nvp_celeba/train \
+--logdir /tmp/real_nvp_celeba/train \
+--data_path ../../celeba/celeba_train.tfrecords
+```
+
+```shell
+# sample from the model
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset celeba \
+--traindir /tmp/real_nvp_celeba/train \
+--logdir /tmp/real_nvp_celeba/sample \
+--data_path ../../celeba/celeba_valid.tfrecords \
+--mode sample
+```
+
+```shell
+# evaluate the model on validation set
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset celeba \
+--traindir /tmp/real_nvp_celeba/train \
+--logdir /tmp/real_nvp_celeba/eval_valid \
+--data_path ../../celeba/celeba_valid.tfrecords \
+--eval_set_size 19867
+--mode eval
+
+# evaluate the model on test set
+python2.7 real_nvp_multiscale_dataset.py \
+--image_size 64 \
+--hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
+--dataset celeba \
+--traindir /tmp/real_nvp_celeba/train \
+--logdir /tmp/real_nvp_celeba/eval_test \
+--data_path ../../celeba/celeba_test.tfrecords \
+--eval_set_size 19962
+--mode eval
+```
+
+## Credits
+This code was written by Laurent Dinh
+([@laurent-dinh](https://github.com/laurent-dinh)) with
+the help of
+Jascha Sohl-Dickstein ([@Sohl-Dickstein](https://github.com/Sohl-Dickstein)
+and [jaschasd@google.com](mailto:jaschasd@google.com)),
+Samy Bengio, Jon Shlens, Sherry Moore and
+David Andersen.

real_nvp/celeba_formatting.py

@@ -0,0 +1,94 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""CelebA dataset formating.
+
+Download img_align_celeba.zip from
+http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html under the
+link "Align&Cropped Images" in the "Img" directory and list_eval_partition.txt
+under the link "Train/Val/Test Partitions" in the "Eval" directory. Then do:
+unzip img_align_celeba.zip
+
+Use the script as follow:
+python celeba_formatting.py \
+    --partition_fn [PARTITION_FILE_PATH] \
+    --file_out [OUTPUT_FILE_PATH_PREFIX] \
+    --fn_root [CELEBA_FOLDER] \
+    --set [SUBSET_INDEX]
+
+"""
+
+import os
+import os.path
+
+import scipy.io
+import scipy.io.wavfile
+import scipy.ndimage
+import tensorflow as tf
+
+
+tf.flags.DEFINE_string("file_out", "",
+                       "Filename of the output .tfrecords file.")
+tf.flags.DEFINE_string("fn_root", "", "Name of root file path.")
+tf.flags.DEFINE_string("partition_fn", "", "Partition file path.")
+tf.flags.DEFINE_string("set", "", "Name of subset.")
+
+FLAGS = tf.flags.FLAGS
+
+
+def _int64_feature(value):
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+
+def _bytes_feature(value):
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+
+def main():
+    """Main converter function."""
+    # Celeb A
+    with open(FLAGS.partition_fn, "r") as infile:
+        img_fn_list = infile.readlines()
+    img_fn_list = [elem.strip().split() for elem in img_fn_list]
+    img_fn_list = [elem[0] for elem in img_fn_list if elem[1] == FLAGS.set]
+    fn_root = FLAGS.fn_root
+    num_examples = len(img_fn_list)
+
+    file_out = "%s.tfrecords" % FLAGS.file_out
+    writer = tf.python_io.TFRecordWriter(file_out)
+    for example_idx, img_fn in enumerate(img_fn_list):
+        if example_idx % 1000 == 0:
+            print example_idx, "/", num_examples
+        image_raw = scipy.ndimage.imread(os.path.join(fn_root, img_fn))
+        rows = image_raw.shape[0]
+        cols = image_raw.shape[1]
+        depth = image_raw.shape[2]
+        image_raw = image_raw.tostring()
+        example = tf.train.Example(
+            features=tf.train.Features(
+                feature={
+                    "height": _int64_feature(rows),
+                    "width": _int64_feature(cols),
+                    "depth": _int64_feature(depth),
+                    "image_raw": _bytes_feature(image_raw)
+                }
+            )
+        )
+        writer.write(example.SerializeToString())
+    writer.close()
+
+
+if __name__ == "__main__":
+    main()

real_nvp/imnet_formatting.py

@@ -0,0 +1,103 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""LSUN dataset formatting.
+
+Download and format the Imagenet dataset as follow:
+mkdir [IMAGENET_PATH]
+cd [IMAGENET_PATH]
+for FILENAME in train_32x32.tar valid_32x32.tar train_64x64.tar valid_64x64.tar
+do
+    curl -O http://image-net.org/small/$FILENAME
+    tar -xvf $FILENAME
+done
+
+Then use the script as follow:
+for DIRNAME in train_32x32 valid_32x32 train_64x64 valid_64x64
+do
+    python imnet_formatting.py \
+        --file_out $DIRNAME \
+        --fn_root $DIRNAME
+done
+
+"""
+
+import os
+import os.path
+
+import scipy.io
+import scipy.io.wavfile
+import scipy.ndimage
+import tensorflow as tf
+
+
+tf.flags.DEFINE_string("file_out", "",
+                       "Filename of the output .tfrecords file.")
+tf.flags.DEFINE_string("fn_root", "", "Name of root file path.")
+
+FLAGS = tf.flags.FLAGS
+
+
+def _int64_feature(value):
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+
+def _bytes_feature(value):
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+
+def main():
+    """Main converter function."""
+    # LSUN
+    fn_root = FLAGS.fn_root
+    img_fn_list = os.listdir(fn_root)
+    img_fn_list = [img_fn for img_fn in img_fn_list
+                   if img_fn.endswith('.png')]
+    num_examples = len(img_fn_list)
+
+    n_examples_per_file = 10000
+    for example_idx, img_fn in enumerate(img_fn_list):
+        if example_idx % n_examples_per_file == 0:
+            file_out = "%s_%05d.tfrecords"
+            file_out = file_out % (FLAGS.file_out,
+                                   example_idx // n_examples_per_file)
+            print "Writing on:", file_out
+            writer = tf.python_io.TFRecordWriter(file_out)
+        if example_idx % 1000 == 0:
+            print example_idx, "/", num_examples
+        image_raw = scipy.ndimage.imread(os.path.join(fn_root, img_fn))
+        rows = image_raw.shape[0]
+        cols = image_raw.shape[1]
+        depth = image_raw.shape[2]
+        image_raw = image_raw.astype("uint8")
+        image_raw = image_raw.tostring()
+        example = tf.train.Example(
+            features=tf.train.Features(
+                feature={
+                    "height": _int64_feature(rows),
+                    "width": _int64_feature(cols),
+                    "depth": _int64_feature(depth),
+                    "image_raw": _bytes_feature(image_raw)
+                }
+            )
+        )
+        writer.write(example.SerializeToString())
+        if example_idx % n_examples_per_file == (n_examples_per_file - 1):
+            writer.close()
+    writer.close()
+
+
+if __name__ == "__main__":
+    main()

real_nvp/lsun_formatting.py

@@ -0,0 +1,104 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""LSUN dataset formatting.
+
+Download and format the LSUN dataset as follow:
+git clone https://github.com/fyu/lsun.git
+cd lsun
+python2.7 download.py -c [CATEGORY]
+
+Then unzip the downloaded .zip files before executing:
+python2.7 data.py export [IMAGE_DB_PATH] --out_dir [LSUN_FOLDER] --flat
+
+Then use the script as follow:
+python lsun_formatting.py \
+    --file_out [OUTPUT_FILE_PATH_PREFIX] \
+    --fn_root [LSUN_FOLDER]
+
+"""
+
+import os
+import os.path
+
+import numpy
+import skimage.transform
+from PIL import Image
+import tensorflow as tf
+
+
+tf.flags.DEFINE_string("file_out", "",
+                       "Filename of the output .tfrecords file.")
+tf.flags.DEFINE_string("fn_root", "", "Name of root file path.")
+
+FLAGS = tf.flags.FLAGS
+
+
+def _int64_feature(value):
+    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+
+def _bytes_feature(value):
+    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+
+def main():
+    """Main converter function."""
+    fn_root = FLAGS.fn_root
+    img_fn_list = os.listdir(fn_root)
+    img_fn_list = [img_fn for img_fn in img_fn_list
+                   if img_fn.endswith('.webp')]
+    num_examples = len(img_fn_list)
+
+    n_examples_per_file = 10000
+    for example_idx, img_fn in enumerate(img_fn_list):
+        if example_idx % n_examples_per_file == 0:
+            file_out = "%s_%05d.tfrecords"
+            file_out = file_out % (FLAGS.file_out,
+                                   example_idx // n_examples_per_file)
+            print "Writing on:", file_out
+            writer = tf.python_io.TFRecordWriter(file_out)
+        if example_idx % 1000 == 0:
+            print example_idx, "/", num_examples
+        image_raw = numpy.array(Image.open(os.path.join(fn_root, img_fn)))
+        rows = image_raw.shape[0]
+        cols = image_raw.shape[1]
+        depth = image_raw.shape[2]
+        downscale = min(rows / 96., cols / 96.)
+        image_raw = skimage.transform.pyramid_reduce(image_raw, downscale)
+        image_raw *= 255.
+        image_raw = image_raw.astype("uint8")
+        rows = image_raw.shape[0]
+        cols = image_raw.shape[1]
+        depth = image_raw.shape[2]
+        image_raw = image_raw.tostring()
+        example = tf.train.Example(
+            features=tf.train.Features(
+                feature={
+                    "height": _int64_feature(rows),
+                    "width": _int64_feature(cols),
+                    "depth": _int64_feature(depth),
+                    "image_raw": _bytes_feature(image_raw)
+                }
+            )
+        )
+        writer.write(example.SerializeToString())
+        if example_idx % n_examples_per_file == (n_examples_per_file - 1):
+            writer.close()
+    writer.close()
+
+
+if __name__ == "__main__":
+    main()

real_nvp/real_nvp_utils.py

@@ -0,0 +1,474 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""Utility functions for Real NVP.
+"""
+
+# pylint: disable=dangerous-default-value
+
+import numpy
+import tensorflow as tf
+from tensorflow.python.framework import ops
+
+DEFAULT_BN_LAG = .0
+
+
+def stable_var(input_, mean=None, axes=[0]):
+    """Numerically more stable variance computation."""
+    if mean is None:
+        mean = tf.reduce_mean(input_, axes)
+    res = tf.square(input_ - mean)
+    max_sqr = tf.reduce_max(res, axes)
+    res /= max_sqr
+    res = tf.reduce_mean(res, axes)
+    res *= max_sqr
+
+    return res
+
+
+def variable_on_cpu(name, shape, initializer, trainable=True):
+    """Helper to create a Variable stored on CPU memory.
+
+    Args:
+            name: name of the variable
+            shape: list of ints
+            initializer: initializer for Variable
+            trainable: boolean defining if the variable is for training
+    Returns:
+            Variable Tensor
+    """
+    var = tf.get_variable(
+        name, shape, initializer=initializer, trainable=trainable)
+    return var
+
+
+# layers
+def conv_layer(input_,
+               filter_size,
+               dim_in,
+               dim_out,
+               name,
+               stddev=1e-2,
+               strides=[1, 1, 1, 1],
+               padding="SAME",
+               nonlinearity=None,
+               bias=False,
+               weight_norm=False,
+               scale=False):
+    """Convolutional layer."""
+    with tf.variable_scope(name) as scope:
+        weights = variable_on_cpu(
+            "weights",
+            filter_size + [dim_in, dim_out],
+            tf.random_uniform_initializer(
+                minval=-stddev, maxval=stddev))
+        # weight normalization
+        if weight_norm:
+            weights /= tf.sqrt(tf.reduce_sum(tf.square(weights), [0, 1, 2]))
+            if scale:
+                magnitude = variable_on_cpu(
+                    "magnitude", [dim_out],
+                    tf.constant_initializer(
+                        stddev * numpy.sqrt(dim_in * numpy.prod(filter_size) / 12.)))
+                weights *= magnitude
+        res = input_
+        # handling filter size bigger than image size
+        if hasattr(input_, "shape"):
+            if input_.get_shape().as_list()[1] < filter_size[0]:
+                pad_1 = tf.zeros([
+                    input_.get_shape().as_list()[0],
+                    filter_size[0] - input_.get_shape().as_list()[1],
+                    input_.get_shape().as_list()[2],
+                    input_.get_shape().as_list()[3]
+                ])
+                pad_2 = tf.zeros([
+                    input_.get_shape().as_list[0],
+                    filter_size[0],
+                    filter_size[1] - input_.get_shape().as_list()[2],
+                    input_.get_shape().as_list()[3]
+                ])
+                res = tf.concat(1, [pad_1, res])
+                res = tf.concat(2, [pad_2, res])
+        res = tf.nn.conv2d(
+            input=res,
+            filter=weights,
+            strides=strides,
+            padding=padding,
+            name=scope.name)
+
+        if hasattr(input_, "shape"):
+            if input_.get_shape().as_list()[1] < filter_size[0]:
+                res = tf.slice(res, [
+                    0, filter_size[0] - input_.get_shape().as_list()[1],
+                    filter_size[1] - input_.get_shape().as_list()[2], 0
+                ], [-1, -1, -1, -1])
+
+        if bias:
+            biases = variable_on_cpu("biases", [dim_out], tf.constant_initializer(0.))
+            res = tf.nn.bias_add(res, biases)
+        if nonlinearity is not None:
+            res = nonlinearity(res)
+
+    return res
+
+
+def max_pool_2x2(input_):
+    """Max pooling."""
+    return tf.nn.max_pool(
+        input_, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME")
+
+
+def depool_2x2(input_, stride=2):
+    """Depooling."""
+    shape = input_.get_shape().as_list()
+    batch_size = shape[0]
+    height = shape[1]
+    width = shape[2]
+    channels = shape[3]
+    res = tf.reshape(input_, [batch_size, height, 1, width, 1, channels])
+    res = tf.concat(
+        2, [res, tf.zeros([batch_size, height, stride - 1, width, 1, channels])])
+    res = tf.concat(4, [
+        res, tf.zeros([batch_size, height, stride, width, stride - 1, channels])
+    ])
+    res = tf.reshape(res, [batch_size, stride * height, stride * width, channels])
+
+    return res
+
+
+# random flip on a batch of images
+def batch_random_flip(input_):
+    """Simultaneous horizontal random flip."""
+    if isinstance(input_, (float, int)):
+        return input_
+    shape = input_.get_shape().as_list()
+    batch_size = shape[0]
+    height = shape[1]
+    width = shape[2]
+    channels = shape[3]
+    res = tf.split(0, batch_size, input_)
+    res = [elem[0, :, :, :] for elem in res]
+    res = [tf.image.random_flip_left_right(elem) for elem in res]
+    res = [tf.reshape(elem, [1, height, width, channels]) for elem in res]
+    res = tf.concat(0, res)
+
+    return res
+
+
+# build a one hot representation corresponding to the integer tensor
+# the one-hot dimension is appended to the integer tensor shape
+def as_one_hot(input_, n_indices):
+    """Convert indices to one-hot."""
+    shape = input_.get_shape().as_list()
+    n_elem = numpy.prod(shape)
+    indices = tf.range(n_elem)
+    indices = tf.cast(indices, tf.int64)
+    indices_input = tf.concat(0, [indices, tf.reshape(input_, [-1])])
+    indices_input = tf.reshape(indices_input, [2, -1])
+    indices_input = tf.transpose(indices_input)
+    res = tf.sparse_to_dense(
+        indices_input, [n_elem, n_indices], 1., 0., name="flat_one_hot")
+    res = tf.reshape(res, [elem for elem in shape] + [n_indices])
+
+    return res
+
+
+def squeeze_2x2(input_):
+    """Squeezing operation: reshape to convert space to channels."""
+    return squeeze_nxn(input_, n_factor=2)
+
+
+def squeeze_nxn(input_, n_factor=2):
+    """Squeezing operation: reshape to convert space to channels."""
+    if isinstance(input_, (float, int)):
+        return input_
+    shape = input_.get_shape().as_list()
+    batch_size = shape[0]
+    height = shape[1]
+    width = shape[2]
+    channels = shape[3]
+    if height % n_factor != 0:
+        raise ValueError("Height not divisible by %d." % n_factor)
+    if width % n_factor != 0:
+        raise ValueError("Width not divisible by %d." % n_factor)
+    res = tf.reshape(
+        input_,
+        [batch_size,
+         height // n_factor,
+         n_factor, width // n_factor,
+         n_factor, channels])
+    res = tf.transpose(res, [0, 1, 3, 5, 2, 4])
+    res = tf.reshape(
+        res,
+        [batch_size,
+         height // n_factor,
+         width // n_factor,
+         channels * n_factor * n_factor])
+
+    return res
+
+
+def unsqueeze_2x2(input_):
+    """Unsqueezing operation: reshape to convert channels into space."""
+    if isinstance(input_, (float, int)):
+        return input_
+    shape = input_.get_shape().as_list()
+    batch_size = shape[0]
+    height = shape[1]
+    width = shape[2]
+    channels = shape[3]
+    if channels % 4 != 0:
+        raise ValueError("Number of channels not divisible by 4.")
+    res = tf.reshape(input_, [batch_size, height, width, channels // 4, 2, 2])
+    res = tf.transpose(res, [0, 1, 4, 2, 5, 3])
+    res = tf.reshape(res, [batch_size, 2 * height, 2 * width, channels // 4])
+
+    return res
+
+
+# batch norm
+def batch_norm(input_,
+               dim,
+               name,
+               scale=True,
+               train=True,
+               epsilon=1e-8,
+               decay=.1,
+               axes=[0],
+               bn_lag=DEFAULT_BN_LAG):
+    """Batch normalization."""
+    # create variables
+    with tf.variable_scope(name):
+        var = variable_on_cpu(
+            "var", [dim], tf.constant_initializer(1.), trainable=False)
+        mean = variable_on_cpu(
+            "mean", [dim], tf.constant_initializer(0.), trainable=False)
+        step = variable_on_cpu("step", [], tf.constant_initializer(0.), trainable=False)
+        if scale:
+            gamma = variable_on_cpu("gamma", [dim], tf.constant_initializer(1.))
+        beta = variable_on_cpu("beta", [dim], tf.constant_initializer(0.))
+    # choose the appropriate moments
+    if train:
+        used_mean, used_var = tf.nn.moments(input_, axes, name="batch_norm")
+        cur_mean, cur_var = used_mean, used_var
+        if bn_lag > 0.:
+            used_mean -= (1. - bn_lag) * (used_mean - tf.stop_gradient(mean))
+            used_var -= (1 - bn_lag) * (used_var - tf.stop_gradient(var))
+            used_mean /= (1. - bn_lag**(step + 1))
+            used_var /= (1. - bn_lag**(step + 1))
+    else:
+        used_mean, used_var = mean, var
+        cur_mean, cur_var = used_mean, used_var
+
+    # normalize
+    res = (input_ - used_mean) / tf.sqrt(used_var + epsilon)
+    # de-normalize
+    if scale:
+        res *= gamma
+    res += beta
+
+    # update variables
+    if train:
+        with tf.name_scope(name, "AssignMovingAvg", [mean, cur_mean, decay]):
+            with ops.colocate_with(mean):
+                new_mean = tf.assign_sub(
+                    mean,
+                    tf.check_numerics(decay * (mean - cur_mean), "NaN in moving mean."))
+        with tf.name_scope(name, "AssignMovingAvg", [var, cur_var, decay]):
+            with ops.colocate_with(var):
+                new_var = tf.assign_sub(
+                    var,
+                    tf.check_numerics(decay * (var - cur_var),
+                                      "NaN in moving variance."))
+        with tf.name_scope(name, "IncrementTime", [step]):
+            with ops.colocate_with(step):
+                new_step = tf.assign_add(step, 1.)
+        res += 0. * new_mean * new_var * new_step
+
+    return res
+
+
+# batch normalization taking into account the volume transformation
+def batch_norm_log_diff(input_,
+                        dim,
+                        name,
+                        train=True,
+                        epsilon=1e-8,
+                        decay=.1,
+                        axes=[0],
+                        reuse=None,
+                        bn_lag=DEFAULT_BN_LAG):
+    """Batch normalization with corresponding log determinant Jacobian."""
+    if reuse is None:
+        reuse = not train
+    # create variables
+    with tf.variable_scope(name) as scope:
+        if reuse:
+            scope.reuse_variables()
+        var = variable_on_cpu(
+            "var", [dim], tf.constant_initializer(1.), trainable=False)
+        mean = variable_on_cpu(
+            "mean", [dim], tf.constant_initializer(0.), trainable=False)
+        step = variable_on_cpu("step", [], tf.constant_initializer(0.), trainable=False)
+    # choose the appropriate moments
+    if train:
+        used_mean, used_var = tf.nn.moments(input_, axes, name="batch_norm")
+        cur_mean, cur_var = used_mean, used_var
+        if bn_lag > 0.:
+            used_var = stable_var(input_=input_, mean=used_mean, axes=axes)
+            cur_var = used_var
+            used_mean -= (1 - bn_lag) * (used_mean - tf.stop_gradient(mean))
+            used_mean /= (1. - bn_lag**(step + 1))
+            used_var -= (1 - bn_lag) * (used_var - tf.stop_gradient(var))
+            used_var /= (1. - bn_lag**(step + 1))
+    else:
+        used_mean, used_var = mean, var
+        cur_mean, cur_var = used_mean, used_var
+
+    # update variables
+    if train:
+        with tf.name_scope(name, "AssignMovingAvg", [mean, cur_mean, decay]):
+            with ops.colocate_with(mean):
+                new_mean = tf.assign_sub(
+                    mean,
+                    tf.check_numerics(
+                        decay * (mean - cur_mean), "NaN in moving mean."))
+        with tf.name_scope(name, "AssignMovingAvg", [var, cur_var, decay]):
+            with ops.colocate_with(var):
+                new_var = tf.assign_sub(
+                    var,
+                    tf.check_numerics(decay * (var - cur_var),
+                                      "NaN in moving variance."))
+        with tf.name_scope(name, "IncrementTime", [step]):
+            with ops.colocate_with(step):
+                new_step = tf.assign_add(step, 1.)
+        used_var += 0. * new_mean * new_var * new_step
+    used_var += epsilon
+
+    return used_mean, used_var
+
+
+def convnet(input_,
+            dim_in,
+            dim_hid,
+            filter_sizes,
+            dim_out,
+            name,
+            use_batch_norm=True,
+            train=True,
+            nonlinearity=tf.nn.relu):
+    """Chaining of convolutional layers."""
+    dims_in = [dim_in] + dim_hid[:-1]
+    dims_out = dim_hid
+    res = input_
+
+    bias = (not use_batch_norm)
+    with tf.variable_scope(name):
+        for layer_idx in xrange(len(dim_hid)):
+            res = conv_layer(
+                input_=res,
+                filter_size=filter_sizes[layer_idx],
+                dim_in=dims_in[layer_idx],
+                dim_out=dims_out[layer_idx],
+                name="h_%d" % layer_idx,
+                stddev=1e-2,
+                nonlinearity=None,
+                bias=bias)
+            if use_batch_norm:
+                res = batch_norm(
+                    input_=res,
+                    dim=dims_out[layer_idx],
+                    name="bn_%d" % layer_idx,
+                    scale=(nonlinearity == tf.nn.relu),
+                    train=train,
+                    epsilon=1e-8,
+                    axes=[0, 1, 2])
+            if nonlinearity is not None:
+                res = nonlinearity(res)
+
+        res = conv_layer(
+            input_=res,
+            filter_size=filter_sizes[-1],
+            dim_in=dims_out[-1],
+            dim_out=dim_out,
+            name="out",
+            stddev=1e-2,
+            nonlinearity=None)
+
+    return res
+
+
+# distributions
+# log-likelihood estimation
+def standard_normal_ll(input_):
+    """Log-likelihood of standard Gaussian distribution."""
+    res = -.5 * (tf.square(input_) + numpy.log(2. * numpy.pi))
+
+    return res
+
+
+def standard_normal_sample(shape):
+    """Samples from standard Gaussian distribution."""
+    return tf.random_normal(shape)
+
+
+SQUEEZE_MATRIX = numpy.array([[[[1., 0., 0., 0.]], [[0., 0., 1., 0.]]],
+                              [[[0., 0., 0., 1.]], [[0., 1., 0., 0.]]]])
+
+
+def squeeze_2x2_ordered(input_, reverse=False):
+    """Squeezing operation with a controlled ordering."""
+    shape = input_.get_shape().as_list()
+    batch_size = shape[0]
+    height = shape[1]
+    width = shape[2]
+    channels = shape[3]
+    if reverse:
+        if channels % 4 != 0:
+            raise ValueError("Number of channels not divisible by 4.")
+        channels /= 4
+    else:
+        if height % 2 != 0:
+            raise ValueError("Height not divisible by 2.")
+        if width % 2 != 0:
+            raise ValueError("Width not divisible by 2.")
+    weights = numpy.zeros((2, 2, channels, 4 * channels))
+    for idx_ch in xrange(channels):
+        slice_2 = slice(idx_ch, (idx_ch + 1))
+        slice_3 = slice((idx_ch * 4), ((idx_ch + 1) * 4))
+        weights[:, :, slice_2, slice_3] = SQUEEZE_MATRIX
+    shuffle_channels = [idx_ch * 4 for idx_ch in xrange(channels)]
+    shuffle_channels += [idx_ch * 4 + 1 for idx_ch in xrange(channels)]
+    shuffle_channels += [idx_ch * 4 + 2 for idx_ch in xrange(channels)]
+    shuffle_channels += [idx_ch * 4 + 3 for idx_ch in xrange(channels)]
+    shuffle_channels = numpy.array(shuffle_channels)
+    weights = weights[:, :, :, shuffle_channels].astype("float32")
+    if reverse:
+        res = tf.nn.conv2d_transpose(
+            value=input_,
+            filter=weights,
+            output_shape=[batch_size, height * 2, width * 2, channels],
+            strides=[1, 2, 2, 1],
+            padding="SAME",
+            name="unsqueeze_2x2")
+    else:
+        res = tf.nn.conv2d(
+            input=input_,
+            filter=weights,
+            strides=[1, 2, 2, 1],
+            padding="SAME",
+            name="squeeze_2x2")
+
+    return res

resnet/resnet_model.py

@@ -225,7 +225,7 @@ class ResNet(object):
 
   def _bottleneck_residual(self, x, in_filter, out_filter, stride,
                            activate_before_residual=False):
-    """Bottleneck resisual unit with 3 sub layers."""
+    """Bottleneck residual unit with 3 sub layers."""
     if activate_before_residual:
       with tf.variable_scope('common_bn_relu'):
         x = self._batch_norm('init_bn', x)

street/python/vgsl_model.py

@@ -454,7 +454,7 @@ class VGSLImageModel(object):
         self.labels = tf.slice(self.labels, [0, 0], [-1, 1])
         self.labels = tf.reshape(self.labels, [-1])
       cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
-          logits, self.labels, name='xent')
+          logits=logits, labels=self.labels, name='xent')
     else:
       # TODO(rays) Labels need an extra dimension for logistic, so different
       # padding functions are needed, as well as a different loss function.

syntaxnet/tensorflow

@@ -1 +1 @@
-Subproject commit 8ed00233c0cd530fec78cfad5b34f54b6f902e31
+Subproject commit aab099711d7e04034cf742ddb9b00dd15edbe99c

transformer/cluttered_mnist.py

@@ -123,7 +123,7 @@ y_logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
 
 # %% Define loss/eval/training functions
 cross_entropy = tf.reduce_mean(
-    tf.nn.softmax_cross_entropy_with_logits(y_logits, y))
+    tf.nn.softmax_cross_entropy_with_logits(logits=y_logits, targets=y))
 opt = tf.train.AdamOptimizer()
 optimizer = opt.minimize(cross_entropy)
 grads = opt.compute_gradients(cross_entropy, [b_fc_loc2])

tutorials/README.md

@@ -0,0 +1,3 @@
+# Tutorial Models
+
+This repository contains models referenced to from the [TensorFlow tutorials](https://www.tensorflow.org/tutorials/). We recommend installing TensorFlow from the [nightly builds](https://github.com/tensorflow/tensorflow#installation) rather than the r0.12 release before running these models.

tutorials/embedding/BUILD

@@ -1,125 +0,0 @@
-# Description:
-# TensorFlow model for word2vec
-
-package(default_visibility = ["//tensorflow:internal"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
-
-py_library(
-    name = "package",
-    srcs = [
-        "__init__.py",
-    ],
-    srcs_version = "PY2AND3",
-    visibility = ["//tensorflow:__subpackages__"],
-    deps = [
-        ":gen_word2vec",
-        ":word2vec",
-        ":word2vec_optimized",
-    ],
-)
-
-py_binary(
-    name = "word2vec",
-    srcs = [
-        "word2vec.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":gen_word2vec",
-        ":word2vec_kernels",
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/python:platform",
-    ],
-)
-
-py_binary(
-    name = "word2vec_optimized",
-    srcs = [
-        "word2vec_optimized.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":gen_word2vec",
-        ":word2vec_kernels",
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/python:platform",
-    ],
-)
-
-py_test(
-    name = "word2vec_test",
-    size = "small",
-    srcs = ["word2vec_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "notsan",  # b/25864127
-    ],
-    deps = [
-        ":word2vec",
-        "//tensorflow:tensorflow_py",
-    ],
-)
-
-py_test(
-    name = "word2vec_optimized_test",
-    size = "small",
-    srcs = ["word2vec_optimized_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "notsan",
-    ],
-    deps = [
-        ":word2vec_optimized",
-        "//tensorflow:tensorflow_py",
-    ],
-)
-
-cc_library(
-    name = "word2vec_ops",
-    srcs = [
-        "word2vec_ops.cc",
-    ],
-    linkstatic = 1,
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        "//tensorflow/core:framework",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "word2vec_kernels",
-    srcs = [
-        "word2vec_kernels.cc",
-    ],
-    linkstatic = 1,
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        ":word2vec_ops",
-        "//tensorflow/core",
-    ],
-    alwayslink = 1,
-)
-
-tf_gen_op_wrapper_py(
-    name = "gen_word2vec",
-    out = "gen_word2vec.py",
-    deps = [":word2vec_ops"],
-)
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)

tutorials/embedding/README.md

@@ -7,9 +7,9 @@ ICLR 2013.
 Detailed instructions on how to get started and use them are available in the
 tutorials. Brief instructions are below.
 
-* [Word2Vec Tutorial](http://tensorflow.org/tutorials/word2vec/index.md)
+* [Word2Vec Tutorial](http://tensorflow.org/tutorials/word2vec)
 
-To download the example text and evaluation data:
+Assuming you have cloned the git repository, navigate into this directory. To download the example text and evaluation data:
 
 ```shell
 wget http://mattmahoney.net/dc/text8.zip -O text8.zip
@@ -19,21 +19,18 @@ unzip -p source-archive.zip  word2vec/trunk/questions-words.txt > questions-word
 rm source-archive.zip
 ```
 
-Assuming you are using the pip package install and have cloned the git
-repository, navigate into this directory and run using:
+You will need to compile the ops as follows:
 
 ```shell
-cd tensorflow/models/embedding
-python word2vec_optimized.py \
-  --train_data=text8 \
-  --eval_data=questions-words.txt \
-  --save_path=/tmp/
+TF_INC=$(python -c 'import tensorflow as tf; print(tf.sysconfig.get_include())')
+g++ -std=c++11 -shared word2vec_ops.cc word2vec_kernels.cc -o word2vec_ops.so -fPIC -I $TF_INC -O2 -D_GLIBCXX_USE_CXX11_ABI=0
 ```
 
-To run the code from sources using bazel:
+(For an explanation of what this is doing, see the tutorial on [Adding a New Op to TensorFlow](https://www.tensorflow.org/how_tos/adding_an_op/#building_the_op_library). The flag `-D_GLIBCXX_USE_CXX11_ABI=0` is included to support newer versions of g++.)
+Then run using:
 
 ```shell
-bazel run -c opt tensorflow/models/embedding/word2vec_optimized -- \
+python word2vec_optimized.py \
   --train_data=text8 \
   --eval_data=questions-words.txt \
   --save_path=/tmp/

tutorials/embedding/__init__.py

@@ -17,5 +17,3 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
-from tensorflow.models.embedding import gen_word2vec

tutorials/embedding/word2vec.py

@@ -42,7 +42,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 import numpy as np
 import tensorflow as tf
 
-from tensorflow.models.embedding import gen_word2vec as word2vec
+word2vec = tf.load_op_library(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'word2vec_ops.so'))
 
 flags = tf.app.flags
 
@@ -263,9 +263,9 @@ class Word2Vec(object):
     # cross-entropy(logits, labels)
     opts = self._options
     true_xent = tf.nn.sigmoid_cross_entropy_with_logits(
-        true_logits, tf.ones_like(true_logits))
+        labels=tf.ones_like(true_logits), logits=true_logits)
     sampled_xent = tf.nn.sigmoid_cross_entropy_with_logits(
-        sampled_logits, tf.zeros_like(sampled_logits))
+        labels=tf.zeros_like(sampled_logits), logits=sampled_logits)
 
     # NCE-loss is the sum of the true and noise (sampled words)
     # contributions, averaged over the batch.

tutorials/embedding/word2vec_optimized.py

@@ -41,7 +41,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 import numpy as np
 import tensorflow as tf
 
-from tensorflow.models.embedding import gen_word2vec as word2vec
+word2vec = tf.load_op_library(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'word2vec_ops.so'))
 
 flags = tf.app.flags
 

tutorials/embedding/word2vec_optimized_test.py

@@ -23,7 +23,7 @@ import os
 
 import tensorflow as tf
 
-from tensorflow.models.embedding import word2vec_optimized
+import word2vec_optimized
 
 flags = tf.app.flags
 

tutorials/embedding/word2vec_test.py

@@ -23,7 +23,7 @@ import os
 
 import tensorflow as tf
 
-from tensorflow.models.embedding import word2vec
+import word2vec
 
 flags = tf.app.flags
 

tutorials/image/alexnet/alexnet_benchmark.py

@@ -17,7 +17,7 @@
 
 To run, use:
   bazel run -c opt --config=cuda \
-      third_party/tensorflow/models/image/alexnet:alexnet_benchmark
+      models/tutorials/image/alexnet:alexnet_benchmark
 
 Across 100 steps on batch size = 128.
 

tutorials/image/cifar10/__init__.py

@@ -18,5 +18,5 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.models.image.cifar10 import cifar10
-from tensorflow.models.image.cifar10 import cifar10_input
+import cifar10
+import cifar10_input

tutorials/image/cifar10/cifar10.py

@@ -35,7 +35,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import gzip
 import os
 import re
 import sys
@@ -44,7 +43,7 @@ import tarfile
 from six.moves import urllib
 import tensorflow as tf
 
-from tensorflow.models.image.cifar10 import cifar10_input
+import cifar10_input
 
 FLAGS = tf.app.flags.FLAGS
 
@@ -287,7 +286,7 @@ def loss(logits, labels):
   # Calculate the average cross entropy loss across the batch.
   labels = tf.cast(labels, tf.int64)
   cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
-      logits, labels, name='cross_entropy_per_example')
+      labels=labels, logits=logits, name='cross_entropy_per_example')
   cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
   tf.add_to_collection('losses', cross_entropy_mean)
 

tutorials/image/cifar10/cifar10_eval.py

@@ -41,7 +41,7 @@ import time
 import numpy as np
 import tensorflow as tf
 
-from tensorflow.models.image.cifar10 import cifar10
+import cifar10
 
 FLAGS = tf.app.flags.FLAGS
 

tutorials/image/cifar10/cifar10_input.py

@@ -242,6 +242,10 @@ def inputs(eval_data, data_dir, batch_size):
   # Subtract off the mean and divide by the variance of the pixels.
   float_image = tf.image.per_image_standardization(resized_image)
 
+  # Set the shapes of tensors.
+  float_image.set_shape([height, width, 3])
+  read_input.label.set_shape([1])
+
   # Ensure that the random shuffling has good mixing properties.
   min_fraction_of_examples_in_queue = 0.4
   min_queue_examples = int(num_examples_per_epoch *

tutorials/image/cifar10/cifar10_input_test.py

@@ -23,7 +23,7 @@ import os
 
 import tensorflow as tf
 
-from tensorflow.models.image.cifar10 import cifar10_input
+import cifar10_input
 
 
 class CIFAR10InputTest(tf.test.TestCase):

tutorials/image/cifar10/cifar10_multi_gpu_train.py

@@ -47,7 +47,7 @@ import time
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 import tensorflow as tf
-from tensorflow.models.image.cifar10 import cifar10
+import cifar10
 
 FLAGS = tf.app.flags.FLAGS
 

tutorials/image/cifar10/cifar10_train.py

@@ -41,7 +41,7 @@ import time
 
 import tensorflow as tf
 
-from tensorflow.models.image.cifar10 import cifar10
+import cifar10
 
 FLAGS = tf.app.flags.FLAGS
 

tutorials/image/mnist/convolutional.py

@@ -228,7 +228,7 @@ def main(_):
   # Training computation: logits + cross-entropy loss.
   logits = model(train_data_node, True)
   loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
-      logits, train_labels_node))
+      labels=train_labels_node, logits=logits))
 
   # L2 regularization for the fully connected parameters.
   regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +

tutorials/rnn/README.md

@@ -2,8 +2,8 @@ This directory contains functions for creating recurrent neural networks
 and sequence-to-sequence models. Detailed instructions on how to get started
 and use them are available in the tutorials.
 
-* [RNN Tutorial](http://tensorflow.org/tutorials/recurrent/index.md)
-* [Sequence-to-Sequence Tutorial](http://tensorflow.org/tutorials/seq2seq/index.md)
+* [RNN Tutorial](http://tensorflow.org/tutorials/recurrent/)
+* [Sequence-to-Sequence Tutorial](http://tensorflow.org/tutorials/seq2seq/)
 
 Here is a short overview of what is in this directory.
 

tutorials/rnn/ptb/__init__.py

@@ -18,4 +18,4 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.models.rnn.ptb import reader
+import reader

tutorials/rnn/ptb/ptb_word_lm.py

@@ -61,7 +61,7 @@ import time
 import numpy as np
 import tensorflow as tf
 
-from tensorflow.models.rnn.ptb import reader
+import reader
 
 flags = tf.flags
 logging = tf.logging
@@ -126,7 +126,7 @@ class PTBModel(object):
     if is_training and config.keep_prob < 1:
       inputs = tf.nn.dropout(inputs, config.keep_prob)
 
-    # Simplified version of tensorflow.models.rnn.rnn.py's rnn().
+    # Simplified version of models/tutorials/rnn/rnn.py's rnn().
     # This builds an unrolled LSTM for tutorial purposes only.
     # In general, use the rnn() or state_saving_rnn() from rnn.py.
     #

tutorials/rnn/ptb/reader_test.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Tests for tensorflow.models.ptb_lstm.ptb_reader."""
+"""Tests for models.tutorials.rnn.ptb.reader."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -23,7 +23,7 @@ import os.path
 
 import tensorflow as tf
 
-from tensorflow.models.rnn.ptb import reader
+import reader
 
 
 class PtbReaderTest(tf.test.TestCase):

tutorials/rnn/translate/__init__.py

@@ -18,5 +18,5 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.models.rnn.translate import data_utils
-from tensorflow.models.rnn.translate import seq2seq_model
+import data_utils
+import seq2seq_model

tutorials/rnn/translate/data_utils.py

@@ -177,7 +177,7 @@ def initialize_vocabulary(vocabulary_path):
     rev_vocab = []
     with gfile.GFile(vocabulary_path, mode="rb") as f:
       rev_vocab.extend(f.readlines())
-    rev_vocab = [line.strip() for line in rev_vocab]
+    rev_vocab = [tf.compat.as_bytes(line.strip()) for line in rev_vocab]
     vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
     return vocab, rev_vocab
   else:

tutorials/rnn/translate/seq2seq_model.py

@@ -25,7 +25,7 @@ import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 import tensorflow as tf
 
-from tensorflow.models.rnn.translate import data_utils
+import data_utils
 
 
 class Seq2SeqModel(object):

tutorials/rnn/translate/translate.py

@@ -42,8 +42,8 @@ import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 import tensorflow as tf
 
-from tensorflow.models.rnn.translate import data_utils
-from tensorflow.models.rnn.translate import seq2seq_model
+import data_utils
+import seq2seq_model
 
 
 tf.app.flags.DEFINE_float("learning_rate", 0.5, "Learning rate.")