Sync w TF r0.12 & Bazel 0.4.3, internal updates (#953)

syntaxnet/Dockerfile

@@ -7,7 +7,7 @@ RUN mkdir -p $SYNTAXNETDIR \
     && apt-get update \
     && apt-get install git zlib1g-dev file swig python2.7 python-dev python-pip python-mock -y \
     && pip install --upgrade pip \
-    && pip install -U protobuf==3.0.0 \
+    && pip install -U protobuf==3.0.0b2 \
     && pip install asciitree \
     && pip install numpy \
     && wget https://github.com/bazelbuild/bazel/releases/download/0.4.3/bazel-0.4.3-installer-linux-x86_64.sh \
@@ -15,7 +15,7 @@ RUN mkdir -p $SYNTAXNETDIR \
     && ./bazel-0.4.3-installer-linux-x86_64.sh --user \
     && git clone --recursive https://github.com/tensorflow/models.git \
     && cd $SYNTAXNETDIR/models/syntaxnet/tensorflow \
-    && echo "\n\n\n\n" | ./configure \
+    && echo -e "\n\n\n\n\n\n" | ./configure \
     && apt-get autoremove -y \
     && apt-get clean
 

syntaxnet/README.md

@@ -78,15 +78,27 @@ source. You'll need to install:
 
 *   python 2.7:
     * python 3 support is not available yet
-*   pip (python package manager)
-    * `apt-get install python-pip` on Ubuntu
-    * `brew` installs pip along with python on OSX
+*   bazel:
+    *   **version 0.4.3**
+    *   follow the instructions [here](http://bazel.build/docs/install.html)
+    *   Alternately, Download bazel (0.4.3) <.deb> from
+        [https://github.com/bazelbuild/bazel/releases]
+        (https://github.com/bazelbuild/bazel/releases) for your system
+        configuration.
+    *   Install it using the command: sudo dpkg -i <.deb file>
+    *   Check for the bazel version by typing: bazel version
+*   swig:
+    *   `apt-get install swig` on Ubuntu
+    *   `brew install swig` on OSX
+*   protocol buffers, with a version supported by TensorFlow:
+    *   check your protobuf version with `pip freeze | grep protobuf`
+    *   upgrade to a supported version with `pip install -U protobuf==3.0.0b2`
+*   mock, the testing package:
+    *   `pip install mock`
 *   asciitree, to draw parse trees on the console for the demo:
     *   `pip install asciitree`
 *   numpy, package for scientific computing:
     *   `pip install numpy`
-*   mock, package for unit testing:
-    *   `pip install mock`
 
 Once you completed the above steps, you can build and test SyntaxNet with the
 following commands:

syntaxnet/WORKSPACE

@@ -3,9 +3,9 @@ local_repository(
   path = "tensorflow",
 )
 
-load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace')
+load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
 tf_workspace()
 
 # Specify the minimum required Bazel version.
 load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version")
-check_version("0.3.0")
+check_version("0.4.3")

syntaxnet/syntaxnet/BUILD

@@ -733,5 +733,5 @@ sh_test(
         ":parser_trainer",
         ":testdata",
     ],
-    tags = ["notsan"],
+    tags = ["slow"],
 )

syntaxnet/syntaxnet/affix.h

@@ -146,10 +146,10 @@ class AffixTable {
   int max_length_;
 
   // Index from affix ids to affix items.
-  vector<Affix *> affixes_;
+  std::vector<Affix *> affixes_;
 
   // Buckets for word-to-affix hash map.
-  vector<Affix *> buckets_;
+  std::vector<Affix *> buckets_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(AffixTable);
 };

syntaxnet/syntaxnet/arc_standard_transitions.cc

@@ -298,6 +298,40 @@ class ArcStandardTransitionSystem : public ParserTransitionSystem {
   ParserTransitionState *NewTransitionState(bool training_mode) const override {
     return new ArcStandardTransitionState();
   }
+
+  // Meta information API. Returns token indices to link parser actions back
+  // to positions in the input sentence.
+  bool SupportsActionMetaData() const override { return true; }
+
+  // Returns the child of a new arc for reduce actions.
+  int ChildIndex(const ParserState &state,
+                 const ParserAction &action) const override {
+    switch (ActionType(action)) {
+      case SHIFT:
+        return -1;
+      case LEFT_ARC:  // left arc pops stack(1)
+        return state.Stack(1);
+      case RIGHT_ARC:
+        return state.Stack(0);
+      default:
+        LOG(FATAL) << "Invalid parser action: " << action;
+    }
+  }
+
+  // Returns the parent of a new arc for reduce actions.
+  int ParentIndex(const ParserState &state,
+                  const ParserAction &action) const override {
+    switch (ActionType(action)) {
+      case SHIFT:
+        return -1;
+      case LEFT_ARC:  // left arc pops stack(1)
+        return state.Stack(0);
+      case RIGHT_ARC:
+        return state.Stack(1);
+      default:
+        LOG(FATAL) << "Invalid parser action: " << action;
+    }
+  }
 };
 
 REGISTER_TRANSITION_SYSTEM("arc-standard", ArcStandardTransitionSystem);

syntaxnet/syntaxnet/beam_reader_ops.cc

@@ -841,7 +841,7 @@ class BeamEvalOutput : public OpKernel {
     BatchState *batch_state =
         reinterpret_cast<BatchState *>(context->input(0).scalar<int64>()());
     const int batch_size = batch_state->BatchSize();
-    vector<Sentence> documents;
+    std::vector<Sentence> documents;
     for (int beam_id = 0; beam_id < batch_size; ++beam_id) {
       if (batch_state->Beam(beam_id).gold_ != nullptr &&
           batch_state->Beam(beam_id).AllFinal()) {

syntaxnet/syntaxnet/binary_segment_state.cc

@@ -54,7 +54,7 @@ void BinarySegmentState::AddParseToDocument(const ParserState &state,
                                             bool rewrite_root_labels,
                                             Sentence *sentence) const {
   if (sentence->token_size() == 0) return;
-  vector<bool> is_starts(sentence->token_size(), false);
+  std::vector<bool> is_starts(sentence->token_size(), false);
   for (int i = 0; i < NumStarts(state); ++i) {
     is_starts[LastStart(i, state)] = true;
   }

syntaxnet/syntaxnet/binary_segment_transitions_test.cc

@@ -95,9 +95,9 @@ class SegmentationTransitionTest : public ::testing::Test {
     return result.size() > 0 ? result.value(0) : -1;
   }
 
-  void CheckStarts(const ParserState &state, const vector<int> &target) {
+  void CheckStarts(const ParserState &state, const std::vector<int> &target) {
     ASSERT_EQ(state.StackSize(), target.size());
-    vector<int> starts;
+    std::vector<int> starts;
     for (int i = 0; i < state.StackSize(); ++i) {
       EXPECT_EQ(state.Stack(i), target[i]);
     }

syntaxnet/syntaxnet/char_properties.cc

@@ -88,7 +88,7 @@ namespace syntaxnet {
 
 struct CharPropertyImplementation {
   unordered_set<char32> chars;
-  vector<vector<int> > rows;
+  std::vector<std::vector<int> > rows;
   CharPropertyImplementation() {
     rows.reserve(10);
     rows.resize(1);
@@ -261,7 +261,7 @@ int CharProperty::NextElementAfter(int c) const {
   return *it;
 }
 
-REGISTER_CLASS_REGISTRY("char property wrapper", CharPropertyWrapper);
+REGISTER_SYNTAXNET_CLASS_REGISTRY("char property wrapper", CharPropertyWrapper);
 
 const CharProperty *CharProperty::Lookup(const char *subclass) {
   // Create a CharPropertyWrapper object and delete it.  We only care about

syntaxnet/syntaxnet/char_properties.h

@@ -92,7 +92,7 @@ struct CharPropertyWrapper : RegisterableClass<CharPropertyWrapper> {
 };
 
 #define REGISTER_CHAR_PROPERTY_WRAPPER(type, component) \
-  REGISTER_CLASS_COMPONENT(CharPropertyWrapper, type, component)
+  REGISTER_SYNTAXNET_CLASS_COMPONENT(CharPropertyWrapper, type, component)
 
 #define REGISTER_CHAR_PROPERTY(lsp, name)                         \
   struct name##CharPropertyWrapper : public CharPropertyWrapper { \

syntaxnet/syntaxnet/document_filters.cc

@@ -55,7 +55,7 @@ void GetTaskContext(OpKernelConstruction *context, TaskContext *task_context) {
 
 // Outputs the given batch of sentences as a tensor and deletes them.
 void OutputDocuments(OpKernelContext *context,
-                     vector<Sentence *> *document_batch) {
+                     std::vector<Sentence *> *document_batch) {
   const int64 size = document_batch->size();
   Tensor *output;
   OP_REQUIRES_OK(context,
@@ -84,7 +84,7 @@ class DocumentSource : public OpKernel {
   void Compute(OpKernelContext *context) override {
     mutex_lock lock(mu_);
     Sentence *document;
-    vector<Sentence *> document_batch;
+    std::vector<Sentence *> document_batch;
     while ((document = corpus_->Read()) != nullptr) {
       document_batch.push_back(document);
       if (static_cast<int>(document_batch.size()) == batch_size_) {
@@ -166,7 +166,7 @@ class WellFormedFilter : public OpKernel {
 
   void Compute(OpKernelContext *context) override {
     auto documents = context->input(0).vec<string>();
-    vector<Sentence *> output_documents;
+    std::vector<Sentence *> output_documents;
     for (int i = 0; i < documents.size(); ++i) {
       Sentence *document = new Sentence;
       OP_REQUIRES(context, document->ParseFromString(documents(i)),
@@ -182,7 +182,7 @@ class WellFormedFilter : public OpKernel {
 
  private:
   bool ShouldKeep(const Sentence &doc)  {
-    vector<int> visited(doc.token_size(), -1);
+    std::vector<int> visited(doc.token_size(), -1);
     for (int i = 0; i < doc.token_size(); ++i) {
       // Already visited node.
       if (visited[i] != -1) continue;
@@ -235,7 +235,7 @@ class ProjectivizeFilter : public OpKernel {
 
   void Compute(OpKernelContext *context) override {
     auto documents = context->input(0).vec<string>();
-    vector<Sentence *> output_documents;
+    std::vector<Sentence *> output_documents;
     for (int i = 0; i < documents.size(); ++i) {
       Sentence *document = new Sentence;
       OP_REQUIRES(context, document->ParseFromString(documents(i)),
@@ -255,8 +255,8 @@ class ProjectivizeFilter : public OpKernel {
     // Left and right boundaries for arcs. The left and right ends of an arc are
     // bounded by the arcs that pass over it. If an arc exceeds these bounds it
     // will cross an arc passing over it, making it a non-projective arc.
-    vector<int> left(num_tokens);
-    vector<int> right(num_tokens);
+    std::vector<int> left(num_tokens);
+    std::vector<int> right(num_tokens);
 
     // Lift the shortest non-projective arc until the document is projective.
     while (true) {

syntaxnet/syntaxnet/document_format.cc

@@ -18,6 +18,6 @@ limitations under the License.
 namespace syntaxnet {
 
 // Component registry for document formatters.
-REGISTER_CLASS_REGISTRY("document format", DocumentFormat);
+REGISTER_SYNTAXNET_CLASS_REGISTRY("document format", DocumentFormat);
 
 }  // namespace syntaxnet

syntaxnet/syntaxnet/document_format.h

@@ -32,7 +32,7 @@ namespace syntaxnet {
 // A document format component converts a key/value pair from a record to one or
 // more documents. The record format is used for selecting the document format
 // component. A document format component can be registered with the
-// REGISTER_DOCUMENT_FORMAT macro.
+// REGISTER_SYNTAXNET_DOCUMENT_FORMAT macro.
 class DocumentFormat : public RegisterableClass<DocumentFormat> {
  public:
   DocumentFormat() {}
@@ -47,7 +47,7 @@ class DocumentFormat : public RegisterableClass<DocumentFormat> {
 
   // Converts a key/value pair to one or more documents.
   virtual void ConvertFromString(const string &key, const string &value,
-                                 vector<Sentence *> *documents) = 0;
+                                 std::vector<Sentence *> *documents) = 0;
 
   // Converts a document to a key/value pair.
   virtual void ConvertToString(const Sentence &document,
@@ -57,8 +57,8 @@ class DocumentFormat : public RegisterableClass<DocumentFormat> {
   TF_DISALLOW_COPY_AND_ASSIGN(DocumentFormat);
 };
 
-#define REGISTER_DOCUMENT_FORMAT(type, component) \
-  REGISTER_CLASS_COMPONENT(DocumentFormat, type, component)
+#define REGISTER_SYNTAXNET_DOCUMENT_FORMAT(type, component) \
+  REGISTER_SYNTAXNET_CLASS_COMPONENT(DocumentFormat, type, component)
 
 }  // namespace syntaxnet
 

syntaxnet/syntaxnet/embedding_feature_extractor.cc

@@ -46,14 +46,16 @@ void GenericEmbeddingFeatureExtractor::Setup(TaskContext *context) {
 void GenericEmbeddingFeatureExtractor::Init(TaskContext *context) {
 }
 
-vector<vector<SparseFeatures>> GenericEmbeddingFeatureExtractor::ConvertExample(
-    const vector<FeatureVector> &feature_vectors) const {
+std::vector<std::vector<SparseFeatures>>
+GenericEmbeddingFeatureExtractor::ConvertExample(
+    const std::vector<FeatureVector> &feature_vectors) const {
   // Extract the features.
-  vector<vector<SparseFeatures>> sparse_features(feature_vectors.size());
+  std::vector<std::vector<SparseFeatures>> sparse_features(
+      feature_vectors.size());
   for (size_t i = 0; i < feature_vectors.size(); ++i) {
     // Convert the nlp_parser::FeatureVector to dist belief format.
-    sparse_features[i] =
-        vector<SparseFeatures>(generic_feature_extractor(i).feature_types());
+    sparse_features[i] = std::vector<SparseFeatures>(
+        generic_feature_extractor(i).feature_types());
 
     for (int j = 0; j < feature_vectors[i].size(); ++j) {
       const FeatureType &feature_type = *feature_vectors[i].type(j);

syntaxnet/syntaxnet/embedding_feature_extractor.h

@@ -78,15 +78,20 @@ class GenericEmbeddingFeatureExtractor {
   int EmbeddingDims(int index) const { return embedding_dims_[index]; }
 
   // Accessor for embedding dims (dimensions of the embedding spaces).
-  const vector<int> &embedding_dims() const { return embedding_dims_; }
+  const std::vector<int> &embedding_dims() const { return embedding_dims_; }
 
-  const vector<string> &embedding_fml() const { return embedding_fml_; }
+  const std::vector<string> &embedding_fml() const { return embedding_fml_; }
 
   // Get parameter name by concatenating the prefix and the original name.
   string GetParamName(const string &param_name) const {
     return tensorflow::strings::StrCat(ArgPrefix(), "_", param_name);
   }
 
+  // Returns the name of the embedding space.
+  const string &embedding_name(int index) const {
+    return embedding_names_[index];
+  }
+
  protected:
   // Provides the generic class with access to the templated extractors. This is
   // used to get the type information out of the feature extractor without
@@ -99,21 +104,21 @@ class GenericEmbeddingFeatureExtractor {
   // single SparseFeatures. The predicates are mapped through map_fn which
   // should point to either mutable_map_fn or const_map_fn depending on whether
   // or not the predicate maps should be updated.
-  vector<vector<SparseFeatures>> ConvertExample(
-      const vector<FeatureVector> &feature_vectors) const;
+  std::vector<std::vector<SparseFeatures>> ConvertExample(
+      const std::vector<FeatureVector> &feature_vectors) const;
 
  private:
   // Embedding space names for parameter sharing.
-  vector<string> embedding_names_;
+  std::vector<string> embedding_names_;
 
   // FML strings for each feature extractor.
-  vector<string> embedding_fml_;
+  std::vector<string> embedding_fml_;
 
   // Size of each of the embedding spaces (maximum predicate id).
-  vector<int> embedding_sizes_;
+  std::vector<int> embedding_sizes_;
 
   // Embedding dimensions of the embedding spaces (i.e. 32, 64 etc.)
-  vector<int> embedding_dims_;
+  std::vector<int> embedding_dims_;
 
   // Whether or not to add string descriptions to converted examples.
   bool add_strings_;
@@ -168,9 +173,9 @@ class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
   // will not be updated and so unrecognized predicates may occur. In such a
   // case the SparseFeatures object associated with a given extractor class and
   // feature will be empty.
-  vector<vector<SparseFeatures>> ExtractSparseFeatures(
+  std::vector<std::vector<SparseFeatures>> ExtractSparseFeatures(
       const WorkspaceSet &workspaces, const OBJ &obj, ARGS... args) const {
-    vector<FeatureVector> features(feature_extractors_.size());
+    std::vector<FeatureVector> features(feature_extractors_.size());
     ExtractFeatures(workspaces, obj, args..., &features);
     return ConvertExample(features);
   }
@@ -180,7 +185,7 @@ class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
   // mapping is applied.
   void ExtractFeatures(const WorkspaceSet &workspaces, const OBJ &obj,
                        ARGS... args,
-                       vector<FeatureVector> *features) const {
+                       std::vector<FeatureVector> *features) const {
     DCHECK(features != nullptr);
     DCHECK_EQ(features->size(), feature_extractors_.size());
     for (int i = 0; i < feature_extractors_.size(); ++i) {
@@ -201,7 +206,7 @@ class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
 
  private:
   // Templated feature extractor class.
-  vector<EXTRACTOR> feature_extractors_;
+  std::vector<EXTRACTOR> feature_extractors_;
 };
 
 class ParserEmbeddingFeatureExtractor

syntaxnet/syntaxnet/feature_extractor.cc

@@ -50,13 +50,13 @@ void GenericFeatureExtractor::InitializeFeatureTypes() {
     }
   }
 
-  vector<string> types_names;
+  std::vector<string> types_names;
   GetFeatureTypeNames(&types_names);
   CHECK_EQ(feature_types_.size(), types_names.size());
 }
 
 void GenericFeatureExtractor::GetFeatureTypeNames(
-    vector<string> *type_names) const {
+    std::vector<string> *type_names) const {
   for (size_t i = 0; i < feature_types_.size(); ++i) {
     FeatureType *ft = feature_types_[i];
     type_names->push_back(ft->name());
@@ -102,7 +102,7 @@ int GenericFeatureFunction::GetIntParameter(const string &name,
 }
 
 void GenericFeatureFunction::GetFeatureTypes(
-    vector<FeatureType *> *types) const {
+    std::vector<FeatureType *> *types) const {
   if (feature_type_ != nullptr) types->push_back(feature_type_);
 }
 
@@ -111,7 +111,7 @@ FeatureType *GenericFeatureFunction::GetFeatureType() const {
   if (feature_type_ != nullptr) return feature_type_;
 
   // Get feature types for function.
-  vector<FeatureType *> types;
+  std::vector<FeatureType *> types;
   GetFeatureTypes(&types);
 
   // If there is exactly one feature type return this, else return null.

syntaxnet/syntaxnet/feature_extractor.h

@@ -101,7 +101,7 @@ class FeatureVector {
   };
 
   // Array for storing feature vector elements.
-  vector<Element> features_;
+  std::vector<Element> features_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(FeatureVector);
 };
@@ -133,7 +133,7 @@ class GenericFeatureExtractor {
 
   // Returns all feature types names used by the extractor. The names are
   // added to the types_names array.  Invalid before Init() has been called.
-  void GetFeatureTypeNames(vector<string> *type_names) const;
+  void GetFeatureTypeNames(std::vector<string> *type_names) const;
 
   // Returns a feature type used in the extractor.  Invalid before Init() has
   // been called.
@@ -157,7 +157,7 @@ class GenericFeatureExtractor {
 
   // Returns all feature types used by the extractor. The feature types are
   // added to the result array.
-  virtual void GetFeatureTypes(vector<FeatureType *> *types) const = 0;
+  virtual void GetFeatureTypes(std::vector<FeatureType *> *types) const = 0;
 
   // Descriptor for the feature extractor. This is a protocol buffer that
   // contains all the information about the feature extractor. The feature
@@ -167,7 +167,7 @@ class GenericFeatureExtractor {
   // All feature types used by the feature extractor. The collection of all the
   // feature types describes the feature space of the feature set produced by
   // the feature extractor.  Not owned.
-  vector<FeatureType *> feature_types_;
+  std::vector<FeatureType *> feature_types_;
 };
 
 // The generic feature function is the type-independent part of a feature
@@ -198,7 +198,7 @@ class GenericFeatureFunction {
   // Appends the feature types produced by the feature function to types.  The
   // default implementation appends feature_type(), if non-null.  Invalid
   // before Init() has been called.
-  virtual void GetFeatureTypes(vector<FeatureType *> *types) const;
+  virtual void GetFeatureTypes(std::vector<FeatureType *> *types) const;
 
   // Returns the feature type for feature produced by this feature function. If
   // the feature function produces features of different types this returns
@@ -383,7 +383,7 @@ class NestedFeatureFunction : public FeatureFunction<OBJ, ARGS...> {
   ~NestedFeatureFunction() override { utils::STLDeleteElements(&nested_); }
 
   // By default, just appends the nested feature types.
-  void GetFeatureTypes(vector<FeatureType *> *types) const override {
+  void GetFeatureTypes(std::vector<FeatureType *> *types) const override {
     CHECK(!this->nested().empty())
         << "Nested features require nested features to be defined.";
     for (auto *function : nested_) function->GetFeatureTypes(types);
@@ -415,14 +415,14 @@ class NestedFeatureFunction : public FeatureFunction<OBJ, ARGS...> {
   }
 
   // Returns the list of nested feature functions.
-  const vector<NES *> &nested() const { return nested_; }
+  const std::vector<NES *> &nested() const { return nested_; }
 
   // Instantiates nested feature functions for a feature function. Creates and
   // initializes one feature function for each sub-descriptor in the feature
   // descriptor.
   static void CreateNested(GenericFeatureExtractor *extractor,
                            FeatureFunctionDescriptor *fd,
-                           vector<NES *> *functions,
+                           std::vector<NES *> *functions,
                            const string &prefix) {
     for (int i = 0; i < fd->feature_size(); ++i) {
       FeatureFunctionDescriptor *sub = fd->mutable_feature(i);
@@ -434,7 +434,7 @@ class NestedFeatureFunction : public FeatureFunction<OBJ, ARGS...> {
  protected:
   // The nested feature functions, if any, in order of declaration in the
   // feature descriptor.  Owned.
-  vector<NES *> nested_;
+  std::vector<NES *> nested_;
 };
 
 // Base class for a nested feature function that takes nested features with the
@@ -506,7 +506,7 @@ template<class DER, class OBJ, class ...ARGS>
 class FeatureLocator : public MetaFeatureFunction<OBJ, ARGS...> {
  public:
   // Feature locators have an additional check that there is no intrinsic type.
-  void GetFeatureTypes(vector<FeatureType *> *types) const override {
+  void GetFeatureTypes(std::vector<FeatureType *> *types) const override {
     CHECK(this->feature_type() == nullptr)
         << "FeatureLocators should not have an intrinsic type.";
     MetaFeatureFunction<OBJ, ARGS...>::GetFeatureTypes(types);
@@ -604,7 +604,7 @@ class FeatureExtractor : public GenericFeatureExtractor {
   }
 
   // Collect all feature types used in the feature extractor.
-  void GetFeatureTypes(vector<FeatureType *> *types) const override {
+  void GetFeatureTypes(std::vector<FeatureType *> *types) const override {
     for (int i = 0; i < functions_.size(); ++i) {
       functions_[i]->GetFeatureTypes(types);
     }
@@ -612,11 +612,11 @@ class FeatureExtractor : public GenericFeatureExtractor {
 
   // Top-level feature functions (and variables) in the feature extractor.
   // Owned.
-  vector<Function *> functions_;
+  std::vector<Function *> functions_;
 };
 
-#define REGISTER_FEATURE_FUNCTION(base, name, component) \
-  REGISTER_CLASS_COMPONENT(base, name, component)
+#define REGISTER_SYNTAXNET_FEATURE_FUNCTION(base, name, component) \
+  REGISTER_SYNTAXNET_CLASS_COMPONENT(base, name, component)
 
 }  // namespace syntaxnet
 

syntaxnet/syntaxnet/feature_types.h

@@ -81,7 +81,7 @@ class ResourceBasedFeatureType : public FeatureType {
   // resource->NumValues() so as to avoid collisions; this is verified with
   // CHECK at creation.
   ResourceBasedFeatureType(const string &name, const Resource *resource,
-                           const map<FeatureValue, string> &values)
+                           const std::map<FeatureValue, string> &values)
       : FeatureType(name), resource_(resource), values_(values) {
     max_value_ = resource->NumValues() - 1;
     for (const auto &pair : values) {
@@ -121,7 +121,7 @@ class ResourceBasedFeatureType : public FeatureType {
   FeatureValue max_value_;
 
   // Mapping for extra feature values not in the resource.
-  map<FeatureValue, string> values_;
+  std::map<FeatureValue, string> values_;
 };
 
 // Feature type that is defined using an explicit map from FeatureValue to
@@ -139,7 +139,7 @@ class ResourceBasedFeatureType : public FeatureType {
 class EnumFeatureType : public FeatureType {
  public:
   EnumFeatureType(const string &name,
-                  const map<FeatureValue, string> &value_names)
+                  const std::map<FeatureValue, string> &value_names)
       : FeatureType(name), value_names_(value_names) {
     for (const auto &pair : value_names) {
       CHECK_GE(pair.first, 0)
@@ -168,7 +168,26 @@ class EnumFeatureType : public FeatureType {
   FeatureValue domain_size_ = 0;
 
   // Names of feature values.
-  map<FeatureValue, string> value_names_;
+  std::map<FeatureValue, string> value_names_;
+};
+
+// Feature type for numeric features.
+class NumericFeatureType : public FeatureType {
+ public:
+  // Initializes numeric feature.
+  NumericFeatureType(const string &name, FeatureValue size)
+      : FeatureType(name), size_(size) {}
+
+  // Returns numeric feature value.
+  string GetFeatureValueName(FeatureValue value) const override {
+    return value < 0 ? "" : tensorflow::strings::Printf("%lld", value);
+  }
+
+  // Returns the number of feature values.
+  FeatureValue GetDomainSize() const override { return size_; }
+
+ private:
+  FeatureValue size_;
 };
 
 }  // namespace syntaxnet

syntaxnet/syntaxnet/graph_builder.py

@@ -69,7 +69,7 @@ def EmbeddingLookupFeatures(params, sparse_features, allow_weights):
 
   if allow_weights:
     # Multiply by weights, reshaping to allow broadcast.
-    broadcast_weights_shape = tf.concat(0, [tf.shape(weights), [1]])
+    broadcast_weights_shape = tf.concat_v2([tf.shape(weights), [1]], 0)
     embeddings *= tf.reshape(weights, broadcast_weights_shape)
 
   # Sum embeddings by index.
@@ -330,7 +330,7 @@ class GreedyParser(object):
                                            i,
                                            return_average=return_average))
 
-    last_layer = tf.concat(1, embeddings)
+    last_layer = tf.concat_v2(embeddings, 1)
     last_layer_size = self.embedding_size
 
     # Create ReLU layers.
@@ -404,8 +404,9 @@ class GreedyParser(object):
     """Cross entropy plus L2 loss on weights and biases of the hidden layers."""
     dense_golden = BatchedSparseToDense(gold_actions, self._num_actions)
     cross_entropy = tf.div(
-        tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(
-            logits, dense_golden)), batch_size)
+        tf.reduce_sum(
+            tf.nn.softmax_cross_entropy_with_logits(
+                labels=dense_golden, logits=logits)), batch_size)
     regularized_params = [tf.nn.l2_loss(p)
                           for k, p in self.params.items()
                           if k.startswith('weights') or k.startswith('bias')]

syntaxnet/syntaxnet/lexicon_builder.cc

@@ -117,7 +117,7 @@ class LexiconBuilder : public OpKernel {
         tag_to_category.SetCategory(token.tag(), token.category());
 
         // Add characters.
-        vector<tensorflow::StringPiece> char_sp;
+        std::vector<tensorflow::StringPiece> char_sp;
         SegmenterUtils::GetUTF8Chars(word, &char_sp);
         for (const auto &c : char_sp) {
           const string c_str = c.ToString();

syntaxnet/syntaxnet/morpher_transitions.cc

@@ -132,10 +132,10 @@ class MorphologyTransitionState : public ParserTransitionState {
 
  private:
   // Currently assigned morphological analysis for each token in this sentence.
-  vector<int> tag_;
+  std::vector<int> tag_;
 
   // Gold morphological analysis from the input document.
-  vector<int> gold_tag_;
+  std::vector<int> gold_tag_;
 
   // Tag map used for conversions between integer and string representations
   // part of speech tags. Not owned.

syntaxnet/syntaxnet/morphology_label_set.cc

@@ -69,7 +69,7 @@ void MorphologyLabelSet::Write(ProtoRecordWriter *writer) const {
 }
 
 string MorphologyLabelSet::StringForMatch(const TokenMorphology &morph) const {
-  vector<string> attributes;
+  std::vector<string> attributes;
   for (const auto &a : morph.attribute()) {
     attributes.push_back(
         tensorflow::strings::StrCat(a.name(), kSeparator, a.value()));
@@ -80,7 +80,7 @@ string MorphologyLabelSet::StringForMatch(const TokenMorphology &morph) const {
 
 string FullLabelFeatureType::GetFeatureValueName(FeatureValue value) const {
   const TokenMorphology &morph = label_set_->Lookup(value);
-  vector<string> attributes;
+  std::vector<string> attributes;
   for (const auto &a : morph.attribute()) {
     attributes.push_back(tensorflow::strings::StrCat(a.name(), ":", a.value()));
   }

syntaxnet/syntaxnet/morphology_label_set.h

@@ -70,7 +70,7 @@ class MorphologyLabelSet {
   // defined as follows:
   //
   //   a == b iff the set of attribute pairs (attribute, value) is identical.
-  vector<TokenMorphology> label_set_;
+  std::vector<TokenMorphology> label_set_;
 
   // Because protocol buffer equality is complicated, we implement our own
   // equality operator based on strings. This unordered_map allows us to do the

syntaxnet/syntaxnet/parser_features.cc

@@ -24,11 +24,12 @@ limitations under the License.
 namespace syntaxnet {
 
 // Registry for the parser feature functions.
-REGISTER_CLASS_REGISTRY("parser feature function", ParserFeatureFunction);
+REGISTER_SYNTAXNET_CLASS_REGISTRY("parser feature function",
+                                  ParserFeatureFunction);
 
 // Registry for the parser state + token index feature functions.
-REGISTER_CLASS_REGISTRY("parser+index feature function",
-                        ParserIndexFeatureFunction);
+REGISTER_SYNTAXNET_CLASS_REGISTRY("parser+index feature function",
+                                  ParserIndexFeatureFunction);
 
 RootFeatureType::RootFeatureType(const string &name,
                                  const FeatureType &wrapped_type,
@@ -228,4 +229,29 @@ class ParserTokenFeatureFunction : public NestedFeatureFunction<
 REGISTER_PARSER_IDX_FEATURE_FUNCTION("token",
                                      ParserTokenFeatureFunction);
 
+// Parser feature that always fetches the focus (position) of the token.
+class FocusFeatureFunction : public ParserIndexFeatureFunction {
+ public:
+  // Initializes the feature function.
+  void Init(TaskContext *context) override {
+    // Note: this feature can return up to N values, where N is the length of
+    // the input sentence. Here, we give the arbitrary number 100 since it
+    // is not used.
+    set_feature_type(new NumericFeatureType(name(), 100));
+  }
+
+  void Evaluate(const WorkspaceSet &workspaces, const ParserState &object,
+                int focus, FeatureVector *result) const override {
+    FeatureValue value = focus;
+    result->add(feature_type(), value);
+  }
+
+  FeatureValue Compute(const WorkspaceSet &workspaces, const ParserState &state,
+                       int focus, const FeatureVector *result) const override {
+    return focus;
+  }
+};
+
+REGISTER_PARSER_IDX_FEATURE_FUNCTION("focus", FocusFeatureFunction);
+
 }  // namespace syntaxnet

syntaxnet/syntaxnet/parser_features.h

@@ -48,10 +48,11 @@ typedef FeatureFunction<ParserState, int> ParserIndexFeatureFunction;
 
 // Utilities to register the two types of parser features.
 #define REGISTER_PARSER_FEATURE_FUNCTION(name, component) \
-  REGISTER_FEATURE_FUNCTION(ParserFeatureFunction, name, component)
+  REGISTER_SYNTAXNET_FEATURE_FUNCTION(ParserFeatureFunction, name, component)
 
-#define REGISTER_PARSER_IDX_FEATURE_FUNCTION(name, component) \
-  REGISTER_FEATURE_FUNCTION(ParserIndexFeatureFunction, name, component)
+#define REGISTER_PARSER_IDX_FEATURE_FUNCTION(name, component)           \
+  REGISTER_SYNTAXNET_FEATURE_FUNCTION(ParserIndexFeatureFunction, name, \
+                                      component)
 
 // Alias for locator type that takes a parser state, and produces a focus
 // integer that can be used on nested ParserIndexFeature objects.

syntaxnet/syntaxnet/parser_state.h

@@ -210,14 +210,14 @@ class ParserState {
   int next_;
 
   // Parse stack of partially processed tokens.
-  vector<int> stack_;
+  std::vector<int> stack_;
 
   // List of head positions for the (partial) dependency tree.
-  vector<int> head_;
+  std::vector<int> head_;
 
   // List of dependency relation labels describing the (partial) dependency
   // tree.
-  vector<int> label_;
+  std::vector<int> label_;
 
   // Score of the parser state.
   double score_ = 0.0;

syntaxnet/syntaxnet/parser_trainer_test.sh

@@ -17,6 +17,9 @@
 # This test trains a parser on a small dataset, then runs it in greedy mode and
 # in structured mode with beam 1, and checks that the result is identical.
 
+
+
+
 set -eux
 
 BINDIR=$TEST_SRCDIR/$TEST_WORKSPACE/syntaxnet

syntaxnet/syntaxnet/parser_transitions.cc

@@ -20,7 +20,7 @@ limitations under the License.
 namespace syntaxnet {
 
 // Transition system registry.
-REGISTER_CLASS_REGISTRY("transition system", ParserTransitionSystem);
+REGISTER_SYNTAXNET_CLASS_REGISTRY("transition system", ParserTransitionSystem);
 
 void ParserTransitionSystem::PerformAction(ParserAction action,
                                            ParserState *state) const {

syntaxnet/syntaxnet/parser_transitions.h

@@ -118,7 +118,7 @@ class ParserTransitionSystem
   // Returns all next gold actions for the parser during training using the
   // dependency relations found in the underlying annotated sentence.
   virtual void GetAllNextGoldActions(const ParserState &state,
-                                     vector<ParserAction> *actions) const {
+                                     std::vector<ParserAction> *actions) const {
     ParserAction action = GetNextGoldAction(state);
     *actions = {action};
   }
@@ -201,7 +201,7 @@ class ParserTransitionSystem
 };
 
 #define REGISTER_TRANSITION_SYSTEM(type, component) \
-  REGISTER_CLASS_COMPONENT(ParserTransitionSystem, type, component)
+  REGISTER_SYNTAXNET_CLASS_COMPONENT(ParserTransitionSystem, type, component)
 
 }  // namespace syntaxnet
 

syntaxnet/syntaxnet/populate_test_inputs.cc

@@ -88,13 +88,13 @@ bool PopulateTestInputs::Populate(
 
 PopulateTestInputs::Create PopulateTestInputs::CreateTFMapFromDocumentTokens(
     const Sentence &document,
-    std::function<vector<string>(const Token &)> token2str) {
+    std::function<std::vector<string>(const Token &)> token2str) {
   return [document, token2str](TaskInput *input) {
     TermFrequencyMap map;
 
     // Build and write the dummy term frequency map.
     for (const Token &token : document.token()) {
-      vector<string> strings_for_token = token2str(token);
+      std::vector<string> strings_for_token = token2str(token);
       for (const string &s : strings_for_token) map.Increment(s);
     }
     string file_name = AddPart(input, "text", "");
@@ -116,22 +116,22 @@ PopulateTestInputs::Create PopulateTestInputs::CreateTagToCategoryFromTokens(
   };
 }
 
-vector<string> PopulateTestInputs::TokenCategory(const Token &token) {
+std::vector<string> PopulateTestInputs::TokenCategory(const Token &token) {
   if (token.has_category()) return {token.category()};
   return {};
 }
 
-vector<string> PopulateTestInputs::TokenLabel(const Token &token) {
+std::vector<string> PopulateTestInputs::TokenLabel(const Token &token) {
   if (token.has_label()) return {token.label()};
   return {};
 }
 
-vector<string> PopulateTestInputs::TokenTag(const Token &token) {
+std::vector<string> PopulateTestInputs::TokenTag(const Token &token) {
   if (token.has_tag()) return {token.tag()};
   return {};
 }
 
-vector<string> PopulateTestInputs::TokenWord(const Token &token) {
+std::vector<string> PopulateTestInputs::TokenWord(const Token &token) {
   if (token.has_word()) return {token.word()};
   return {};
 }

syntaxnet/syntaxnet/populate_test_inputs.h

@@ -130,17 +130,17 @@ class PopulateTestInputs {
   // then saved to FLAGS_test_tmpdir/name.
   static Create CreateTFMapFromDocumentTokens(
       const Sentence &document,
-      std::function<vector<string>(const Token &)> token2str);
+      std::function<std::vector<string>(const Token &)> token2str);
 
   // Creates a StringToStringMap protocol buffer input that maps tags to
   // categories. Uses whatever mapping is present in the document.
   static Create CreateTagToCategoryFromTokens(const Sentence &document);
 
   // Default implementations for "token2str" above.
-  static vector<string> TokenCategory(const Token &token);
-  static vector<string> TokenLabel(const Token &token);
-  static vector<string> TokenTag(const Token &token);
-  static vector<string> TokenWord(const Token &token);
+  static std::vector<string> TokenCategory(const Token &token);
+  static std::vector<string> TokenLabel(const Token &token);
+  static std::vector<string> TokenTag(const Token &token);
+  static std::vector<string> TokenWord(const Token &token);
 
   // Utility function. Sets the TaskInput->part() fields for a new input part.
   // Returns the file name.

syntaxnet/syntaxnet/proto_io.h

@@ -131,7 +131,7 @@ class StdIn : public tensorflow::RandomAccessFile {
                       char *scratch) const {
     memcpy(scratch, buffer_.data(), buffer_.size());
     buffer_ = buffer_.substr(n);
-    result->set(scratch, n);
+    *result = tensorflow::StringPiece(scratch, n);
     expected_offset_ += n;
   }
 
@@ -161,7 +161,7 @@ class TextReader {
   Sentence *Read() {
     // Skips emtpy sentences, e.g., blank lines at the beginning of a file or
     // commented out blocks.
-    vector<Sentence *> sentences;
+    std::vector<Sentence *> sentences;
     string key, value;
     while (sentences.empty() && format_->ReadRecord(buffer_.get(), &value)) {
       key = tensorflow::strings::StrCat(filename_, ":", sentence_count_);

syntaxnet/syntaxnet/reader_ops.cc

@@ -143,7 +143,7 @@ class ParsingReader : public OpKernel {
     }
 
     // Create the outputs for each feature space.
-    vector<Tensor *> feature_outputs(features_->NumEmbeddings());
+    std::vector<Tensor *> feature_outputs(features_->NumEmbeddings());
     for (size_t i = 0; i < feature_outputs.size(); ++i) {
       OP_REQUIRES_OK(context, context->allocate_output(
                                   i, TensorShape({sentence_batch_->size(),
@@ -399,7 +399,7 @@ class DecodedParseReader : public ParsingReader {
     // pull from the back of the docids queue as long as the sentences have been
     // completely processed. If the next document has not been completely
     // processed yet, then the docid will not be found in 'sentence_map_'.
-    vector<Sentence> sentences;
+    std::vector<Sentence> sentences;
     while (!docids_.empty() &&
            sentence_map_.find(docids_.back()) != sentence_map_.end()) {
       sentences.emplace_back(sentence_map_[docids_.back()]);
@@ -427,7 +427,7 @@ class DecodedParseReader : public ParsingReader {
   string scoring_type_;
 
   mutable std::deque<string> docids_;
-  mutable map<string, Sentence> sentence_map_;
+  mutable std::map<string, Sentence> sentence_map_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(DecodedParseReader);
 };

syntaxnet/syntaxnet/registry.h

@@ -28,11 +28,11 @@ limitations under the License.
 //   };
 //
 //   #define REGISTER_FUNCTION(type, component)
-//     REGISTER_INSTANCE_COMPONENT(Function, type, component);
+//     REGISTER_SYNTAXNET_INSTANCE_COMPONENT(Function, type, component);
 //
 //  function.cc:
 //
-//   REGISTER_INSTANCE_REGISTRY("function", Function);
+//   REGISTER_SYNTAXNET_INSTANCE_REGISTRY("function", Function);
 //
 //   class Cos : public Function {
 //    public:
@@ -218,22 +218,22 @@ class RegisterableInstance {
   static Registry registry_;
 };
 
-#define REGISTER_CLASS_COMPONENT(base, type, component)             \
+#define REGISTER_SYNTAXNET_CLASS_COMPONENT(base, type, component)   \
   static base *__##component##__factory() { return new component; } \
   static base::Registry::Registrar __##component##__##registrar(    \
       base::registry(), type, #component, __FILE__, __LINE__,       \
       __##component##__factory)
 
-#define REGISTER_CLASS_REGISTRY(type, classname)                  \
+#define REGISTER_SYNTAXNET_CLASS_REGISTRY(type, classname)        \
   template <>                                                     \
   classname::Registry RegisterableClass<classname>::registry_ = { \
       type, #classname, __FILE__, __LINE__, NULL}
 
-#define REGISTER_INSTANCE_COMPONENT(base, type, component)       \
-  static base::Registry::Registrar __##component##__##registrar( \
+#define REGISTER_SYNTAXNET_INSTANCE_COMPONENT(base, type, component) \
+  static base::Registry::Registrar __##component##__##registrar(     \
       base::registry(), type, #component, __FILE__, __LINE__, new component)
 
-#define REGISTER_INSTANCE_REGISTRY(type, classname)                  \
+#define REGISTER_SYNTAXNET_INSTANCE_REGISTRY(type, classname)        \
   template <>                                                        \
   classname::Registry RegisterableInstance<classname>::registry_ = { \
       type, #classname, __FILE__, __LINE__, NULL}

syntaxnet/syntaxnet/segmenter_utils.cc

@@ -38,7 +38,7 @@ const std::unordered_set<int> SegmenterUtils::kBreakChars({
 });
 
 void SegmenterUtils::GetUTF8Chars(const string &text,
-                                  vector<tensorflow::StringPiece> *chars) {
+                                  std::vector<tensorflow::StringPiece> *chars) {
   const char *start = text.c_str();
   const char *end = text.c_str() + text.size();
   while (start < end) {
@@ -50,7 +50,7 @@ void SegmenterUtils::GetUTF8Chars(const string &text,
 
 void SegmenterUtils::SetCharsAsTokens(
     const string &text,
-    const vector<tensorflow::StringPiece> &chars,
+    const std::vector<tensorflow::StringPiece> &chars,
     Sentence *sentence) {
   sentence->clear_token();
   sentence->set_text(text);

syntaxnet/syntaxnet/segmenter_utils.h

@@ -32,15 +32,15 @@ class SegmenterUtils {
   // Takes a text and convert it into a vector, where each element is a utf8
   // character.
   static void GetUTF8Chars(const string &text,
-                           vector<tensorflow::StringPiece> *chars);
+                           std::vector<tensorflow::StringPiece> *chars);
 
   // Sets tokens in the sentence so that each token is a single character.
   // Assigns the start/end byte offsets.
   //
   // If the sentence is not empty, the current tokens will be cleared.
-  static void SetCharsAsTokens(const string &text,
-                               const vector<tensorflow::StringPiece> &chars,
-                               Sentence *sentence);
+  static void SetCharsAsTokens(
+      const string &text, const std::vector<tensorflow::StringPiece> &chars,
+      Sentence *sentence);
 
   // Returns true for UTF-8 characters that cannot be 'real' tokens. This is
   // defined as any whitespace, line break or paragraph break.

syntaxnet/syntaxnet/segmenter_utils_test.cc

@@ -59,9 +59,9 @@ static Sentence GetKoSentence() {
 
 // Gets the start end bytes of the given chars in the given text.
 static void GetStartEndBytes(const string &text,
-                             const vector<tensorflow::StringPiece> &chars,
-                             vector<int> *starts,
-                             vector<int> *ends) {
+                             const std::vector<tensorflow::StringPiece> &chars,
+                             std::vector<int> *starts,
+                             std::vector<int> *ends) {
   SegmenterUtils segment_utils;
   for (const tensorflow::StringPiece &c : chars) {
     int start; int end;
@@ -75,14 +75,14 @@ static void GetStartEndBytes(const string &text,
 TEST(SegmenterUtilsTest, GetCharsTest) {
   // Create test sentence.
   const Sentence sentence = GetKoSentence();
-  vector<tensorflow::StringPiece> chars;
+  std::vector<tensorflow::StringPiece> chars;
   SegmenterUtils::GetUTF8Chars(sentence.text(), &chars);
 
   // Check the number of characters is correct.
   CHECK_EQ(chars.size(), 12);
 
-  vector<int> starts;
-  vector<int> ends;
+  std::vector<int> starts;
+  std::vector<int> ends;
   GetStartEndBytes(sentence.text(), chars, &starts, &ends);
 
   // Check start positions.
@@ -118,12 +118,12 @@ TEST(SegmenterUtilsTest, GetCharsTest) {
 TEST(SegmenterUtilsTest, SetCharsAsTokensTest) {
   // Create test sentence.
   const Sentence sentence = GetKoSentence();
-  vector<tensorflow::StringPiece> chars;
+  std::vector<tensorflow::StringPiece> chars;
   SegmenterUtils segment_utils;
   segment_utils.GetUTF8Chars(sentence.text(), &chars);
 
-  vector<int> starts;
-  vector<int> ends;
+  std::vector<int> starts;
+  std::vector<int> ends;
   GetStartEndBytes(sentence.text(), chars, &starts, &ends);
 
   // Check that the new docs word, start and end positions are properly set.

syntaxnet/syntaxnet/sentence_features.cc

@@ -83,7 +83,8 @@ string TermFrequencyMapSetFeature::WorkspaceName() const {
 }
 
 namespace {
-void GetUTF8Chars(const string &word, vector<tensorflow::StringPiece> *chars) {
+void GetUTF8Chars(const string &word,
+                  std::vector<tensorflow::StringPiece> *chars) {
   UnicodeText text;
   text.PointToUTF8(word.c_str(), word.size());
   for (UnicodeText::const_iterator it = text.begin(); it != text.end(); ++it) {
@@ -98,9 +99,10 @@ int UTF8FirstLetterNumBytes(const char *utf8_str) {
 
 }  // namespace
 
-void CharNgram::GetTokenIndices(const Token &token, vector<int> *values) const {
+void CharNgram::GetTokenIndices(const Token &token,
+                                std::vector<int> *values) const {
   values->clear();
-  vector<tensorflow::StringPiece> char_sp;
+  std::vector<tensorflow::StringPiece> char_sp;
   if (use_terminators_) char_sp.push_back("^");
   GetUTF8Chars(token.word(), &char_sp);
   if (use_terminators_) char_sp.push_back("$");
@@ -121,7 +123,7 @@ void CharNgram::GetTokenIndices(const Token &token, vector<int> *values) const {
 }
 
 void MorphologySet::GetTokenIndices(const Token &token,
-                                    vector<int> *values) const {
+                                    std::vector<int> *values) const {
   values->clear();
   const TokenMorphology &token_morphology =
       token.GetExtension(TokenMorphology::morphology);
@@ -401,7 +403,8 @@ string AffixTableFeature::GetFeatureValueName(FeatureValue value) const {
 }
 
 // Registry for the Sentence + token index feature functions.
-REGISTER_CLASS_REGISTRY("sentence+index feature function", SentenceFeature);
+REGISTER_SYNTAXNET_CLASS_REGISTRY("sentence+index feature function",
+                                  SentenceFeature);
 
 // Register the features defined in the header.
 REGISTER_SENTENCE_IDX_FEATURE("word", Word);

syntaxnet/syntaxnet/sentence_features.h

@@ -110,7 +110,7 @@ class TokenLookupSetFeature : public SentenceFeature {
   // feature value set. The index is relative to the start of the sentence.
   virtual void LookupToken(const WorkspaceSet &workspaces,
                            const Sentence &sentence, int index,
-                           vector<int> *values) const = 0;
+                           std::vector<int> *values) const = 0;
 
   // Given a feature value, returns a string representation.
   virtual string GetFeatureValueName(int value) const = 0;
@@ -137,7 +137,7 @@ class TokenLookupSetFeature : public SentenceFeature {
 
   // Returns a pre-computed token value from the cache. This assumes the cache
   // is populated.
-  const vector<int> &GetCachedValueSet(const WorkspaceSet &workspaces,
+  const std::vector<int> &GetCachedValueSet(const WorkspaceSet &workspaces,
                                        const Sentence &sentence,
                                        int focus) const {
     // Do bounds checking on focus.
@@ -152,7 +152,7 @@ class TokenLookupSetFeature : public SentenceFeature {
   void Evaluate(const WorkspaceSet &workspaces, const Sentence &sentence,
                 int focus, FeatureVector *result) const override {
     if (focus >= 0 && focus < sentence.token_size()) {
-      const vector<int> &elements =
+      const std::vector<int> &elements =
           GetCachedValueSet(workspaces, sentence, focus);
       for (auto &value : elements) {
         result->add(this->feature_type(), value);
@@ -233,7 +233,7 @@ class TermFrequencyMapSetFeature : public TokenLookupSetFeature {
 
   // Returns index of raw word text.
   virtual void GetTokenIndices(const Token &token,
-                               vector<int> *values) const = 0;
+                               std::vector<int> *values) const = 0;
 
   // Requests the resource inputs.
   void Setup(TaskContext *context) override;
@@ -261,7 +261,7 @@ class TermFrequencyMapSetFeature : public TokenLookupSetFeature {
   // Given a position in a sentence and workspaces, looks up the corresponding
   // feature value set. The index is relative to the start of the sentence.
   void LookupToken(const WorkspaceSet &workspaces, const Sentence &sentence,
-                   int index, vector<int> *values) const override {
+                   int index, std::vector<int> *values) const override {
     GetTokenIndices(sentence.token(index), values);
   }
 
@@ -376,7 +376,8 @@ class CharNgram : public TermFrequencyMapSetFeature {
   }
 
   // Returns index of raw word text.
-  void GetTokenIndices(const Token &token, vector<int> *values) const override;
+  void GetTokenIndices(const Token &token,
+                       std::vector<int> *values) const override;
 
  private:
   // Size parameter (n) for the ngrams.
@@ -401,7 +402,8 @@ class MorphologySet : public TermFrequencyMapSetFeature {
   }
 
   // Returns index of raw word text.
-  void GetTokenIndices(const Token &token, vector<int> *values) const override;
+  void GetTokenIndices(const Token &token,
+                       std::vector<int> *values) const override;
 };
 
 class LexicalCategoryFeature : public TokenLookupFeature {
@@ -635,7 +637,7 @@ typedef FeatureExtractor<Sentence, int> SentenceExtractor;
 
 // Utility to register the sentence_instance::Feature functions.
 #define REGISTER_SENTENCE_IDX_FEATURE(name, type) \
-  REGISTER_FEATURE_FUNCTION(SentenceFeature, name, type)
+  REGISTER_SYNTAXNET_FEATURE_FUNCTION(SentenceFeature, name, type)
 
 }  // namespace syntaxnet
 

syntaxnet/syntaxnet/sentence_features_test.cc

@@ -73,8 +73,8 @@ class SentenceFeaturesTest : public ::testing::Test {
 
   // Extracts a vector of string representations from evaluating the prepared
   // set feature (returning multiple values) at the given index.
-  virtual vector<string> ExtractMultiFeature(int index) {
-    vector<string> values;
+  virtual std::vector<string> ExtractMultiFeature(int index) {
+    std::vector<string> values;
     FeatureVector result;
     extractor_->ExtractFeatures(workspaces_, sentence_, index,
                                 &result);
@@ -97,8 +97,8 @@ class SentenceFeaturesTest : public ::testing::Test {
 
   // Checks that a vector workspace is equal to a target vector.
   void CheckVectorWorkspace(const VectorIntWorkspace &workspace,
-                            vector<int> target) {
-    vector<int> src;
+                            std::vector<int> target) {
+    std::vector<int> src;
     for (int i = 0; i < workspace.size(); ++i) {
       src.push_back(workspace.element(i));
     }

syntaxnet/syntaxnet/structured_graph_builder.py

@@ -36,7 +36,8 @@ def AddCrossEntropy(batch_size, n):
     return tf.constant(0, dtype=tf.float32, shape=[1])
 
   for beam_id in range(batch_size):
-    beam_gold_slot = tf.reshape(tf.slice(n['gold_slot'], [beam_id], [1]), [1])
+    beam_gold_slot = tf.reshape(
+        tf.strided_slice(n['gold_slot'], [beam_id], [beam_id + 1], [1]), [1])
     def _ComputeCrossEntropy():
       """Adds ops to compute cross entropy of the gold path in a beam."""
       # Requires a cast so that UnsortedSegmentSum, in the gradient,
@@ -48,8 +49,9 @@ def AddCrossEntropy(batch_size, n):
       beam_scores = tf.reshape(tf.gather(n['all_path_scores'], idx), [1, -1])
       num = tf.shape(idx)
       return tf.nn.softmax_cross_entropy_with_logits(
-          beam_scores, tf.expand_dims(
-              tf.sparse_to_dense(beam_gold_slot, num, [1.], 0.), 0))
+          labels=tf.expand_dims(
+              tf.sparse_to_dense(beam_gold_slot, num, [1.], 0.), 0),
+          logits=beam_scores)
     # The conditional here is needed to deal with the last few batches of the
     # corpus which can contain -1 in beam_gold_slot for empty batch slots.
     cross_entropies.append(cf.cond(

syntaxnet/syntaxnet/tagger_transitions.cc

@@ -128,10 +128,10 @@ class TaggerTransitionState : public ParserTransitionState {
 
  private:
   // Currently assigned POS tags for each token in this sentence.
-  vector<int> tag_;
+  std::vector<int> tag_;
 
   // Gold POS tags from the input document.
-  vector<int> gold_tag_;
+  std::vector<int> gold_tag_;
 
   // Tag map used for conversions between integer and string representations
   // part of speech tags. Not owned.

syntaxnet/syntaxnet/task_context.h

@@ -72,7 +72,7 @@ class TaskContext {
 
   // Vector of parameters required by this task.  These must be specified in the
   // task rather than relying on default values.
-  vector<string> required_parameters_;
+  std::vector<string> required_parameters_;
 };
 
 }  // namespace syntaxnet

syntaxnet/syntaxnet/term_frequency_map.cc

@@ -32,7 +32,7 @@ int TermFrequencyMap::Increment(const string &term) {
   const TermIndex::const_iterator it = term_index_.find(term);
   if (term_index_.find(term) != term_index_.end()) {
     // Increment the existing term.
-    pair<string, int64> &data = term_data_[it->second];
+    std::pair<string, int64> &data = term_data_[it->second];
     CHECK_EQ(term, data.first);
     ++(data.second);
     return it->second;
@@ -41,7 +41,7 @@ int TermFrequencyMap::Increment(const string &term) {
     const int index = term_index_.size();
     CHECK_LT(index, std::numeric_limits<int32>::max());  // overflow
     term_index_[term] = index;
-    term_data_.push_back(pair<string, int64>(term, 1));
+    term_data_.push_back(std::pair<string, int64>(term, 1));
     return index;
   }
 }
@@ -74,7 +74,7 @@ void TermFrequencyMap::Load(const string &filename, int min_frequency,
   int64 last_frequency = -1;
   for (int i = 0; i < total && i < max_num_terms; ++i) {
     TF_CHECK_OK(buffer.ReadLine(&line));
-    vector<string> elements = utils::Split(line, ' ');
+    std::vector<string> elements = utils::Split(line, ' ');
     CHECK_EQ(2, elements.size());
     CHECK(!elements[0].empty());
     CHECK(!elements[1].empty());
@@ -97,7 +97,7 @@ void TermFrequencyMap::Load(const string &filename, int min_frequency,
     // Assign the next available index.
     const int index = term_index_.size();
     term_index_[term] = index;
-    term_data_.push_back(pair<string, int64>(term, frequency));
+    term_data_.push_back(std::pair<string, int64>(term, frequency));
   }
   CHECK_EQ(term_index_.size(), term_data_.size());
   LOG(INFO) << "Loaded " << term_index_.size() << " terms from " << filename
@@ -107,8 +107,8 @@ void TermFrequencyMap::Load(const string &filename, int min_frequency,
 struct TermFrequencyMap::SortByFrequencyThenTerm {
   // Return a > b to sort in descending order of frequency; otherwise,
   // lexicographic sort on term.
-  bool operator()(const pair<string, int64> &a,
-                  const pair<string, int64> &b) const {
+  bool operator()(const std::pair<string, int64> &a,
+                  const std::pair<string, int64> &b) const {
     return (a.second > b.second || (a.second == b.second && a.first < b.first));
   }
 };
@@ -117,7 +117,7 @@ void TermFrequencyMap::Save(const string &filename) const {
   CHECK_EQ(term_index_.size(), term_data_.size());
 
   // Copy and sort the term data.
-  vector<pair<string, int64>> sorted_data(term_data_);
+  std::vector<std::pair<string, int64>> sorted_data(term_data_);
   std::sort(sorted_data.begin(), sorted_data.end(), SortByFrequencyThenTerm());
 
   // Write the number of terms.
@@ -149,7 +149,7 @@ TagToCategoryMap::TagToCategoryMap(const string &filename) {
   tensorflow::io::BufferedInputStream buffer(&stream, kInputBufferSize);
   string line;
   while (buffer.ReadLine(&line) == tensorflow::Status::OK()) {
-    vector<string> pair = utils::Split(line, '\t');
+    std::vector<string> pair = utils::Split(line, '\t');
     CHECK(line.empty() || pair.size() == 2) << line;
     tag_to_category_[pair[0]] = pair[1];
   }

syntaxnet/syntaxnet/term_frequency_map.h

@@ -83,7 +83,7 @@ class TermFrequencyMap {
   TermIndex term_index_;
 
   // Mapping from indices to term and frequency.
-  vector<pair<string, int64>> term_data_;
+  std::vector<std::pair<string, int64>> term_data_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(TermFrequencyMap);
 };
@@ -107,7 +107,7 @@ class TagToCategoryMap {
   void Save(const string &filename) const;
 
  private:
-  map<string, string> tag_to_category_;
+  std::map<string, string> tag_to_category_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(TagToCategoryMap);
 };

syntaxnet/syntaxnet/text_formats.cc

@@ -83,16 +83,16 @@ class CoNLLSyntaxFormat : public DocumentFormat {
   }
 
   void ConvertFromString(const string &key, const string &value,
-                         vector<Sentence *> *sentences) override {
+                         std::vector<Sentence *> *sentences) override {
     // Create new sentence.
     Sentence *sentence = new Sentence();
 
     // Each line corresponds to one token.
     string text;
-    vector<string> lines = utils::Split(value, '\n');
+    std::vector<string> lines = utils::Split(value, '\n');
 
     // Add each token to the sentence.
-    vector<string> fields;
+    std::vector<string> fields;
     int expected_id = 1;
     for (size_t i = 0; i < lines.size(); ++i) {
       // Split line into tab-separated fields.
@@ -166,12 +166,12 @@ class CoNLLSyntaxFormat : public DocumentFormat {
   void ConvertToString(const Sentence &sentence, string *key,
                        string *value) override {
     *key = sentence.docid();
-    vector<string> lines;
+    std::vector<string> lines;
     for (int i = 0; i < sentence.token_size(); ++i) {
       Token token = sentence.token(i);
       if (join_category_to_pos_) SplitCategoryFromPos(&token);
       if (add_pos_as_attribute_) RemovePosFromAttributes(&token);
-      vector<string> fields(10);
+      std::vector<string> fields(10);
       fields[0] = tensorflow::strings::Printf("%d", i + 1);
       fields[1] = UnderscoreIfEmpty(token.word());
       fields[2] = "_";
@@ -198,14 +198,14 @@ class CoNLLSyntaxFormat : public DocumentFormat {
   void AddMorphAttributes(const string &attributes, Token *token) {
     TokenMorphology *morph =
         token->MutableExtension(TokenMorphology::morphology);
-    vector<string> att_vals = utils::Split(attributes, '|');
+    std::vector<string> att_vals = utils::Split(attributes, '|');
     for (int i = 0; i < att_vals.size(); ++i) {
-      vector<string> att_val = utils::SplitOne(att_vals[i], '=');
+      std::vector<string> att_val = utils::SplitOne(att_vals[i], '=');
 
       // Format is either:
       //   1) a1=v1|a2=v2..., e.g., Czech CoNLL data, or,
       //   2) v1|v2|..., e.g., German CoNLL data.
-      const pair<string, string> name_value =
+      const std::pair<string, string> name_value =
           att_val.size() == 2 ? std::make_pair(att_val[0], att_val[1])
                               : std::make_pair(att_val[0], "on");
 
@@ -282,7 +282,7 @@ class CoNLLSyntaxFormat : public DocumentFormat {
   TF_DISALLOW_COPY_AND_ASSIGN(CoNLLSyntaxFormat);
 };
 
-REGISTER_DOCUMENT_FORMAT("conll-sentence", CoNLLSyntaxFormat);
+REGISTER_SYNTAXNET_DOCUMENT_FORMAT("conll-sentence", CoNLLSyntaxFormat);
 
 // Reader for segmentation training data format. This reader assumes the input
 // format is similar to CoNLL format but with only two fileds:
@@ -325,16 +325,16 @@ class SegmentationTrainingDataFormat : public CoNLLSyntaxFormat {
   // to SPACE_BREAK to indicate that the corresponding gold transition for that
   // character token is START. Otherwise NO_BREAK to indicate MERGE.
   void ConvertFromString(const string &key, const string &value,
-                         vector<Sentence *> *sentences) override {
+                         std::vector<Sentence *> *sentences) override {
     // Create new sentence.
     Sentence *sentence = new Sentence();
 
     // Each line corresponds to one token.
     string text;
-    vector<string> lines = utils::Split(value, '\n');
+    std::vector<string> lines = utils::Split(value, '\n');
 
     // Add each token to the sentence.
-    vector<string> fields;
+    std::vector<string> fields;
     for (size_t i = 0; i < lines.size(); ++i) {
       // Split line into tab-separated fields.
       fields.clear();
@@ -362,7 +362,7 @@ class SegmentationTrainingDataFormat : public CoNLLSyntaxFormat {
       }
 
       // Add character-based token to sentence.
-      vector<tensorflow::StringPiece> chars;
+      std::vector<tensorflow::StringPiece> chars;
       SegmenterUtils::GetUTF8Chars(word, &chars);
       bool is_first_char = true;
       for (auto utf8char : chars) {
@@ -398,7 +398,8 @@ class SegmentationTrainingDataFormat : public CoNLLSyntaxFormat {
   }
 };
 
-REGISTER_DOCUMENT_FORMAT("segment-train-data", SegmentationTrainingDataFormat);
+REGISTER_SYNTAXNET_DOCUMENT_FORMAT("segment-train-data",
+                                   SegmentationTrainingDataFormat);
 
 // Reader for tokenized text. This reader expects every sentence to be on a
 // single line and tokens on that line to be separated by single spaces.
@@ -414,7 +415,7 @@ class TokenizedTextFormat : public DocumentFormat {
   }
 
   void ConvertFromString(const string &key, const string &value,
-                         vector<Sentence *> *sentences) override {
+                         std::vector<Sentence *> *sentences) override {
     Sentence *sentence = new Sentence();
     string text;
     for (const string &word : utils::Split(value, ' ')) {
@@ -463,7 +464,7 @@ class TokenizedTextFormat : public DocumentFormat {
   TF_DISALLOW_COPY_AND_ASSIGN(TokenizedTextFormat);
 };
 
-REGISTER_DOCUMENT_FORMAT("tokenized-text", TokenizedTextFormat);
+REGISTER_SYNTAXNET_DOCUMENT_FORMAT("tokenized-text", TokenizedTextFormat);
 
 // Reader for un-tokenized text. This reader expects every sentence to be on a
 // single line. For each line in the input, a sentence proto will be created,
@@ -474,9 +475,9 @@ class UntokenizedTextFormat : public TokenizedTextFormat {
   UntokenizedTextFormat() {}
 
   void ConvertFromString(const string &key, const string &value,
-                         vector<Sentence *> *sentences) override {
+                         std::vector<Sentence *> *sentences) override {
     Sentence *sentence = new Sentence();
-    vector<tensorflow::StringPiece> chars;
+    std::vector<tensorflow::StringPiece> chars;
     SegmenterUtils::GetUTF8Chars(value, &chars);
     int start = 0;
     for (auto utf8char : chars) {
@@ -502,7 +503,7 @@ class UntokenizedTextFormat : public TokenizedTextFormat {
   TF_DISALLOW_COPY_AND_ASSIGN(UntokenizedTextFormat);
 };
 
-REGISTER_DOCUMENT_FORMAT("untokenized-text", UntokenizedTextFormat);
+REGISTER_SYNTAXNET_DOCUMENT_FORMAT("untokenized-text", UntokenizedTextFormat);
 
 // Text reader that attmpts to perform Penn Treebank tokenization on arbitrary
 // raw text. Adapted from https://www.cis.upenn.edu/~treebank/tokenizer.sed
@@ -514,8 +515,8 @@ class EnglishTextFormat : public TokenizedTextFormat {
   EnglishTextFormat() {}
 
   void ConvertFromString(const string &key, const string &value,
-                         vector<Sentence *> *sentences) override {
-    vector<pair<string, string>> preproc_rules = {
+                         std::vector<Sentence *> *sentences) override {
+    std::vector<std::pair<string, string>> preproc_rules = {
         // Punctuation.
         {"’", "'"},
         {"…", "..."},
@@ -570,7 +571,7 @@ class EnglishTextFormat : public TokenizedTextFormat {
         {"♦", ""},
     };
 
-    vector<pair<string, string>> rules = {
+    std::vector<std::pair<string, string>> rules = {
         // attempt to get correct directional quotes
         {R"re(^")re", "`` "},
         {R"re(([ \([{<])")re", "\\1 `` "},
@@ -639,10 +640,10 @@ class EnglishTextFormat : public TokenizedTextFormat {
     };
 
     string rewritten = value;
-    for (const pair<string, string> &rule : preproc_rules) {
+    for (const std::pair<string, string> &rule : preproc_rules) {
       RE2::GlobalReplace(&rewritten, rule.first, rule.second);
     }
-    for (const pair<string, string> &rule : rules) {
+    for (const std::pair<string, string> &rule : rules) {
       RE2::GlobalReplace(&rewritten, rule.first, rule.second);
     }
     TokenizedTextFormat::ConvertFromString(key, rewritten, sentences);
@@ -652,6 +653,6 @@ class EnglishTextFormat : public TokenizedTextFormat {
   TF_DISALLOW_COPY_AND_ASSIGN(EnglishTextFormat);
 };
 
-REGISTER_DOCUMENT_FORMAT("english-text", EnglishTextFormat);
+REGISTER_SYNTAXNET_DOCUMENT_FORMAT("english-text", EnglishTextFormat);
 
 }  // namespace syntaxnet

syntaxnet/syntaxnet/workspace.cc

@@ -37,7 +37,7 @@ VectorIntWorkspace::VectorIntWorkspace(int size) : elements_(size) {}
 VectorIntWorkspace::VectorIntWorkspace(int size, int value)
     : elements_(size, value) {}
 
-VectorIntWorkspace::VectorIntWorkspace(const vector<int> &elements)
+VectorIntWorkspace::VectorIntWorkspace(const std::vector<int> &elements)
     : elements_(elements) {}
 
 string VectorIntWorkspace::TypeName() { return "Vector"; }

syntaxnet/syntaxnet/workspace.h

@@ -57,7 +57,7 @@ class WorkspaceRegistry {
   int Request(const string &name) {
     const std::type_index id = std::type_index(typeid(W));
     workspace_types_[id] = W::TypeName();
-    vector<string> &names = workspace_names_[id];
+    std::vector<string> &names = workspace_names_[id];
     for (int i = 0; i < names.size(); ++i) {
       if (names[i] == name) return i;
     }
@@ -65,8 +65,8 @@ class WorkspaceRegistry {
     return names.size() - 1;
   }
 
-  const std::unordered_map<std::type_index, vector<string> > &WorkspaceNames()
-      const {
+  const std::unordered_map<std::type_index, std::vector<string> >
+      &WorkspaceNames() const {
     return workspace_names_;
   }
 
@@ -78,7 +78,7 @@ class WorkspaceRegistry {
   std::unordered_map<std::type_index, string> workspace_types_;
 
   // Workspace names, indexed as workspace_names_[typeid][workspace].
-  std::unordered_map<std::type_index, vector<string> > workspace_names_;
+  std::unordered_map<std::type_index, std::vector<string> > workspace_names_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(WorkspaceRegistry);
 };
@@ -137,7 +137,7 @@ class WorkspaceSet {
 
  private:
   // The set of workspaces, indexed as workspaces_[typeid][index].
-  std::unordered_map<std::type_index, vector<Workspace *> > workspaces_;
+  std::unordered_map<std::type_index, std::vector<Workspace *> > workspaces_;
 };
 
 // A workspace that wraps around a single int.
@@ -170,7 +170,7 @@ class VectorIntWorkspace : public Workspace {
   explicit VectorIntWorkspace(int size);
 
   // Creates a vector initialized with the given array.
-  explicit VectorIntWorkspace(const vector<int> &elements);
+  explicit VectorIntWorkspace(const std::vector<int> &elements);
 
   // Creates a vector of the given size, with each element initialized to the
   // given value.
@@ -189,7 +189,7 @@ class VectorIntWorkspace : public Workspace {
 
  private:
   // The enclosed vector.
-  vector<int> elements_;
+  std::vector<int> elements_;
 };
 
 // A workspace that wraps around a vector of vector of int.
@@ -202,14 +202,14 @@ class VectorVectorIntWorkspace : public Workspace {
   static string TypeName();
 
   // Returns the i'th vector of elements.
-  const vector<int> &elements(int i) const { return elements_[i]; }
+  const std::vector<int> &elements(int i) const { return elements_[i]; }
 
   // Mutable access to the i'th vector of elements.
-  vector<int> *mutable_elements(int i) { return &(elements_[i]); }
+  std::vector<int> *mutable_elements(int i) { return &(elements_[i]); }
 
  private:
   // The enclosed vector of vector of elements.
-  vector<vector<int> > elements_;
+  std::vector<std::vector<int> > elements_;
 };
 
 }  // namespace syntaxnet

syntaxnet/tensorflow

@@ -1 +1 @@
-Subproject commit aab099711d7e04034cf742ddb9b00dd15edbe99c
+Subproject commit 45ab528211c962b19e12f6b77165848310271624