TensorFlow-Models/syntaxnet/syntaxnet/segmenter_utils.h

/* Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#ifndef SYNTAXNET_SEGMENTER_UTILS_H_
#define SYNTAXNET_SEGMENTER_UTILS_H_

#include <string>
#include <vector>
#include <unordered_set>

#include "syntaxnet/sentence.pb.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "util/utf8/unicodetext.h"

namespace syntaxnet {

// A set of common convenience functions.
class SegmenterUtils {
 public:
  // Takes a text and convert it into a vector, where each element is a utf8
  // character.
  static void GetUTF8Chars(const string &text,
                           vector<tensorflow::StringPiece> *chars);

  // Sets tokens in the sentence so that each token is a single character.
  // Assigns the start/end byte offsets.
  //
  // If the sentence is not empty, the current tokens will be cleared.
  static void SetCharsAsTokens(const string &text,
                               const vector<tensorflow::StringPiece> &chars,
                               Sentence *sentence);

  // Returns true for UTF-8 characters that cannot be 'real' tokens. This is
  // defined as any whitespace, line break or paragraph break.
  static bool IsBreakChar(const string &word) {
    if (word == "\n" || word == "\t") return true;
    UnicodeText text;
    text.PointToUTF8(word.c_str(), word.length());
    CHECK_EQ(text.size(), 1);
    return kBreakChars.find(*text.begin()) != kBreakChars.end();
  }

  // Returns the break level for the next token based on the current character.
  static Token::BreakLevel BreakLevel(const string &word) {
    UnicodeText text;
    text.PointToUTF8(word.c_str(), word.length());
    auto point = *text.begin();
    if (word == "\n" || point == kLineSeparator) {
      return Token::LINE_BREAK;
    } else if (point == kParagraphSeparator) {
      return Token::SENTENCE_BREAK;  // No PARAGRAPH_BREAK in sentence proto.
    } else if (word == "\t" || kBreakChars.find(point) != kBreakChars.end()) {
      return Token::SPACE_BREAK;
    }
    return Token::NO_BREAK;
  }

  // Convenience function for computing start/end byte offsets of a character
  // StringPiece relative to original text.
  static void GetCharStartEndBytes(const string &text,
                                   tensorflow::StringPiece c,
                                   int *start,
                                   int *end) {
    *start = c.data() - text.data();
    *end = *start + c.size() - 1;
  }

  // Returns true if this segment is a valid segment. Currently checks:
  // 1) It is non-empty
  // 2) It is valid UTF8
  static bool IsValidSegment(const Sentence &sentence, const Token &token);

  // Set for utf8 break characters.
  static const std::unordered_set<int> kBreakChars;
  static const int kLineSeparator = 0x2028;
  static const int kParagraphSeparator = 0x2029;
};

}  // namespace syntaxnet

#endif  // SYNTAXNET_SEGMENTER_UTILS_H_