/*
|
* Copyright (C) 2018 The Android Open Source Project
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*/
|
|
#include "utils/sentencepiece/normalizer.h"
|
|
#include "utils/base/logging.h"
|
#include "utils/strings/utf8.h"
|
|
namespace libtextclassifier3 {
|
|
bool SentencePieceNormalizer::Normalize(StringPiece input,
|
std::string* normalized_input) const {
|
// Ignores heading space.
|
if (remove_extra_whitespaces_) {
|
while (!input.empty()) {
|
std::pair<StringPiece, int> suffix_and_length;
|
if (!NormalizePrefix(input, &suffix_and_length)) {
|
TC3_LOG(ERROR) << "Couldn't find match in normalization table.";
|
return false;
|
}
|
if (suffix_and_length.second <= 0) {
|
TC3_LOG(ERROR) << "Consumed string is empty.";
|
return false;
|
}
|
if (suffix_and_length.first.size() != 1 ||
|
suffix_and_length.first[0] != ' ') {
|
break;
|
}
|
input.RemovePrefix(suffix_and_length.second);
|
}
|
}
|
|
if (input.empty()) {
|
*normalized_input = "";
|
return true;
|
}
|
|
// Reserves the output buffer to avoid re-allocations.
|
const int kReservedSize = input.size() * 3;
|
normalized_input->reserve(kReservedSize);
|
|
// Replaces white space with U+2581 (LOWER ONE EIGHT BLOCK)
|
// if escape_whitespaces() is set (default = true).
|
const StringPiece kSpaceSymbol = "\xe2\x96\x81";
|
|
// Adds a space symbol as a prefix (default is true)
|
// With this prefix, "world" and "hello world" are converted into
|
// "_world" and "_hello_world", which help the trainer to extract
|
// "_world" as one symbol.
|
if (add_dummy_prefix_) {
|
if (escape_whitespaces_) {
|
normalized_input->append(kSpaceSymbol.data(), kSpaceSymbol.size());
|
} else {
|
normalized_input->append(" ");
|
}
|
}
|
|
bool is_prev_space = remove_extra_whitespaces_;
|
while (!input.empty()) {
|
std::pair<StringPiece, int> p;
|
if (!NormalizePrefix(input, &p)) {
|
TC3_LOG(ERROR) << "Couldn't normalize string.";
|
return false;
|
}
|
if (p.second <= 0) {
|
TC3_LOG(ERROR) << "Consumed string is empty.";
|
return false;
|
}
|
|
StringPiece sp = p.first;
|
|
// Removes heading spaces in sentence piece,
|
// if the previous sentence piece ends with whitespace.
|
while (is_prev_space && ConsumePrefix(&sp, " ")) {
|
}
|
|
if (!sp.empty()) {
|
const char* data = sp.data();
|
for (int n = 0; n < sp.size(); ++n) {
|
if (escape_whitespaces_ && data[n] == ' ') {
|
normalized_input->append(kSpaceSymbol.data(), kSpaceSymbol.size());
|
} else {
|
*normalized_input += data[n];
|
}
|
}
|
// Checks whether the last character of sp is whitespace.
|
is_prev_space = EndsWith(sp, " ");
|
}
|
input.RemovePrefix(p.second);
|
is_prev_space = is_prev_space && remove_extra_whitespaces_;
|
}
|
|
// Ignores tailing space.
|
if (remove_extra_whitespaces_) {
|
const StringPiece space = escape_whitespaces_ ? kSpaceSymbol : " ";
|
while (EndsWith(*normalized_input, space)) {
|
const int length = normalized_input->size() - space.size();
|
normalized_input->resize(length);
|
}
|
}
|
return true;
|
}
|
|
bool SentencePieceNormalizer::NormalizePrefix(
|
StringPiece input, std::pair<StringPiece, int>* prefix) const {
|
if (input.empty()) return true;
|
TrieMatch match;
|
if (!charsmap_trie_.LongestPrefixMatch(input, &match)) {
|
TC3_LOG(ERROR) << "Couldn't find match in normalization table.";
|
return false;
|
}
|
const bool no_match = match.match_length <= 0;
|
if (no_match) {
|
const int char_length = ValidUTF8CharLength(input.data(), input.size());
|
if (char_length <= 0) {
|
// Found a malformed utf8.
|
// The rune is set to be 0xFFFD (REPLACEMENT CHARACTER),
|
// which is a valid Unicode of three bytes in utf8,
|
// but here we only consume one byte.
|
static const char kReplacementChar[] = "\xEF\xBF\xBD";
|
prefix->first = StringPiece(kReplacementChar, 3);
|
prefix->second = 1; // Consumes 1 byte, buts emit 0xFFFD.
|
} else {
|
prefix->first = StringPiece(input.data(), char_length);
|
prefix->second = char_length;
|
}
|
} else {
|
if (match.id < 0 || match.id >= charsmap_normalized_.size()) {
|
TC3_LOG(ERROR) << "Invalid entry in normalization table.";
|
return false;
|
}
|
prefix->first = StringPiece(&charsmap_normalized_.data()[match.id]);
|
prefix->second = match.match_length;
|
}
|
return true;
|
}
|
|
} // namespace libtextclassifier3
|
