/*
|
* Copyright (C) 2018 The Android Open Source Project
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*/
|
|
#include "actions/actions-suggestions.h"
|
|
#include <memory>
|
|
#include "actions/lua-actions.h"
|
#include "actions/types.h"
|
#include "actions/zlib-utils.h"
|
#include "utils/base/logging.h"
|
#include "utils/flatbuffers.h"
|
#include "utils/lua-utils.h"
|
#include "utils/regex-match.h"
|
#include "utils/strings/split.h"
|
#include "utils/strings/stringpiece.h"
|
#include "utils/utf8/unicodetext.h"
|
#include "utils/zlib/zlib_regex.h"
|
#include "tensorflow/lite/string_util.h"
|
|
namespace libtextclassifier3 {
|
|
const std::string& ActionsSuggestions::kViewCalendarType =
|
*[]() { return new std::string("view_calendar"); }();
|
const std::string& ActionsSuggestions::kViewMapType =
|
*[]() { return new std::string("view_map"); }();
|
const std::string& ActionsSuggestions::kTrackFlightType =
|
*[]() { return new std::string("track_flight"); }();
|
const std::string& ActionsSuggestions::kOpenUrlType =
|
*[]() { return new std::string("open_url"); }();
|
const std::string& ActionsSuggestions::kSendSmsType =
|
*[]() { return new std::string("send_sms"); }();
|
const std::string& ActionsSuggestions::kCallPhoneType =
|
*[]() { return new std::string("call_phone"); }();
|
const std::string& ActionsSuggestions::kSendEmailType =
|
*[]() { return new std::string("send_email"); }();
|
const std::string& ActionsSuggestions::kShareLocation =
|
*[]() { return new std::string("share_location"); }();
|
|
namespace {
|
|
const ActionsModel* LoadAndVerifyModel(const uint8_t* addr, int size) {
|
flatbuffers::Verifier verifier(addr, size);
|
if (VerifyActionsModelBuffer(verifier)) {
|
return GetActionsModel(addr);
|
} else {
|
return nullptr;
|
}
|
}
|
|
template <typename T>
|
T ValueOrDefault(const flatbuffers::Table* values, const int32 field_offset,
|
const T default_value) {
|
if (values == nullptr) {
|
return default_value;
|
}
|
return values->GetField<T>(field_offset, default_value);
|
}
|
|
// Returns number of (tail) messages of a conversation to consider.
|
int NumMessagesToConsider(const Conversation& conversation,
|
const int max_conversation_history_length) {
|
return ((max_conversation_history_length < 0 ||
|
conversation.messages.size() < max_conversation_history_length)
|
? conversation.messages.size()
|
: max_conversation_history_length);
|
}
|
|
} // namespace
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromUnownedBuffer(
|
const uint8_t* buffer, const int size, const UniLib* unilib,
|
const std::string& triggering_preconditions_overlay) {
|
auto actions = std::unique_ptr<ActionsSuggestions>(new ActionsSuggestions());
|
const ActionsModel* model = LoadAndVerifyModel(buffer, size);
|
if (model == nullptr) {
|
return nullptr;
|
}
|
actions->model_ = model;
|
actions->SetOrCreateUnilib(unilib);
|
actions->triggering_preconditions_overlay_buffer_ =
|
triggering_preconditions_overlay;
|
if (!actions->ValidateAndInitialize()) {
|
return nullptr;
|
}
|
return actions;
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromScopedMmap(
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap, const UniLib* unilib,
|
const std::string& triggering_preconditions_overlay) {
|
if (!mmap->handle().ok()) {
|
TC3_VLOG(1) << "Mmap failed.";
|
return nullptr;
|
}
|
const ActionsModel* model = LoadAndVerifyModel(
|
reinterpret_cast<const uint8_t*>(mmap->handle().start()),
|
mmap->handle().num_bytes());
|
if (!model) {
|
TC3_LOG(ERROR) << "Model verification failed.";
|
return nullptr;
|
}
|
auto actions = std::unique_ptr<ActionsSuggestions>(new ActionsSuggestions());
|
actions->model_ = model;
|
actions->mmap_ = std::move(mmap);
|
actions->SetOrCreateUnilib(unilib);
|
actions->triggering_preconditions_overlay_buffer_ =
|
triggering_preconditions_overlay;
|
if (!actions->ValidateAndInitialize()) {
|
return nullptr;
|
}
|
return actions;
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromScopedMmap(
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap,
|
std::unique_ptr<UniLib> unilib,
|
const std::string& triggering_preconditions_overlay) {
|
if (!mmap->handle().ok()) {
|
TC3_VLOG(1) << "Mmap failed.";
|
return nullptr;
|
}
|
const ActionsModel* model = LoadAndVerifyModel(
|
reinterpret_cast<const uint8_t*>(mmap->handle().start()),
|
mmap->handle().num_bytes());
|
if (!model) {
|
TC3_LOG(ERROR) << "Model verification failed.";
|
return nullptr;
|
}
|
auto actions = std::unique_ptr<ActionsSuggestions>(new ActionsSuggestions());
|
actions->model_ = model;
|
actions->mmap_ = std::move(mmap);
|
actions->owned_unilib_ = std::move(unilib);
|
actions->unilib_ = actions->owned_unilib_.get();
|
actions->triggering_preconditions_overlay_buffer_ =
|
triggering_preconditions_overlay;
|
if (!actions->ValidateAndInitialize()) {
|
return nullptr;
|
}
|
return actions;
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromFileDescriptor(
|
const int fd, const int offset, const int size, const UniLib* unilib,
|
const std::string& triggering_preconditions_overlay) {
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap;
|
if (offset >= 0 && size >= 0) {
|
mmap.reset(new libtextclassifier3::ScopedMmap(fd, offset, size));
|
} else {
|
mmap.reset(new libtextclassifier3::ScopedMmap(fd));
|
}
|
return FromScopedMmap(std::move(mmap), unilib,
|
triggering_preconditions_overlay);
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromFileDescriptor(
|
const int fd, const int offset, const int size,
|
std::unique_ptr<UniLib> unilib,
|
const std::string& triggering_preconditions_overlay) {
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap;
|
if (offset >= 0 && size >= 0) {
|
mmap.reset(new libtextclassifier3::ScopedMmap(fd, offset, size));
|
} else {
|
mmap.reset(new libtextclassifier3::ScopedMmap(fd));
|
}
|
return FromScopedMmap(std::move(mmap), std::move(unilib),
|
triggering_preconditions_overlay);
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromFileDescriptor(
|
const int fd, const UniLib* unilib,
|
const std::string& triggering_preconditions_overlay) {
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap(
|
new libtextclassifier3::ScopedMmap(fd));
|
return FromScopedMmap(std::move(mmap), unilib,
|
triggering_preconditions_overlay);
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromFileDescriptor(
|
const int fd, std::unique_ptr<UniLib> unilib,
|
const std::string& triggering_preconditions_overlay) {
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap(
|
new libtextclassifier3::ScopedMmap(fd));
|
return FromScopedMmap(std::move(mmap), std::move(unilib),
|
triggering_preconditions_overlay);
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromPath(
|
const std::string& path, const UniLib* unilib,
|
const std::string& triggering_preconditions_overlay) {
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap(
|
new libtextclassifier3::ScopedMmap(path));
|
return FromScopedMmap(std::move(mmap), unilib,
|
triggering_preconditions_overlay);
|
}
|
|
std::unique_ptr<ActionsSuggestions> ActionsSuggestions::FromPath(
|
const std::string& path, std::unique_ptr<UniLib> unilib,
|
const std::string& triggering_preconditions_overlay) {
|
std::unique_ptr<libtextclassifier3::ScopedMmap> mmap(
|
new libtextclassifier3::ScopedMmap(path));
|
return FromScopedMmap(std::move(mmap), std::move(unilib),
|
triggering_preconditions_overlay);
|
}
|
|
void ActionsSuggestions::SetOrCreateUnilib(const UniLib* unilib) {
|
if (unilib != nullptr) {
|
unilib_ = unilib;
|
} else {
|
owned_unilib_.reset(new UniLib);
|
unilib_ = owned_unilib_.get();
|
}
|
}
|
|
bool ActionsSuggestions::ValidateAndInitialize() {
|
if (model_ == nullptr) {
|
TC3_LOG(ERROR) << "No model specified.";
|
return false;
|
}
|
|
if (model_->smart_reply_action_type() == nullptr) {
|
TC3_LOG(ERROR) << "No smart reply action type specified.";
|
return false;
|
}
|
|
if (!InitializeTriggeringPreconditions()) {
|
TC3_LOG(ERROR) << "Could not initialize preconditions.";
|
return false;
|
}
|
|
if (model_->locales() &&
|
!ParseLocales(model_->locales()->c_str(), &locales_)) {
|
TC3_LOG(ERROR) << "Could not parse model supported locales.";
|
return false;
|
}
|
|
if (model_->tflite_model_spec() != nullptr) {
|
model_executor_ = TfLiteModelExecutor::FromBuffer(
|
model_->tflite_model_spec()->tflite_model());
|
if (!model_executor_) {
|
TC3_LOG(ERROR) << "Could not initialize model executor.";
|
return false;
|
}
|
}
|
|
if (model_->annotation_actions_spec() != nullptr &&
|
model_->annotation_actions_spec()->annotation_mapping() != nullptr) {
|
for (const AnnotationActionsSpec_::AnnotationMapping* mapping :
|
*model_->annotation_actions_spec()->annotation_mapping()) {
|
annotation_entity_types_.insert(mapping->annotation_collection()->str());
|
}
|
}
|
|
std::unique_ptr<ZlibDecompressor> decompressor = ZlibDecompressor::Instance();
|
if (!InitializeRules(decompressor.get())) {
|
TC3_LOG(ERROR) << "Could not initialize rules.";
|
return false;
|
}
|
|
if (model_->actions_entity_data_schema() != nullptr) {
|
entity_data_schema_ = LoadAndVerifyFlatbuffer<reflection::Schema>(
|
model_->actions_entity_data_schema()->Data(),
|
model_->actions_entity_data_schema()->size());
|
if (entity_data_schema_ == nullptr) {
|
TC3_LOG(ERROR) << "Could not load entity data schema data.";
|
return false;
|
}
|
|
entity_data_builder_.reset(
|
new ReflectiveFlatbufferBuilder(entity_data_schema_));
|
} else {
|
entity_data_schema_ = nullptr;
|
}
|
|
std::string actions_script;
|
if (GetUncompressedString(model_->lua_actions_script(),
|
model_->compressed_lua_actions_script(),
|
decompressor.get(), &actions_script) &&
|
!actions_script.empty()) {
|
if (!Compile(actions_script, &lua_bytecode_)) {
|
TC3_LOG(ERROR) << "Could not precompile lua actions snippet.";
|
return false;
|
}
|
}
|
|
if (!(ranker_ = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
|
model_->ranking_options(), decompressor.get(),
|
model_->smart_reply_action_type()->str()))) {
|
TC3_LOG(ERROR) << "Could not create an action suggestions ranker.";
|
return false;
|
}
|
|
// Create feature processor if specified.
|
const ActionsTokenFeatureProcessorOptions* options =
|
model_->feature_processor_options();
|
if (options != nullptr) {
|
if (options->tokenizer_options() == nullptr) {
|
TC3_LOG(ERROR) << "No tokenizer options specified.";
|
return false;
|
}
|
|
feature_processor_.reset(new ActionsFeatureProcessor(options, unilib_));
|
embedding_executor_ = TFLiteEmbeddingExecutor::FromBuffer(
|
options->embedding_model(), options->embedding_size(),
|
options->embedding_quantization_bits());
|
|
if (embedding_executor_ == nullptr) {
|
TC3_LOG(ERROR) << "Could not initialize embedding executor.";
|
return false;
|
}
|
|
// Cache embedding of padding, start and end token.
|
if (!EmbedTokenId(options->padding_token_id(), &embedded_padding_token_) ||
|
!EmbedTokenId(options->start_token_id(), &embedded_start_token_) ||
|
!EmbedTokenId(options->end_token_id(), &embedded_end_token_)) {
|
TC3_LOG(ERROR) << "Could not precompute token embeddings.";
|
return false;
|
}
|
token_embedding_size_ = feature_processor_->GetTokenEmbeddingSize();
|
}
|
|
// Create low confidence model if specified.
|
if (model_->low_confidence_ngram_model() != nullptr) {
|
ngram_model_ = NGramModel::Create(model_->low_confidence_ngram_model(),
|
feature_processor_ == nullptr
|
? nullptr
|
: feature_processor_->tokenizer(),
|
unilib_);
|
if (ngram_model_ == nullptr) {
|
TC3_LOG(ERROR) << "Could not create ngram linear regression model.";
|
return false;
|
}
|
}
|
|
return true;
|
}
|
|
bool ActionsSuggestions::InitializeTriggeringPreconditions() {
|
triggering_preconditions_overlay_ =
|
LoadAndVerifyFlatbuffer<TriggeringPreconditions>(
|
triggering_preconditions_overlay_buffer_);
|
|
if (triggering_preconditions_overlay_ == nullptr &&
|
!triggering_preconditions_overlay_buffer_.empty()) {
|
TC3_LOG(ERROR) << "Could not load triggering preconditions overwrites.";
|
return false;
|
}
|
const flatbuffers::Table* overlay =
|
reinterpret_cast<const flatbuffers::Table*>(
|
triggering_preconditions_overlay_);
|
const TriggeringPreconditions* defaults = model_->preconditions();
|
if (defaults == nullptr) {
|
TC3_LOG(ERROR) << "No triggering conditions specified.";
|
return false;
|
}
|
|
preconditions_.min_smart_reply_triggering_score = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_MIN_SMART_REPLY_TRIGGERING_SCORE,
|
defaults->min_smart_reply_triggering_score());
|
preconditions_.max_sensitive_topic_score = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_MAX_SENSITIVE_TOPIC_SCORE,
|
defaults->max_sensitive_topic_score());
|
preconditions_.suppress_on_sensitive_topic = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_SUPPRESS_ON_SENSITIVE_TOPIC,
|
defaults->suppress_on_sensitive_topic());
|
preconditions_.min_input_length =
|
ValueOrDefault(overlay, TriggeringPreconditions::VT_MIN_INPUT_LENGTH,
|
defaults->min_input_length());
|
preconditions_.max_input_length =
|
ValueOrDefault(overlay, TriggeringPreconditions::VT_MAX_INPUT_LENGTH,
|
defaults->max_input_length());
|
preconditions_.min_locale_match_fraction = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_MIN_LOCALE_MATCH_FRACTION,
|
defaults->min_locale_match_fraction());
|
preconditions_.handle_missing_locale_as_supported = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_HANDLE_MISSING_LOCALE_AS_SUPPORTED,
|
defaults->handle_missing_locale_as_supported());
|
preconditions_.handle_unknown_locale_as_supported = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_HANDLE_UNKNOWN_LOCALE_AS_SUPPORTED,
|
defaults->handle_unknown_locale_as_supported());
|
preconditions_.suppress_on_low_confidence_input = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_SUPPRESS_ON_LOW_CONFIDENCE_INPUT,
|
defaults->suppress_on_low_confidence_input());
|
preconditions_.diversification_distance_threshold = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_DIVERSIFICATION_DISTANCE_THRESHOLD,
|
defaults->diversification_distance_threshold());
|
preconditions_.confidence_threshold =
|
ValueOrDefault(overlay, TriggeringPreconditions::VT_CONFIDENCE_THRESHOLD,
|
defaults->confidence_threshold());
|
preconditions_.empirical_probability_factor = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_EMPIRICAL_PROBABILITY_FACTOR,
|
defaults->empirical_probability_factor());
|
preconditions_.min_reply_score_threshold = ValueOrDefault(
|
overlay, TriggeringPreconditions::VT_MIN_REPLY_SCORE_THRESHOLD,
|
defaults->min_reply_score_threshold());
|
|
return true;
|
}
|
|
bool ActionsSuggestions::EmbedTokenId(const int32 token_id,
|
std::vector<float>* embedding) const {
|
return feature_processor_->AppendFeatures(
|
{token_id},
|
/*dense_features=*/{}, embedding_executor_.get(), embedding);
|
}
|
|
bool ActionsSuggestions::InitializeRules(ZlibDecompressor* decompressor) {
|
if (model_->rules() != nullptr) {
|
if (!InitializeRules(decompressor, model_->rules(), &rules_)) {
|
TC3_LOG(ERROR) << "Could not initialize action rules.";
|
return false;
|
}
|
}
|
|
if (model_->low_confidence_rules() != nullptr) {
|
if (!InitializeRules(decompressor, model_->low_confidence_rules(),
|
&low_confidence_rules_)) {
|
TC3_LOG(ERROR) << "Could not initialize low confidence rules.";
|
return false;
|
}
|
}
|
|
// Extend by rules provided by the overwrite.
|
// NOTE: The rules from the original models are *not* cleared.
|
if (triggering_preconditions_overlay_ != nullptr &&
|
triggering_preconditions_overlay_->low_confidence_rules() != nullptr) {
|
// These rules are optionally compressed, but separately.
|
std::unique_ptr<ZlibDecompressor> overwrite_decompressor =
|
ZlibDecompressor::Instance();
|
if (overwrite_decompressor == nullptr) {
|
TC3_LOG(ERROR) << "Could not initialze decompressor for overwrite rules.";
|
return false;
|
}
|
if (!InitializeRules(
|
overwrite_decompressor.get(),
|
triggering_preconditions_overlay_->low_confidence_rules(),
|
&low_confidence_rules_)) {
|
TC3_LOG(ERROR)
|
<< "Could not initialize low confidence rules from overwrite.";
|
return false;
|
}
|
}
|
|
return true;
|
}
|
|
bool ActionsSuggestions::InitializeRules(
|
ZlibDecompressor* decompressor, const RulesModel* rules,
|
std::vector<CompiledRule>* compiled_rules) const {
|
for (const RulesModel_::Rule* rule : *rules->rule()) {
|
std::unique_ptr<UniLib::RegexPattern> compiled_pattern =
|
UncompressMakeRegexPattern(
|
*unilib_, rule->pattern(), rule->compressed_pattern(),
|
rules->lazy_regex_compilation(), decompressor);
|
if (compiled_pattern == nullptr) {
|
TC3_LOG(ERROR) << "Failed to load rule pattern.";
|
return false;
|
}
|
|
// Check whether there is a check on the output.
|
std::unique_ptr<UniLib::RegexPattern> compiled_output_pattern;
|
if (rule->output_pattern() != nullptr ||
|
rule->compressed_output_pattern() != nullptr) {
|
compiled_output_pattern = UncompressMakeRegexPattern(
|
*unilib_, rule->output_pattern(), rule->compressed_output_pattern(),
|
rules->lazy_regex_compilation(), decompressor);
|
if (compiled_output_pattern == nullptr) {
|
TC3_LOG(ERROR) << "Failed to load rule output pattern.";
|
return false;
|
}
|
}
|
|
compiled_rules->emplace_back(rule, std::move(compiled_pattern),
|
std::move(compiled_output_pattern));
|
}
|
|
return true;
|
}
|
|
bool ActionsSuggestions::IsLowConfidenceInput(
|
const Conversation& conversation, const int num_messages,
|
std::vector<int>* post_check_rules) const {
|
for (int i = 1; i <= num_messages; i++) {
|
const std::string& message =
|
conversation.messages[conversation.messages.size() - i].text;
|
const UnicodeText message_unicode(
|
UTF8ToUnicodeText(message, /*do_copy=*/false));
|
|
// Run ngram linear regression model.
|
if (ngram_model_ != nullptr) {
|
if (ngram_model_->Eval(message_unicode)) {
|
return true;
|
}
|
}
|
|
// Run the regex based rules.
|
for (int low_confidence_rule = 0;
|
low_confidence_rule < low_confidence_rules_.size();
|
low_confidence_rule++) {
|
const CompiledRule& rule = low_confidence_rules_[low_confidence_rule];
|
const std::unique_ptr<UniLib::RegexMatcher> matcher =
|
rule.pattern->Matcher(message_unicode);
|
int status = UniLib::RegexMatcher::kNoError;
|
if (matcher->Find(&status) && status == UniLib::RegexMatcher::kNoError) {
|
// Rule only applies to input-output pairs, so defer the check.
|
if (rule.output_pattern != nullptr) {
|
post_check_rules->push_back(low_confidence_rule);
|
continue;
|
}
|
return true;
|
}
|
}
|
}
|
return false;
|
}
|
|
bool ActionsSuggestions::FilterConfidenceOutput(
|
const std::vector<int>& post_check_rules,
|
std::vector<ActionSuggestion>* actions) const {
|
if (post_check_rules.empty() || actions->empty()) {
|
return true;
|
}
|
std::vector<ActionSuggestion> filtered_text_replies;
|
for (const ActionSuggestion& action : *actions) {
|
if (action.response_text.empty()) {
|
filtered_text_replies.push_back(action);
|
continue;
|
}
|
bool passes_post_check = true;
|
const UnicodeText text_reply_unicode(
|
UTF8ToUnicodeText(action.response_text, /*do_copy=*/false));
|
for (const int rule_id : post_check_rules) {
|
const std::unique_ptr<UniLib::RegexMatcher> matcher =
|
low_confidence_rules_[rule_id].output_pattern->Matcher(
|
text_reply_unicode);
|
if (matcher == nullptr) {
|
TC3_LOG(ERROR) << "Could not create matcher for post check rule.";
|
return false;
|
}
|
int status = UniLib::RegexMatcher::kNoError;
|
if (matcher->Find(&status) || status != UniLib::RegexMatcher::kNoError) {
|
passes_post_check = false;
|
break;
|
}
|
}
|
if (passes_post_check) {
|
filtered_text_replies.push_back(action);
|
}
|
}
|
*actions = std::move(filtered_text_replies);
|
return true;
|
}
|
|
ActionSuggestion ActionsSuggestions::SuggestionFromSpec(
|
const ActionSuggestionSpec* action, const std::string& default_type,
|
const std::string& default_response_text,
|
const std::string& default_serialized_entity_data,
|
const float default_score, const float default_priority_score) const {
|
ActionSuggestion suggestion;
|
suggestion.score = action != nullptr ? action->score() : default_score;
|
suggestion.priority_score =
|
action != nullptr ? action->priority_score() : default_priority_score;
|
suggestion.type = action != nullptr && action->type() != nullptr
|
? action->type()->str()
|
: default_type;
|
suggestion.response_text =
|
action != nullptr && action->response_text() != nullptr
|
? action->response_text()->str()
|
: default_response_text;
|
suggestion.serialized_entity_data =
|
action != nullptr && action->serialized_entity_data() != nullptr
|
? action->serialized_entity_data()->str()
|
: default_serialized_entity_data;
|
return suggestion;
|
}
|
|
std::vector<std::vector<Token>> ActionsSuggestions::Tokenize(
|
const std::vector<std::string>& context) const {
|
std::vector<std::vector<Token>> tokens;
|
tokens.reserve(context.size());
|
for (const std::string& message : context) {
|
tokens.push_back(feature_processor_->tokenizer()->Tokenize(message));
|
}
|
return tokens;
|
}
|
|
bool ActionsSuggestions::EmbedTokensPerMessage(
|
const std::vector<std::vector<Token>>& tokens,
|
std::vector<float>* embeddings, int* max_num_tokens_per_message) const {
|
const int num_messages = tokens.size();
|
*max_num_tokens_per_message = 0;
|
for (int i = 0; i < num_messages; i++) {
|
const int num_message_tokens = tokens[i].size();
|
if (num_message_tokens > *max_num_tokens_per_message) {
|
*max_num_tokens_per_message = num_message_tokens;
|
}
|
}
|
|
if (model_->feature_processor_options()->min_num_tokens_per_message() >
|
*max_num_tokens_per_message) {
|
*max_num_tokens_per_message =
|
model_->feature_processor_options()->min_num_tokens_per_message();
|
}
|
if (model_->feature_processor_options()->max_num_tokens_per_message() > 0 &&
|
*max_num_tokens_per_message >
|
model_->feature_processor_options()->max_num_tokens_per_message()) {
|
*max_num_tokens_per_message =
|
model_->feature_processor_options()->max_num_tokens_per_message();
|
}
|
|
// Embed all tokens and add paddings to pad tokens of each message to the
|
// maximum number of tokens in a message of the conversation.
|
// If a number of tokens is specified in the model config, tokens at the
|
// beginning of a message are dropped if they don't fit in the limit.
|
for (int i = 0; i < num_messages; i++) {
|
const int start =
|
std::max<int>(tokens[i].size() - *max_num_tokens_per_message, 0);
|
for (int pos = start; pos < tokens[i].size(); pos++) {
|
if (!feature_processor_->AppendTokenFeatures(
|
tokens[i][pos], embedding_executor_.get(), embeddings)) {
|
TC3_LOG(ERROR) << "Could not run token feature extractor.";
|
return false;
|
}
|
}
|
// Add padding.
|
for (int k = tokens[i].size(); k < *max_num_tokens_per_message; k++) {
|
embeddings->insert(embeddings->end(), embedded_padding_token_.begin(),
|
embedded_padding_token_.end());
|
}
|
}
|
|
return true;
|
}
|
|
bool ActionsSuggestions::EmbedAndFlattenTokens(
|
const std::vector<std::vector<Token>> tokens,
|
std::vector<float>* embeddings, int* total_token_count) const {
|
const int num_messages = tokens.size();
|
int start_message = 0;
|
int message_token_offset = 0;
|
|
// If a maximum model input length is specified, we need to check how
|
// much we need to trim at the start.
|
const int max_num_total_tokens =
|
model_->feature_processor_options()->max_num_total_tokens();
|
if (max_num_total_tokens > 0) {
|
int total_tokens = 0;
|
start_message = num_messages - 1;
|
for (; start_message >= 0; start_message--) {
|
// Tokens of the message + start and end token.
|
const int num_message_tokens = tokens[start_message].size() + 2;
|
total_tokens += num_message_tokens;
|
|
// Check whether we exhausted the budget.
|
if (total_tokens >= max_num_total_tokens) {
|
message_token_offset = total_tokens - max_num_total_tokens;
|
break;
|
}
|
}
|
}
|
|
// Add embeddings.
|
*total_token_count = 0;
|
for (int i = start_message; i < num_messages; i++) {
|
if (message_token_offset == 0) {
|
++(*total_token_count);
|
// Add `start message` token.
|
embeddings->insert(embeddings->end(), embedded_start_token_.begin(),
|
embedded_start_token_.end());
|
}
|
|
for (int pos = std::max(0, message_token_offset - 1);
|
pos < tokens[i].size(); pos++) {
|
++(*total_token_count);
|
if (!feature_processor_->AppendTokenFeatures(
|
tokens[i][pos], embedding_executor_.get(), embeddings)) {
|
TC3_LOG(ERROR) << "Could not run token feature extractor.";
|
return false;
|
}
|
}
|
|
// Add `end message` token.
|
++(*total_token_count);
|
embeddings->insert(embeddings->end(), embedded_end_token_.begin(),
|
embedded_end_token_.end());
|
|
// Reset for the subsequent messages.
|
message_token_offset = 0;
|
}
|
|
// Add optional padding.
|
const int min_num_total_tokens =
|
model_->feature_processor_options()->min_num_total_tokens();
|
for (; *total_token_count < min_num_total_tokens; ++(*total_token_count)) {
|
embeddings->insert(embeddings->end(), embedded_padding_token_.begin(),
|
embedded_padding_token_.end());
|
}
|
|
return true;
|
}
|
|
bool ActionsSuggestions::AllocateInput(const int conversation_length,
|
const int max_tokens,
|
const int total_token_count,
|
tflite::Interpreter* interpreter) const {
|
if (model_->tflite_model_spec()->resize_inputs()) {
|
if (model_->tflite_model_spec()->input_context() >= 0) {
|
interpreter->ResizeInputTensor(
|
interpreter->inputs()[model_->tflite_model_spec()->input_context()],
|
{1, conversation_length});
|
}
|
if (model_->tflite_model_spec()->input_user_id() >= 0) {
|
interpreter->ResizeInputTensor(
|
interpreter->inputs()[model_->tflite_model_spec()->input_user_id()],
|
{1, conversation_length});
|
}
|
if (model_->tflite_model_spec()->input_time_diffs() >= 0) {
|
interpreter->ResizeInputTensor(
|
interpreter
|
->inputs()[model_->tflite_model_spec()->input_time_diffs()],
|
{1, conversation_length});
|
}
|
if (model_->tflite_model_spec()->input_num_tokens() >= 0) {
|
interpreter->ResizeInputTensor(
|
interpreter
|
->inputs()[model_->tflite_model_spec()->input_num_tokens()],
|
{conversation_length, 1});
|
}
|
if (model_->tflite_model_spec()->input_token_embeddings() >= 0) {
|
interpreter->ResizeInputTensor(
|
interpreter
|
->inputs()[model_->tflite_model_spec()->input_token_embeddings()],
|
{conversation_length, max_tokens, token_embedding_size_});
|
}
|
if (model_->tflite_model_spec()->input_flattened_token_embeddings() >= 0) {
|
interpreter->ResizeInputTensor(
|
interpreter->inputs()[model_->tflite_model_spec()
|
->input_flattened_token_embeddings()],
|
{1, total_token_count});
|
}
|
}
|
|
return interpreter->AllocateTensors() == kTfLiteOk;
|
}
|
|
bool ActionsSuggestions::SetupModelInput(
|
const std::vector<std::string>& context, const std::vector<int>& user_ids,
|
const std::vector<float>& time_diffs, const int num_suggestions,
|
const float confidence_threshold, const float diversification_distance,
|
const float empirical_probability_factor,
|
tflite::Interpreter* interpreter) const {
|
// Compute token embeddings.
|
std::vector<std::vector<Token>> tokens;
|
std::vector<float> token_embeddings;
|
std::vector<float> flattened_token_embeddings;
|
int max_tokens = 0;
|
int total_token_count = 0;
|
if (model_->tflite_model_spec()->input_num_tokens() >= 0 ||
|
model_->tflite_model_spec()->input_token_embeddings() >= 0 ||
|
model_->tflite_model_spec()->input_flattened_token_embeddings() >= 0) {
|
if (feature_processor_ == nullptr) {
|
TC3_LOG(ERROR) << "No feature processor specified.";
|
return false;
|
}
|
|
// Tokenize the messages in the conversation.
|
tokens = Tokenize(context);
|
if (model_->tflite_model_spec()->input_token_embeddings() >= 0) {
|
if (!EmbedTokensPerMessage(tokens, &token_embeddings, &max_tokens)) {
|
TC3_LOG(ERROR) << "Could not extract token features.";
|
return false;
|
}
|
}
|
if (model_->tflite_model_spec()->input_flattened_token_embeddings() >= 0) {
|
if (!EmbedAndFlattenTokens(tokens, &flattened_token_embeddings,
|
&total_token_count)) {
|
TC3_LOG(ERROR) << "Could not extract token features.";
|
return false;
|
}
|
}
|
}
|
|
if (!AllocateInput(context.size(), max_tokens, total_token_count,
|
interpreter)) {
|
TC3_LOG(ERROR) << "TensorFlow Lite model allocation failed.";
|
return false;
|
}
|
if (model_->tflite_model_spec()->input_context() >= 0) {
|
model_executor_->SetInput<std::string>(
|
model_->tflite_model_spec()->input_context(), context, interpreter);
|
}
|
if (model_->tflite_model_spec()->input_context_length() >= 0) {
|
model_executor_->SetInput<int>(
|
model_->tflite_model_spec()->input_context_length(), context.size(),
|
interpreter);
|
}
|
if (model_->tflite_model_spec()->input_user_id() >= 0) {
|
model_executor_->SetInput<int>(model_->tflite_model_spec()->input_user_id(),
|
user_ids, interpreter);
|
}
|
if (model_->tflite_model_spec()->input_num_suggestions() >= 0) {
|
model_executor_->SetInput<int>(
|
model_->tflite_model_spec()->input_num_suggestions(), num_suggestions,
|
interpreter);
|
}
|
if (model_->tflite_model_spec()->input_time_diffs() >= 0) {
|
model_executor_->SetInput<float>(
|
model_->tflite_model_spec()->input_time_diffs(), time_diffs,
|
interpreter);
|
}
|
if (model_->tflite_model_spec()->input_diversification_distance() >= 0) {
|
model_executor_->SetInput<float>(
|
model_->tflite_model_spec()->input_diversification_distance(),
|
diversification_distance, interpreter);
|
}
|
if (model_->tflite_model_spec()->input_confidence_threshold() >= 0) {
|
model_executor_->SetInput<float>(
|
model_->tflite_model_spec()->input_confidence_threshold(),
|
confidence_threshold, interpreter);
|
}
|
if (model_->tflite_model_spec()->input_empirical_probability_factor() >= 0) {
|
model_executor_->SetInput<float>(
|
model_->tflite_model_spec()->input_empirical_probability_factor(),
|
confidence_threshold, interpreter);
|
}
|
if (model_->tflite_model_spec()->input_num_tokens() >= 0) {
|
std::vector<int> num_tokens_per_message(tokens.size());
|
for (int i = 0; i < tokens.size(); i++) {
|
num_tokens_per_message[i] = tokens[i].size();
|
}
|
model_executor_->SetInput<int>(
|
model_->tflite_model_spec()->input_num_tokens(), num_tokens_per_message,
|
interpreter);
|
}
|
if (model_->tflite_model_spec()->input_token_embeddings() >= 0) {
|
model_executor_->SetInput<float>(
|
model_->tflite_model_spec()->input_token_embeddings(), token_embeddings,
|
interpreter);
|
}
|
if (model_->tflite_model_spec()->input_flattened_token_embeddings() >= 0) {
|
model_executor_->SetInput<float>(
|
model_->tflite_model_spec()->input_flattened_token_embeddings(),
|
flattened_token_embeddings, interpreter);
|
}
|
return true;
|
}
|
|
bool ActionsSuggestions::ReadModelOutput(
|
tflite::Interpreter* interpreter, const ActionSuggestionOptions& options,
|
ActionsSuggestionsResponse* response) const {
|
// Read sensitivity and triggering score predictions.
|
if (model_->tflite_model_spec()->output_triggering_score() >= 0) {
|
const TensorView<float>& triggering_score =
|
model_executor_->OutputView<float>(
|
model_->tflite_model_spec()->output_triggering_score(),
|
interpreter);
|
if (!triggering_score.is_valid() || triggering_score.size() == 0) {
|
TC3_LOG(ERROR) << "Could not compute triggering score.";
|
return false;
|
}
|
response->triggering_score = triggering_score.data()[0];
|
response->output_filtered_min_triggering_score =
|
(response->triggering_score <
|
preconditions_.min_smart_reply_triggering_score);
|
}
|
if (model_->tflite_model_spec()->output_sensitive_topic_score() >= 0) {
|
const TensorView<float>& sensitive_topic_score =
|
model_executor_->OutputView<float>(
|
model_->tflite_model_spec()->output_sensitive_topic_score(),
|
interpreter);
|
if (!sensitive_topic_score.is_valid() ||
|
sensitive_topic_score.dim(0) != 1) {
|
TC3_LOG(ERROR) << "Could not compute sensitive topic score.";
|
return false;
|
}
|
response->sensitivity_score = sensitive_topic_score.data()[0];
|
response->output_filtered_sensitivity =
|
(response->sensitivity_score >
|
preconditions_.max_sensitive_topic_score);
|
}
|
|
// Suppress model outputs.
|
if (response->output_filtered_sensitivity) {
|
return true;
|
}
|
|
// Read smart reply predictions.
|
std::vector<ActionSuggestion> text_replies;
|
if (!response->output_filtered_min_triggering_score &&
|
model_->tflite_model_spec()->output_replies() >= 0) {
|
const std::vector<tflite::StringRef> replies =
|
model_executor_->Output<tflite::StringRef>(
|
model_->tflite_model_spec()->output_replies(), interpreter);
|
TensorView<float> scores = model_executor_->OutputView<float>(
|
model_->tflite_model_spec()->output_replies_scores(), interpreter);
|
for (int i = 0; i < replies.size(); i++) {
|
if (replies[i].len == 0) continue;
|
const float score = scores.data()[i];
|
if (score < preconditions_.min_reply_score_threshold) {
|
continue;
|
}
|
response->actions.push_back({std::string(replies[i].str, replies[i].len),
|
model_->smart_reply_action_type()->str(),
|
score});
|
}
|
}
|
|
// Read actions suggestions.
|
if (model_->tflite_model_spec()->output_actions_scores() >= 0) {
|
const TensorView<float> actions_scores = model_executor_->OutputView<float>(
|
model_->tflite_model_spec()->output_actions_scores(), interpreter);
|
for (int i = 0; i < model_->action_type()->Length(); i++) {
|
const ActionTypeOptions* action_type = model_->action_type()->Get(i);
|
// Skip disabled action classes, such as the default other category.
|
if (!action_type->enabled()) {
|
continue;
|
}
|
const float score = actions_scores.data()[i];
|
if (score < action_type->min_triggering_score()) {
|
continue;
|
}
|
ActionSuggestion suggestion =
|
SuggestionFromSpec(action_type->action(),
|
/*default_type=*/action_type->name()->str());
|
suggestion.score = score;
|
response->actions.push_back(suggestion);
|
}
|
}
|
|
return true;
|
}
|
|
bool ActionsSuggestions::SuggestActionsFromModel(
|
const Conversation& conversation, const int num_messages,
|
const ActionSuggestionOptions& options,
|
ActionsSuggestionsResponse* response,
|
std::unique_ptr<tflite::Interpreter>* interpreter) const {
|
TC3_CHECK_LE(num_messages, conversation.messages.size());
|
|
if (!model_executor_) {
|
return true;
|
}
|
*interpreter = model_executor_->CreateInterpreter();
|
|
if (!*interpreter) {
|
TC3_LOG(ERROR) << "Could not build TensorFlow Lite interpreter for the "
|
"actions suggestions model.";
|
return false;
|
}
|
|
std::vector<std::string> context;
|
std::vector<int> user_ids;
|
std::vector<float> time_diffs;
|
context.reserve(num_messages);
|
user_ids.reserve(num_messages);
|
time_diffs.reserve(num_messages);
|
|
// Gather last `num_messages` messages from the conversation.
|
int64 last_message_reference_time_ms_utc = 0;
|
const float second_in_ms = 1000;
|
for (int i = conversation.messages.size() - num_messages;
|
i < conversation.messages.size(); i++) {
|
const ConversationMessage& message = conversation.messages[i];
|
context.push_back(message.text);
|
user_ids.push_back(message.user_id);
|
|
float time_diff_secs = 0;
|
if (message.reference_time_ms_utc != 0 &&
|
last_message_reference_time_ms_utc != 0) {
|
time_diff_secs = std::max(0.0f, (message.reference_time_ms_utc -
|
last_message_reference_time_ms_utc) /
|
second_in_ms);
|
}
|
if (message.reference_time_ms_utc != 0) {
|
last_message_reference_time_ms_utc = message.reference_time_ms_utc;
|
}
|
time_diffs.push_back(time_diff_secs);
|
}
|
|
if (!SetupModelInput(context, user_ids, time_diffs,
|
/*num_suggestions=*/model_->num_smart_replies(),
|
preconditions_.confidence_threshold,
|
preconditions_.diversification_distance_threshold,
|
preconditions_.empirical_probability_factor,
|
interpreter->get())) {
|
TC3_LOG(ERROR) << "Failed to setup input for TensorFlow Lite model.";
|
return false;
|
}
|
|
if ((*interpreter)->Invoke() != kTfLiteOk) {
|
TC3_LOG(ERROR) << "Failed to invoke TensorFlow Lite interpreter.";
|
return false;
|
}
|
|
return ReadModelOutput(interpreter->get(), options, response);
|
}
|
|
AnnotationOptions ActionsSuggestions::AnnotationOptionsForMessage(
|
const ConversationMessage& message) const {
|
AnnotationOptions options;
|
options.detected_text_language_tags = message.detected_text_language_tags;
|
options.reference_time_ms_utc = message.reference_time_ms_utc;
|
options.reference_timezone = message.reference_timezone;
|
options.annotation_usecase =
|
model_->annotation_actions_spec()->annotation_usecase();
|
options.is_serialized_entity_data_enabled =
|
model_->annotation_actions_spec()->is_serialized_entity_data_enabled();
|
options.entity_types = annotation_entity_types_;
|
return options;
|
}
|
|
void ActionsSuggestions::SuggestActionsFromAnnotations(
|
const Conversation& conversation, const ActionSuggestionOptions& options,
|
const Annotator* annotator, std::vector<ActionSuggestion>* actions) const {
|
if (model_->annotation_actions_spec() == nullptr ||
|
model_->annotation_actions_spec()->annotation_mapping() == nullptr ||
|
model_->annotation_actions_spec()->annotation_mapping()->size() == 0) {
|
return;
|
}
|
|
// Create actions based on the annotations.
|
const int max_from_any_person =
|
model_->annotation_actions_spec()->max_history_from_any_person();
|
const int max_from_last_person =
|
model_->annotation_actions_spec()->max_history_from_last_person();
|
const int last_person = conversation.messages.back().user_id;
|
|
int num_messages_last_person = 0;
|
int num_messages_any_person = 0;
|
bool all_from_last_person = true;
|
for (int message_index = conversation.messages.size() - 1; message_index >= 0;
|
message_index--) {
|
const ConversationMessage& message = conversation.messages[message_index];
|
std::vector<AnnotatedSpan> annotations = message.annotations;
|
|
// Update how many messages we have processed from the last person in the
|
// conversation and from any person in the conversation.
|
num_messages_any_person++;
|
if (all_from_last_person && message.user_id == last_person) {
|
num_messages_last_person++;
|
} else {
|
all_from_last_person = false;
|
}
|
|
if (num_messages_any_person > max_from_any_person &&
|
(!all_from_last_person ||
|
num_messages_last_person > max_from_last_person)) {
|
break;
|
}
|
|
if (message.user_id == kLocalUserId) {
|
if (model_->annotation_actions_spec()->only_until_last_sent()) {
|
break;
|
}
|
if (!model_->annotation_actions_spec()->include_local_user_messages()) {
|
continue;
|
}
|
}
|
|
if (annotations.empty() && annotator != nullptr) {
|
annotations = annotator->Annotate(message.text,
|
AnnotationOptionsForMessage(message));
|
}
|
std::vector<ActionSuggestionAnnotation> action_annotations;
|
action_annotations.reserve(annotations.size());
|
for (const AnnotatedSpan& annotation : annotations) {
|
if (annotation.classification.empty()) {
|
continue;
|
}
|
|
const ClassificationResult& classification_result =
|
annotation.classification[0];
|
|
ActionSuggestionAnnotation action_annotation;
|
action_annotation.span = {
|
message_index, annotation.span,
|
UTF8ToUnicodeText(message.text, /*do_copy=*/false)
|
.UTF8Substring(annotation.span.first, annotation.span.second)};
|
action_annotation.entity = classification_result;
|
action_annotation.name = classification_result.collection;
|
action_annotations.push_back(action_annotation);
|
}
|
|
if (model_->annotation_actions_spec()->deduplicate_annotations()) {
|
// Create actions only for deduplicated annotations.
|
for (const int annotation_id :
|
DeduplicateAnnotations(action_annotations)) {
|
SuggestActionsFromAnnotation(
|
message_index, action_annotations[annotation_id], actions);
|
}
|
} else {
|
// Create actions for all annotations.
|
for (const ActionSuggestionAnnotation& annotation : action_annotations) {
|
SuggestActionsFromAnnotation(message_index, annotation, actions);
|
}
|
}
|
}
|
}
|
|
void ActionsSuggestions::SuggestActionsFromAnnotation(
|
const int message_index, const ActionSuggestionAnnotation& annotation,
|
std::vector<ActionSuggestion>* actions) const {
|
for (const AnnotationActionsSpec_::AnnotationMapping* mapping :
|
*model_->annotation_actions_spec()->annotation_mapping()) {
|
if (annotation.entity.collection ==
|
mapping->annotation_collection()->str()) {
|
if (annotation.entity.score < mapping->min_annotation_score()) {
|
continue;
|
}
|
ActionSuggestion suggestion = SuggestionFromSpec(mapping->action());
|
if (mapping->use_annotation_score()) {
|
suggestion.score = annotation.entity.score;
|
}
|
|
// Set annotation text as (additional) entity data field.
|
if (mapping->entity_field() != nullptr) {
|
std::unique_ptr<ReflectiveFlatbuffer> entity_data =
|
entity_data_builder_->NewRoot();
|
TC3_CHECK(entity_data != nullptr);
|
|
// Merge existing static entity data.
|
if (!suggestion.serialized_entity_data.empty()) {
|
entity_data->MergeFromSerializedFlatbuffer(
|
StringPiece(suggestion.serialized_entity_data.c_str(),
|
suggestion.serialized_entity_data.size()));
|
}
|
|
entity_data->ParseAndSet(mapping->entity_field(), annotation.span.text);
|
suggestion.serialized_entity_data = entity_data->Serialize();
|
}
|
|
suggestion.annotations = {annotation};
|
actions->push_back(suggestion);
|
}
|
}
|
}
|
|
std::vector<int> ActionsSuggestions::DeduplicateAnnotations(
|
const std::vector<ActionSuggestionAnnotation>& annotations) const {
|
std::map<std::pair<std::string, std::string>, int> deduplicated_annotations;
|
|
for (int i = 0; i < annotations.size(); i++) {
|
const std::pair<std::string, std::string> key = {annotations[i].name,
|
annotations[i].span.text};
|
auto entry = deduplicated_annotations.find(key);
|
if (entry != deduplicated_annotations.end()) {
|
// Kepp the annotation with the higher score.
|
if (annotations[entry->second].entity.score <
|
annotations[i].entity.score) {
|
entry->second = i;
|
}
|
continue;
|
}
|
deduplicated_annotations.insert(entry, {key, i});
|
}
|
|
std::vector<int> result;
|
result.reserve(deduplicated_annotations.size());
|
for (const auto& key_and_annotation : deduplicated_annotations) {
|
result.push_back(key_and_annotation.second);
|
}
|
return result;
|
}
|
|
bool ActionsSuggestions::FillAnnotationFromMatchGroup(
|
const UniLib::RegexMatcher* matcher,
|
const RulesModel_::Rule_::RuleActionSpec_::RuleCapturingGroup* group,
|
const int message_index, ActionSuggestionAnnotation* annotation) const {
|
if (group->annotation_name() != nullptr ||
|
group->annotation_type() != nullptr) {
|
int status = UniLib::RegexMatcher::kNoError;
|
const CodepointSpan span = {matcher->Start(group->group_id(), &status),
|
matcher->End(group->group_id(), &status)};
|
std::string text =
|
matcher->Group(group->group_id(), &status).ToUTF8String();
|
if (status != UniLib::RegexMatcher::kNoError) {
|
TC3_LOG(ERROR) << "Could not extract span from rule capturing group.";
|
return false;
|
}
|
|
// The capturing group was not part of the match.
|
if (span.first == kInvalidIndex || span.second == kInvalidIndex) {
|
return false;
|
}
|
annotation->span.span = span;
|
annotation->span.message_index = message_index;
|
annotation->span.text = text;
|
if (group->annotation_name() != nullptr) {
|
annotation->name = group->annotation_name()->str();
|
}
|
if (group->annotation_type() != nullptr) {
|
annotation->entity.collection = group->annotation_type()->str();
|
}
|
}
|
return true;
|
}
|
|
bool ActionsSuggestions::SuggestActionsFromRules(
|
const Conversation& conversation,
|
std::vector<ActionSuggestion>* actions) const {
|
// Create actions based on rules checking the last message.
|
const int message_index = conversation.messages.size() - 1;
|
const std::string& message = conversation.messages.back().text;
|
const UnicodeText message_unicode(
|
UTF8ToUnicodeText(message, /*do_copy=*/false));
|
for (const CompiledRule& rule : rules_) {
|
const std::unique_ptr<UniLib::RegexMatcher> matcher =
|
rule.pattern->Matcher(message_unicode);
|
int status = UniLib::RegexMatcher::kNoError;
|
while (matcher->Find(&status) && status == UniLib::RegexMatcher::kNoError) {
|
for (const RulesModel_::Rule_::RuleActionSpec* rule_action :
|
*rule.rule->actions()) {
|
const ActionSuggestionSpec* action = rule_action->action();
|
std::vector<ActionSuggestionAnnotation> annotations;
|
|
bool sets_entity_data = false;
|
std::unique_ptr<ReflectiveFlatbuffer> entity_data =
|
entity_data_builder_ != nullptr ? entity_data_builder_->NewRoot()
|
: nullptr;
|
|
// Set static entity data.
|
if (action != nullptr && action->serialized_entity_data() != nullptr) {
|
TC3_CHECK(entity_data != nullptr);
|
sets_entity_data = true;
|
entity_data->MergeFromSerializedFlatbuffer(
|
StringPiece(action->serialized_entity_data()->c_str(),
|
action->serialized_entity_data()->size()));
|
}
|
|
// Add entity data from rule capturing groups.
|
if (rule_action->capturing_group() != nullptr) {
|
for (const RulesModel_::Rule_::RuleActionSpec_::RuleCapturingGroup*
|
group : *rule_action->capturing_group()) {
|
if (group->entity_field() != nullptr) {
|
TC3_CHECK(entity_data != nullptr);
|
sets_entity_data = true;
|
if (!SetFieldFromCapturingGroup(
|
group->group_id(), group->entity_field(), matcher.get(),
|
entity_data.get())) {
|
TC3_LOG(ERROR)
|
<< "Could not set entity data from rule capturing group.";
|
return false;
|
}
|
}
|
|
// Create a text annotation for the group span.
|
ActionSuggestionAnnotation annotation;
|
if (FillAnnotationFromMatchGroup(matcher.get(), group,
|
message_index, &annotation)) {
|
annotations.push_back(annotation);
|
}
|
|
// Create text reply.
|
if (group->text_reply() != nullptr) {
|
int status = UniLib::RegexMatcher::kNoError;
|
const std::string group_text =
|
matcher->Group(group->group_id(), &status).ToUTF8String();
|
if (status != UniLib::RegexMatcher::kNoError) {
|
TC3_LOG(ERROR) << "Could get text from capturing group.";
|
return false;
|
}
|
if (group_text.empty()) {
|
// The group was not part of the match, ignore and continue.
|
continue;
|
}
|
actions->push_back(SuggestionFromSpec(
|
group->text_reply(),
|
/*default_type=*/model_->smart_reply_action_type()->str(),
|
/*default_response_text=*/group_text));
|
}
|
}
|
}
|
|
if (action != nullptr) {
|
ActionSuggestion suggestion = SuggestionFromSpec(action);
|
suggestion.annotations = annotations;
|
if (sets_entity_data) {
|
suggestion.serialized_entity_data = entity_data->Serialize();
|
}
|
actions->push_back(suggestion);
|
}
|
}
|
}
|
}
|
return true;
|
}
|
|
bool ActionsSuggestions::SuggestActionsFromLua(
|
const Conversation& conversation, const TfLiteModelExecutor* model_executor,
|
const tflite::Interpreter* interpreter,
|
const reflection::Schema* annotation_entity_data_schema,
|
std::vector<ActionSuggestion>* actions) const {
|
if (lua_bytecode_.empty()) {
|
return true;
|
}
|
|
auto lua_actions = LuaActionsSuggestions::CreateLuaActionsSuggestions(
|
lua_bytecode_, conversation, model_executor, model_->tflite_model_spec(),
|
interpreter, entity_data_schema_, annotation_entity_data_schema);
|
if (lua_actions == nullptr) {
|
TC3_LOG(ERROR) << "Could not create lua actions.";
|
return false;
|
}
|
return lua_actions->SuggestActions(actions);
|
}
|
|
bool ActionsSuggestions::GatherActionsSuggestions(
|
const Conversation& conversation, const Annotator* annotator,
|
const ActionSuggestionOptions& options,
|
ActionsSuggestionsResponse* response) const {
|
if (conversation.messages.empty()) {
|
return true;
|
}
|
|
const int num_messages = NumMessagesToConsider(
|
conversation, model_->max_conversation_history_length());
|
|
if (num_messages <= 0) {
|
TC3_LOG(INFO) << "No messages provided for actions suggestions.";
|
return false;
|
}
|
|
SuggestActionsFromAnnotations(conversation, options, annotator,
|
&response->actions);
|
|
int input_text_length = 0;
|
int num_matching_locales = 0;
|
for (int i = conversation.messages.size() - num_messages;
|
i < conversation.messages.size(); i++) {
|
input_text_length += conversation.messages[i].text.length();
|
std::vector<Locale> message_languages;
|
if (!ParseLocales(conversation.messages[i].detected_text_language_tags,
|
&message_languages)) {
|
continue;
|
}
|
if (Locale::IsAnyLocaleSupported(
|
message_languages, locales_,
|
preconditions_.handle_unknown_locale_as_supported)) {
|
++num_matching_locales;
|
}
|
}
|
|
// Bail out if we are provided with too few or too much input.
|
if (input_text_length < preconditions_.min_input_length ||
|
(preconditions_.max_input_length >= 0 &&
|
input_text_length > preconditions_.max_input_length)) {
|
TC3_LOG(INFO) << "Too much or not enough input for inference.";
|
return response;
|
}
|
|
// Bail out if the text does not look like it can be handled by the model.
|
const float matching_fraction =
|
static_cast<float>(num_matching_locales) / num_messages;
|
if (matching_fraction < preconditions_.min_locale_match_fraction) {
|
TC3_LOG(INFO) << "Not enough locale matches.";
|
response->output_filtered_locale_mismatch = true;
|
return true;
|
}
|
|
std::vector<int> post_check_rules;
|
if (preconditions_.suppress_on_low_confidence_input &&
|
IsLowConfidenceInput(conversation, num_messages, &post_check_rules)) {
|
response->output_filtered_low_confidence = true;
|
return true;
|
}
|
|
std::unique_ptr<tflite::Interpreter> interpreter;
|
if (!SuggestActionsFromModel(conversation, num_messages, options, response,
|
&interpreter)) {
|
TC3_LOG(ERROR) << "Could not run model.";
|
return false;
|
}
|
|
// Suppress all predictions if the conversation was deemed sensitive.
|
if (preconditions_.suppress_on_sensitive_topic &&
|
response->output_filtered_sensitivity) {
|
return true;
|
}
|
|
if (!SuggestActionsFromLua(
|
conversation, model_executor_.get(), interpreter.get(),
|
annotator != nullptr ? annotator->entity_data_schema() : nullptr,
|
&response->actions)) {
|
TC3_LOG(ERROR) << "Could not suggest actions from script.";
|
return false;
|
}
|
|
if (!SuggestActionsFromRules(conversation, &response->actions)) {
|
TC3_LOG(ERROR) << "Could not suggest actions from rules.";
|
return false;
|
}
|
|
if (preconditions_.suppress_on_low_confidence_input &&
|
!FilterConfidenceOutput(post_check_rules, &response->actions)) {
|
TC3_LOG(ERROR) << "Could not post-check actions.";
|
return false;
|
}
|
|
return true;
|
}
|
|
ActionsSuggestionsResponse ActionsSuggestions::SuggestActions(
|
const Conversation& conversation, const Annotator* annotator,
|
const ActionSuggestionOptions& options) const {
|
ActionsSuggestionsResponse response;
|
if (!GatherActionsSuggestions(conversation, annotator, options, &response)) {
|
TC3_LOG(ERROR) << "Could not gather actions suggestions.";
|
response.actions.clear();
|
} else if (!ranker_->RankActions(conversation, &response, entity_data_schema_,
|
annotator != nullptr
|
? annotator->entity_data_schema()
|
: nullptr)) {
|
TC3_LOG(ERROR) << "Could not rank actions.";
|
response.actions.clear();
|
}
|
return response;
|
}
|
|
ActionsSuggestionsResponse ActionsSuggestions::SuggestActions(
|
const Conversation& conversation,
|
const ActionSuggestionOptions& options) const {
|
return SuggestActions(conversation, /*annotator=*/nullptr, options);
|
}
|
|
const ActionsModel* ActionsSuggestions::model() const { return model_; }
|
const reflection::Schema* ActionsSuggestions::entity_data_schema() const {
|
return entity_data_schema_;
|
}
|
|
const ActionsModel* ViewActionsModel(const void* buffer, int size) {
|
if (buffer == nullptr) {
|
return nullptr;
|
}
|
return LoadAndVerifyModel(reinterpret_cast<const uint8_t*>(buffer), size);
|
}
|
|
} // namespace libtextclassifier3
|
