Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow fast word piece tokenizer to take in external word piece model. #1327

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Allow fast word piece tokenizer to take in external word piece model. #1327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion tensorflow_text/core/kernels/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ load("//tensorflow_text:tftext.bzl", "tf_cc_library", "tflite_cc_library")
licenses(["notice"])

# Visibility rules
package(default_visibility = ["//visibility:public"])
package(default_visibility = [
"//inputmethod/keyboard:__subpackages__",
"//visibility:public",
])

exports_files(["LICENSE"])

Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/ops/regex_split_ops.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
namespace tensorflow {
namespace text {

Status RegexSplitOpShape(shape_inference::InferenceContext* c) {
absl::Status RegexSplitOpShape(shape_inference::InferenceContext* c) {
shape_inference::ShapeHandle unused;
TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &unused));
TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
Expand Down
4 changes: 2 additions & 2 deletions tensorflow_text/core/ops/rouge_l_op.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ using shape_inference::DimensionHandle;
using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

Status RougeLShapeFn(InferenceContext* c);
absl::Status RougeLShapeFn(InferenceContext* c);

REGISTER_OP("RougeL")
.Input("hyp_values: Tvalues")
Expand Down Expand Up @@ -67,7 +67,7 @@ p_measure: a 1D Tensor of shape [S-1] containing LCS P-measure scores
r_measure: a 1D Tensor of shape [S-1] containing LCS R-measure scores
)doc");

Status RougeLShapeFn(InferenceContext* c) {
absl::Status RougeLShapeFn(InferenceContext* c) {
ShapeHandle unused;

// Check rank of inner values
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/ops/sentence_breaking_ops.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
namespace tensorflow {
namespace text {

Status SentenceFragmentShapeFn(
absl::Status SentenceFragmentShapeFn(
::tensorflow::shape_inference::InferenceContext* c) {
for (int i = 0; i < c->num_outputs(); ++i) {
c->set_output(i, c->UnknownShapeOfRank(1));
Expand Down
4 changes: 2 additions & 2 deletions tensorflow_text/core/ops/split_merge_tokenize_op.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ using shape_inference::DimensionHandle;
using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

Status SplitMergeTokenizeWithOffsetsShapeFn(InferenceContext* c);
absl::Status SplitMergeTokenizeWithOffsetsShapeFn(InferenceContext* c);

REGISTER_OP("SplitMergeTokenizeWithOffsets")
.Input("input_values: string")
Expand Down Expand Up @@ -76,7 +76,7 @@ REGISTER_OP("SplitMergeTokenizeWithOffsets")
A 2D RaggedTensor can be constructed from this and output_row_splits.
)doc");

Status SplitMergeTokenizeWithOffsetsShapeFn(InferenceContext* c) {
absl::Status SplitMergeTokenizeWithOffsetsShapeFn(InferenceContext* c) {
ShapeHandle input_values = c->input(0);
ShapeHandle labels = c->input(1);
ShapeHandle row_splits = c->input(2);
Expand Down
4 changes: 2 additions & 2 deletions tensorflow_text/core/ops/tokenizer_from_logits_op.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ using shape_inference::DimensionHandle;
using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

Status TokenizerFromLogitsShapeFn(InferenceContext* c);
absl::Status TokenizerFromLogitsShapeFn(InferenceContext* c);

REGISTER_OP("TokenizerFromLogits")
.Input("strings: string")
Expand Down Expand Up @@ -100,7 +100,7 @@ REGISTER_OP("TokenizerFromLogits")
A 2D RaggedTensor can be constructed from this and row_splits.
)doc");

Status TokenizerFromLogitsShapeFn(InferenceContext* c) {
absl::Status TokenizerFromLogitsShapeFn(InferenceContext* c) {
ShapeHandle strings = c->input(0);
ShapeHandle logits = c->input(1);
ShapeHandle force_split_at_break_character = c->input(2);
Expand Down
5 changes: 2 additions & 3 deletions tensorflow_text/core/ops/wordpiece_op.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ using shape_inference::DimensionHandle;
using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

Status WordpieceTokenizeWithOffsetsShapeFn(InferenceContext* c);
absl::Status WordpieceTokenizeWithOffsetsShapeFn(InferenceContext* c);

REGISTER_OP("WordpieceTokenizeWithOffsets")
.Input("input_values: string")
Expand Down Expand Up @@ -90,7 +90,7 @@ REGISTER_OP("WordpieceTokenizeWithOffsets")
A 2D RaggedTensor can be constructed from this and output_row_lengths.
)doc");

Status WordpieceTokenizeWithOffsetsShapeFn(InferenceContext* c) {
absl::Status WordpieceTokenizeWithOffsetsShapeFn(InferenceContext* c) {
ShapeHandle input_values = c->input(0);
ShapeHandle vocab_lookup_table = c->input(1);
string output_row_partition_type;
Expand All @@ -112,5 +112,4 @@ Status WordpieceTokenizeWithOffsetsShapeFn(InferenceContext* c) {
return absl::OkStatus();
}


} // namespace tensorflow