BPE tokenizer (#389)

Add more test cases. Co-authored-by: jessechancy <[email protected]> add merge file Make cache a tf module Delete testdata address comments address comments fix docstring fix docstring
keras-team · Nov 10, 2022 · 82e4914 · 82e4914
1 parent f9abc8f
commit 82e4914
Show file tree

Hide file tree

Showing 3 changed files with 679 additions and 0 deletions.
diff --git a/keras_nlp/tokenizers/__init__.py b/keras_nlp/tokenizers/__init__.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 from keras_nlp.tokenizers.byte_tokenizer import ByteTokenizer
 from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
 from keras_nlp.tokenizers.tokenizer import Tokenizer