From 7774071e5f04be2ea0718ad1dca2007211a8946e Mon Sep 17 00:00:00 2001 From: ADITYA DAS <58718316+ADITYADAS1999@users.noreply.github.com> Date: Wed, 4 Jan 2023 06:48:42 +0530 Subject: [PATCH] Update word_piece_tokenizer.py (#617) * Update word_piece_tokenizer.py * Revert "Update word_piece_tokenizer.py" This reverts commit fcad4b8a5b5c427f50573af0393132d172073e2e. * Update word_piece_tokenizer.py --- keras_nlp/tokenizers/word_piece_tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_nlp/tokenizers/word_piece_tokenizer.py b/keras_nlp/tokenizers/word_piece_tokenizer.py index d78a0d348b..97225c45df 100644 --- a/keras_nlp/tokenizers/word_piece_tokenizer.py +++ b/keras_nlp/tokenizers/word_piece_tokenizer.py @@ -93,7 +93,7 @@ def pretokenize( text, - lowercase=True, + lowercase=False, strip_accents=True, split=True, split_on_cjk=True,