Fixes GitHub issue #158. This replaces the ICU character class for wh…

…itespace.

If we want to support these, we need to add the RE2_USE_ICU build flag and link in ICU to the regex ops. I have a working patch, but am not convinced it is worth submitting.

PiperOrigin-RevId: 279791264

tensorflow_text/BUILD

@@ -96,7 +96,6 @@ py_library(
         ":tokenization",
         ":unicode_script_tokenizer",
         ":wordpiece_tokenizer",
-        ":wordshape_ops",
         # python:array_ops tensorflow dep,
         # python:dtypes tensorflow dep,
         # python:math_ops tensorflow dep,

tensorflow_text/python/ops/bert_tokenizer.py

@@ -28,11 +28,10 @@
 from tensorflow_text.python.ops.tokenization import Tokenizer
 from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
 from tensorflow_text.python.ops.wordpiece_tokenizer import WordpieceTokenizer
-from tensorflow_text.python.ops.wordshape_ops import WordShape
 
 
 _DELIM_REGEX = [
-    WordShape.IS_WHITESPACE.value,
+    r"\s+",
     r"|".join([
         r"[!-/]",
         r"[:-@]",
@@ -54,7 +53,7 @@
 
 _DELIM_REGEX_PATTERN = "|".join(_DELIM_REGEX)
 _KEEP_DELIM_NO_WHITESPACE = copy.deepcopy(_DELIM_REGEX)
-_KEEP_DELIM_NO_WHITESPACE.remove(WordShape.IS_WHITESPACE.value)
+_KEEP_DELIM_NO_WHITESPACE.remove(r"\s+")
 
 _KEEP_DELIM_NO_WHITESPACE_PATTERN = "|".join(_KEEP_DELIM_NO_WHITESPACE)