Skip to content

Commit

Permalink
Conditionally import tf text (#452)
Browse files Browse the repository at this point in the history
In keeping with other layers, we should not rely on tf text being
installed to import the library (this is useful for building keras.io
for example).
  • Loading branch information
mattdangerw authored Nov 10, 2022
1 parent ebcbc30 commit 152ec7c
Showing 1 changed file with 10 additions and 2 deletions.
12 changes: 10 additions & 2 deletions keras_nlp/tokenizers/byte_pair_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,15 @@
from typing import List

import tensorflow as tf
import tensorflow_text as tf_text
from tensorflow import keras

from keras_nlp.tokenizers import tokenizer
from keras_nlp.utils.tf_utils import assert_tf_text_installed

try:
import tensorflow_text as tf_text
except ImportError:
tf_text = None

# As python and TF handles special spaces differently, we need to
# manually handle special spaces during string split.
Expand Down Expand Up @@ -220,6 +225,8 @@ def __init__(
sequence_length=None,
**kwargs,
) -> None:
assert_tf_text_installed(self.__class__.__name__)

# Check dtype and provide a default.
if "dtype" not in kwargs or kwargs["dtype"] is None:
kwargs["dtype"] = tf.int32
Expand All @@ -241,7 +248,8 @@ def __init__(
else:
raise ValueError(
"Vocabulary must be an file path or dictionary mapping string "
f"token to int ids. Received: `type(vocabulary)={type(vocabulary)}`."
"token to int ids. Received: "
f"`type(vocabulary)={type(vocabulary)}`."
)
if isinstance(merges, str):
self.merges = [bp.rstrip() for bp in tf.io.gfile.GFile(merges)]
Expand Down

0 comments on commit 152ec7c

Please sign in to comment.