Adding Utility to Detokenize as list of Strings to Tokenizer Base Class

keras_nlp/tokenizers/tokenizer.py

@@ -14,6 +14,7 @@
 
 from typing import List
 
+import tensorflow as tf
 from tensorflow import keras
 
 
@@ -129,3 +130,40 @@ def call(self, *args, mode="tokenize", training=None, **kwargs):
             raise ValueError(
                 f"Unsupported tokenizer mode. Received: mode={mode}"
             )
+
+    def recursive_utf8_decoder(self, inputs, *args, **kwargs):
+        """Recursively decodes to list of strings with 'utf-8' encoding."""
+        if str(type(inputs)) == "<class 'bytes'>":
+            inputs = inputs.decode("utf-8")
+            return inputs
+        if str(type(inputs[0])) == "<class 'bytes'>":
+            for i in range(len(inputs)):
+                inputs[i] = inputs[i].decode("utf-8")
+            return inputs
+        else:
+            for i in range(len(inputs)):
+                inputs[i] = self.recursive_utf8_decoder(
+                    inputs[i], *args, **kwargs
+                )
+
+    def detokenize_to_strings(self, inputs, *args, **kwargs):
+        """Transform detokenized inputs to strings.
+        Args:
+            inputs: Input tensor, or dict/list/tuple of input tensors.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+        detokenized_input = self.detokenize(inputs)
+        scalar = detokenized_input.shape.rank == 0
+        if isinstance(detokenized_input, tf.RaggedTensor):
+            detokenized_input = detokenized_input.to_list()
+        elif isinstance(detokenized_input, tf.Tensor):
+            if scalar:
+                detokenized_input = detokenized_input.numpy()
+                return detokenized_input.decode("utf-8")
+            else:
+                detokenized_input = detokenized_input.numpy().tolist()
+        detokenized_input = list(
+            map(self.recursive_utf8_decoder, detokenized_input)
+        )
+        return detokenized_input