Add add_bos, add_eos to SentencePieceTokenizer.get_config()

keras-team · Sep 6, 2024 · b57396c · b57396c
1 parent 32ee45f
commit b57396c
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/keras_nlp/src/tokenizers/sentence_piece_tokenizer.py b/keras_nlp/src/tokenizers/sentence_piece_tokenizer.py
@@ -221,6 +221,8 @@ def get_config(self):
             {
                 "proto": None,  # Save vocabulary via an asset!
                 "sequence_length": self.sequence_length,
+                "add_bos": self.add_bos,
+                "add_eos": self.add_eos,
             }
         )
         return config