Fix tokenizer tests

abuelnasr0 · May 6, 2024 · 10967a4 · 10967a4
1 parent 88dd003
commit 10967a4
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 0 deletions.
diff --git a/...s_nlp/tests/test_data/phi3_test_vocab.spm → ...p/src/tests/test_data/phi3_test_vocab.spm b/...s_nlp/tests/test_data/phi3_test_vocab.spm → ...p/src/tests/test_data/phi3_test_vocab.spm
diff --git a/tools/sentencepiece_testing/create_phi3_test_proto.py b/tools/sentencepiece_testing/create_phi3_test_proto.py
@@ -37,6 +37,7 @@ def add_added_tokens(filename):
     with open(
         pathlib.Path(__file__).parent.parent.parent
         / "keras_nlp"
+        / "src"
         / "tests"
         / "test_data"
         / filename,
@@ -53,6 +54,7 @@ def add_added_tokens(filename):
     with open(
         pathlib.Path(__file__).parent.parent.parent
         / "keras_nlp"
+        / "src"
         / "tests"
         / "test_data"
         / filename,

diff --git a/tools/sentencepiece_testing/utils.py b/tools/sentencepiece_testing/utils.py
@@ -25,6 +25,7 @@ def train_sentencepiece(data, filename, *args, **kwargs):
     with open(
         pathlib.Path(__file__).parent.parent.parent
         / "keras_nlp"
+        / "src"
         / "tests"
         / "test_data"
         / filename,