Fix dataloader bug

allenai · Oct 7, 2024 · a90eb94 · a90eb94
1 parent 3d36545
commit a90eb94
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pdelfin/train/dataloader.py b/pdelfin/train/dataloader.py
@@ -51,7 +51,7 @@ def load_jsonl_into_ds(s3_glob_path: str, first_n_files: int = None) -> Dataset:
     """
     Loads JSONL files from the specified S3 path into a Hugging Face Dataset.
     """
-    all_json_files = s3_glob_path(s3_glob_path)
+    all_json_files = list_dataset_files(s3_glob_path)
 
     if first_n_files:
         all_json_files = all_json_files[:first_n_files]