Merge branch 'main' of https://github.com/allenai/pdelfin

allenai · Jan 23, 2025 · aa59d38 · aa59d38
2 parents eacd044 + 01469af
commit aa59d38
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/pdelfin/train/loaddataset.py b/pdelfin/train/loaddataset.py
@@ -20,7 +20,13 @@ def main():
 
     print("Training dataset........")
     print(train_dataset)
-    print(train_dataset[0])
+
+    train_example = train_dataset[0]
+    print(train_example)
+    print({(x, y.shape) for x,y in train_example.items()})
+    print("\nTokens")
+    print(processor.tokenizer.batch_decode(train_example["input_ids"]))
+
     print("\n\n")
 
     print("Validation dataset........")

diff --git a/tests/test_molmo.py b/tests/test_molmo.py
@@ -37,6 +37,7 @@ def test_molmo_demo(self):
         print(inputs)
 
         print("\nShapes")
+        # {('input_ids', torch.Size([1, 589])), ('images', torch.Size([1, 5, 576, 588])), ('image_masks', torch.Size([1, 5, 576])), ('image_input_idx', torch.Size([1, 5, 144]))}
         print({(x, y.shape) for x,y in inputs.items()})
 
         print("\nTokens")