Refactoring

allenai · Oct 9, 2024 · 4bf6e7a · 4bf6e7a
1 parent 0c56dec
commit 4bf6e7a
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 61 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 # ml stuff
 wandb/
+*histogram.png
 
 /*.html
 

diff --git a/pdelfin/filter/imagedetect.py b/pdelfin/filter/imagedetect.py
diff --git a/tests/test_dataloader.py b/tests/test_dataloader.py
@@ -117,8 +117,8 @@ def testExtractResponse(self):
         print(response_data[0])
 
     def testPyArrowDirectJson(self):
-        query_glob_path="s3://ai2-oe-data/jakep/pdfdata/openai_batch_data_v5_1_train/*.jsonl"
-        response_glob_path="s3://ai2-oe-data/jakep/pdfdata/openai_batch_done_v5_1_train/*.json"
+        query_glob_path="s3://ai2-oe-data/jakep/pdfdata/openai_batch_data_v5_1_eval/*.jsonl"
+        response_glob_path="s3://ai2-oe-data/jakep/pdfdata/openai_batch_done_v5_1_eval/*.json"
 
         all_files = list_dataset_files(query_glob_path)
 

diff --git a/tests/test_filter.py b/tests/test_filter.py
@@ -4,7 +4,6 @@
 from pypdf import PdfReader
 
 from pdelfin.filter import PdfFilter
-from pdelfin.filter.imagedetect import pdf_page_image_area
 
 
 class PdfFilterTest(unittest.TestCase):
@@ -16,10 +15,3 @@ def testFormLaterPages(self):
             self.filter._is_form(os.path.join(os.path.dirname(__file__), "gnarly_pdfs", "form_on_later_pages.pdf"))
         )
 
-
-class ImageDetectionTest(unittest.TestCase):
-    def testSlideshowMostlyImages(self):
-        self.pdf = PdfReader(os.path.join(os.path.dirname(__file__), "gnarly_pdfs", "slideshow_mostly_images.pdf"))
-
-        for page in range(self.pdf.get_num_pages()):
-            print(page, pdf_page_image_area(self.pdf, page + 1))