fix: added extraction of byte-images in excel (#804)

* fix(msexcel): ignore Mypy checking for _find_images_in_sheet function Signed-off-by: Jiun An Tsai <[email protected]> * fixed some issues Signed-off-by: Peter Staar <[email protected]> * reformatted the code Signed-off-by: Peter Staar <[email protected]> * pinned pillow in pyproject Signed-off-by: Peter Staar <[email protected]> --------- Signed-off-by: Jiun An Tsai <[email protected]> Signed-off-by: Peter Staar <[email protected]> Signed-off-by: Michele Dolfi <[email protected]> Co-authored-by: Jiun An Tsai <[email protected]> Co-authored-by: Michele Dolfi <[email protected]>
DS4SD · Jan 24, 2025 · a458e29 · a458e29
1 parent 16a218d
commit a458e29
Show file tree

Hide file tree

Showing 8 changed files with 87 additions and 44 deletions.
diff --git a/docling/backend/msexcel_backend.py b/docling/backend/msexcel_backend.py
@@ -26,6 +26,7 @@
 
 from typing import Any, List
 
+from PIL import Image as PILImage
 from pydantic import BaseModel
 
 
@@ -325,49 +326,61 @@ def _find_images_in_sheet(
         self, doc: DoclingDocument, sheet: Worksheet
     ) -> DoclingDocument:
 
-        # FIXME: mypy does not agree with _images ...
-        """
-        # Iterate over images in the sheet
-        for idx, image in enumerate(sheet._images):  # Access embedded images
+        # Iterate over byte images in the sheet
+        for idx, image in enumerate(sheet._images):  # type: ignore
 
-            image_bytes = BytesIO(image.ref.blob)
-            pil_image = Image.open(image_bytes)
+            try:
+                pil_image = PILImage.open(image.ref)
 
-            doc.add_picture(
-                parent=self.parents[0],
-                image=ImageRef.from_pil(image=pil_image, dpi=72),
-                caption=None,
-            )
-        """
+                doc.add_picture(
+                    parent=self.parents[0],
+                    image=ImageRef.from_pil(image=pil_image, dpi=72),
+                    caption=None,
+                )
+            except:
+                _log.error("could not extract the image from excel sheets")
 
-        # FIXME: mypy does not agree with _charts ...
         """
-        for idx, chart in enumerate(sheet._charts):  # Access embedded charts
-            chart_path = f"chart_{idx + 1}.png"
-            _log.info(
-                f"Chart found, but dynamic rendering is required for: {chart_path}"
-            )
+        for idx, chart in enumerate(sheet._charts):  # type: ignore
+            try:
+                chart_path = f"chart_{idx + 1}.png"
+                _log.info(
+                    f"Chart found, but dynamic rendering is required for: {chart_path}"
+                )
 
-            _log.info(f"Chart {idx + 1}:")
-        
-            # Chart type
-            _log.info(f"Type: {type(chart).__name__}")
-            
-            # Title
-            if chart.title:
-                _log.info(f"Title: {chart.title}")
-            else:
-                _log.info("No title")
-            
-            # Data series
-            for series in chart.series:
-                _log.info(" => series ...")
-                _log.info(f"Data Series: {series.title}")
-                _log.info(f"Values: {series.values}")
-                _log.info(f"Categories: {series.categories}")
+                _log.info(f"Chart {idx + 1}:")
                 
-            # Position
-            # _log.info(f"Anchor Cell: {chart.anchor}")
+                # Chart type
+                # _log.info(f"Type: {type(chart).__name__}")
+                print(f"Type: {type(chart).__name__}")
+
+                # Extract series data
+                for series_idx, series in enumerate(chart.series):
+                    #_log.info(f"Series {series_idx + 1}:")
+                    print(f"Series {series_idx + 1} type: {type(series).__name__}")
+                    #print(f"x-values: {series.xVal}")
+                    #print(f"y-values: {series.yVal}")
+
+                    print(f"xval type: {type(series.xVal).__name__}")
+                    
+                    xvals = []
+                    for _ in series.xVal.numLit.pt:
+                        print(f"xval type: {type(_).__name__}")
+                        if hasattr(_, 'v'):
+                            xvals.append(_.v)
+
+                    print(f"x-values: {xvals}")
+                            
+                    yvals = []
+                    for _ in series.yVal:
+                        if hasattr(_, 'v'):
+                            yvals.append(_.v)
+                            
+                    print(f"y-values: {yvals}")                    
+                    
+            except Exception as exc:
+                print(exc)
+                continue
         """
 
         return doc
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -56,6 +56,7 @@ onnxruntime = [
   { version = ">=1.7.0,<1.20.0", optional = true, markers = "python_version < '3.10'" },
   { version = "^1.7.0", optional = true, markers = "python_version >= '3.10'" }
 ]
+pillow = "^10.0.0"
 
 [tool.poetry.group.dev.dependencies]
 black = {extras = ["jupyter"], version = "^24.4.2"}

diff --git a/tests/data/groundtruth/docling_v2/test-01.xlsx.itxt b/tests/data/groundtruth/docling_v2/test-01.xlsx.itxt
@@ -7,4 +7,5 @@ item-0 at level 0: unspecified: group _root_
     item-6 at level 2: table with [5x3]
   item-7 at level 1: section: group sheet: Sheet3
     item-8 at level 2: table with [7x3]
-    item-9 at level 2: table with [7x3]
+    item-9 at level 2: table with [7x3]
+    item-10 at level 2: picture
diff --git a/tests/data/groundtruth/docling_v2/test-01.xlsx.json b/tests/data/groundtruth/docling_v2/test-01.xlsx.json
diff --git a/tests/data/groundtruth/docling_v2/test-01.xlsx.md b/tests/data/groundtruth/docling_v2/test-01.xlsx.md
@@ -48,4 +48,6 @@
 | 3           | 4            | 5            |
 | 3           | 6            | 7            |
 | 8           | 9            | 9            |
-| 10          | 9            | 9            |
+| 10          | 9            | 9            |
+
+<!-- image -->
diff --git a/tests/data/xlsx/test-01.xlsx b/tests/data/xlsx/test-01.xlsx
diff --git a/tests/test_backend_msexcel.py b/tests/test_backend_msexcel.py
@@ -53,7 +53,7 @@ def test_e2e_xlsx_conversions():
     converter = get_converter()
 
     for xlsx_path in xlsx_paths:
-        # print(f"converting {xlsx_path}")
+        print(f"converting {xlsx_path}")
 
         gt_path = (
             xlsx_path.parent.parent / "groundtruth" / "docling_v2" / xlsx_path.name