fix(docx): read the text inside a DrawingML text box

packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py

@@ -125,6 +125,66 @@ def _pre_process_strike(content: bytes) -> bytes:
     return str(soup).encode()
 
 
+def _pre_process_text_boxes(content: bytes) -> bytes:
+    """
+    Rewrites DrawingML text boxes into the VML form Mammoth already reads.
+
+    A text box keeps its own paragraphs in a ``w:txbxContent``. Mammoth reads that
+    element, but only reaches it through the legacy VML path
+    (``w:pict`` -> ``v:shape`` -> ``v:textbox``). A modern text box is a DrawingML
+    shape instead — ``w:drawing`` -> ``wp:inline`` -> ``wps:wsp`` -> ``wps:txbx`` —
+    and ``wp:inline`` is read as a picture, so the shape's text is dropped with no
+    warning. Callouts, pull quotes, sidebars and diagram labels vanish.
+
+    A copy of the ``w:txbxContent`` is therefore inserted after the ``w:drawing``
+    that holds it, wrapped in ``w:pict``/``v:shape``/``v:textbox``, and the
+    original is removed so the text cannot be read twice. Mammoth treats a
+    ``w:pict`` as extra content that follows the paragraph containing it, which is
+    where a reader of the page sees the text box anyway.
+
+    Text boxes inside ``mc:AlternateContent`` are left alone: Mammoth reads the
+    ``mc:Fallback`` branch there, which Word fills with the same text as a VML
+    shape, so promoting the ``mc:Choice`` branch as well would duplicate it.
+
+    Args:
+        content (bytes): The XML content of the DOCX file as bytes.
+
+    Returns:
+        bytes: The processed content, encoded as bytes.
+    """
+    # Parsing and reserializing is expensive on large documents, so skip the
+    # round-trip when there is no text box to promote.
+    if b"txbxContent" not in content:
+        return content
+
+    soup = BeautifulSoup(content.decode(), features="xml")
+    changed = False
+
+    for text_box in soup.find_all("txbxContent"):
+        ancestors = {parent.name for parent in text_box.parents}
+        # Already reachable, or handled through the fallback branch.
+        if "pict" in ancestors or "AlternateContent" in ancestors:
+            continue
+
+        drawing = text_box.find_parent("drawing")
+        if drawing is None:
+            continue
+
+        pict = soup.new_tag("pict", nsprefix="w")
+        shape = soup.new_tag("shape", nsprefix="v")
+        textbox = soup.new_tag("textbox", nsprefix="v")
+        pict.append(shape)
+        shape.append(textbox)
+        textbox.append(text_box.extract())
+        drawing.insert_after(pict)
+        changed = True
+
+    if not changed:
+        return content
+
+    return str(soup).encode()
+
+
 def _pre_process_math(content: bytes) -> bytes:
     """
     Pre-processes the math content in a DOCX -> XML file by converting OMML (Office Math Markup Language) elements to LaTeX.
@@ -267,7 +327,11 @@ def pre_process_docx(input_docx: BinaryIO) -> BinaryIO:
     output_docx = BytesIO()
     # The pre-processing steps to apply to each file in the .docx
     pre_process_enable_files = {
-        "word/document.xml": (_pre_process_strike, _pre_process_math),
+        "word/document.xml": (
+            _pre_process_strike,
+            _pre_process_text_boxes,
+            _pre_process_math,
+        ),
         "word/footnotes.xml": (_pre_process_strike, _pre_process_math),
         "word/endnotes.xml": (_pre_process_strike, _pre_process_math),
         "word/styles.xml": (_pre_process_styles,),

packages/markitdown/tests/test_docx_text_boxes.py

@@ -0,0 +1,83 @@
+"""A DrawingML text box keeps its text in the document and must not be dropped.
+
+Mammoth reaches a ``w:txbxContent`` only through the legacy VML path
+(``w:pict`` -> ``v:shape`` -> ``v:textbox``). A modern text box is a DrawingML
+shape instead, and the element that holds it is read as a picture, so its text is
+dropped silently. ``_pre_process_text_boxes`` rewrites it into the form Mammoth
+reads.
+"""
+
+import io
+import re
+import zipfile
+from pathlib import Path
+
+from markitdown import MarkItDown
+
+TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx"
+
+DRAWING_TEXT_BOX = (
+    "<w:r><w:drawing><wp:inline distT='0' distB='0' distL='0' distR='0'>"
+    "<wp:extent cx='2743200' cy='914400'/><wp:docPr id='9001' name='Text Box 1'/>"
+    "<a:graphic><a:graphicData "
+    "uri='http://schemas.microsoft.com/office/word/2010/wordprocessingShape'>"
+    "<wps:wsp><wps:txbx><w:txbxContent>"
+    "<w:p><w:r><w:t>CALLOUT MARKER</w:t></w:r></w:p>"
+    "</w:txbxContent></wps:txbx></wps:wsp>"
+    "</a:graphicData></a:graphic></wp:inline></w:drawing></w:r>"
+)
+
+VML_TEXT_BOX = (
+    "<w:pict><v:shape><v:textbox><w:txbxContent>"
+    "<w:p><w:r><w:t>CALLOUT MARKER</w:t></w:r></w:p>"
+    "</w:txbxContent></v:textbox></v:shape></w:pict>"
+)
+
+ALTERNATE_CONTENT_TEXT_BOX = (
+    "<w:r><mc:AlternateContent>"
+    f"<mc:Choice Requires='wps'>{DRAWING_TEXT_BOX}</mc:Choice>"
+    f"<mc:Fallback>{VML_TEXT_BOX}</mc:Fallback>"
+    "</mc:AlternateContent></w:r>"
+)
+
+
+def _docx_with(markup: str) -> io.BytesIO:
+    """Return test.docx with `markup` appended to its first paragraph."""
+    fixture = io.BytesIO()
+    with zipfile.ZipFile(TEST_DOCX) as source, zipfile.ZipFile(fixture, "w") as target:
+        for item in source.infolist():
+            content = source.read(item)
+            if item.filename == "word/document.xml":
+                document = content.decode()
+                paragraph = re.search(r"<w:p\b[^>]*>.*?</w:p>", document, re.DOTALL)
+                assert paragraph is not None
+                patched = paragraph.group(0).replace(
+                    "</w:p>", markup.replace("'", '"') + "</w:p>", 1
+                )
+                document = document.replace(paragraph.group(0), patched, 1)
+                content = document.encode()
+            target.writestr(item, content)
+    fixture.seek(0)
+    return fixture
+
+
+def test_a_drawing_text_box_is_read() -> None:
+    markdown = MarkItDown().convert(_docx_with(DRAWING_TEXT_BOX)).markdown
+
+    assert markdown.count("CALLOUT MARKER") == 1
+
+
+def test_an_alternate_content_text_box_is_read_once() -> None:
+    """Word writes the same text twice, as a choice and as a VML fallback.
+
+    Mammoth reads the fallback, so promoting the choice as well would double it.
+    """
+    markdown = MarkItDown().convert(_docx_with(ALTERNATE_CONTENT_TEXT_BOX)).markdown
+
+    assert markdown.count("CALLOUT MARKER") == 1
+
+
+def test_a_document_without_a_text_box_is_unchanged() -> None:
+    expected = MarkItDown().convert(TEST_DOCX).markdown
+
+    assert MarkItDown().convert(_docx_with("")).markdown == expected