diff --git a/README.md b/README.md index fea4cb1..79e47a2 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,28 @@ python3 matter.py packet ~/cases/my-case --layer 3min --out COUNSEL_PACKET.md Tools: `matter.status` · `matter.fact_add` · `matter.fact_list` · `matter.deadline_list` · `matter.packet_render` · `matter.cite_extract`. +## Document extraction and locators (RFC 0004, RFC 0005) + +```bash +python3 matter.py pages ~/cases/my-case # every ingested document +python3 matter.py pages ~/cases/my-case Filings/motion.docx # one file +``` + +Stdlib only — no pypdf, no python-docx. Each format gets the locator its own +bytes can support, and the two never mix: + +| scheme | format | locator | stored in | +|---|---|---|---| +| `pdf-page` | PDF | 1-based page number read from the file's structure | `document_pages` | +| `docx-structural` | DOCX | path through the OOXML block tree, e.g. `word/document.xml#b3/r1/c0/b0` | `document_blocks` | +| `whole-file` | `.txt` / `.md` | the file is one unit | `document_pages` | + +**DOCX never gets a page number.** Pagination is produced by the renderer at +layout time and is not in the file, so one would have to be invented — see +RFC 0005. Extraction that cannot be done honestly fails and says why +(`extraction-failed` with a reason); it is never a silent empty string. +Sources are read-only and are never rewritten. + ## Citations and stages (v0.2, RFC 0001) ```bash diff --git a/matter.py b/matter.py index cb5759d..85953d6 100644 --- a/matter.py +++ b/matter.py @@ -164,6 +164,77 @@ def cmd_cite(args): f"(text {r['text_sha256'][:12]}, parser {r['parser']})") +def cmd_pages(args): + """pages [path] — extract document text with source locators. + + Stdlib-only. Scanned/LZW/CID-font PDFs fail honestly per page; nothing is + ever silently empty. PDF/text results write to document_pages keyed by + sha256+page (RFC 0004). + + DOCX (RFC 0005) is routed to structural locators and written to + document_blocks instead. A DOCX never produces a document_pages row: the + format carries no rendered page numbers, so one would have to be invented. + """ + conn = store.connect(args.case_dir) + from matterkit import extract, ingest + import hashlib + if args.path: + files = [(args.path, ingest.sha256_file(args.path))] + else: + docs = conn.execute("SELECT id, sha256, rel_path FROM documents").fetchall() + files = [] + for d in docs: + # rel_path is relative to the cwd at ingest time; the id is a hash, + # not a path. Resolve rel_path against cwd first, then fall back to + # the matter dir (covers both the demo layout and older ingests). + full = os.path.abspath(d["rel_path"]) + if not os.path.isfile(full): + full = os.path.join(args.case_dir, d["rel_path"]) + if os.path.isfile(full): + files.append((full, d["sha256"])) + total_ok = total_fail = 0 + for path, sha in files: + doc = extract.extract_text(path) + if doc.locator_scheme == extract.LOCATOR_SCHEME_DOCX: + extract.record_blocks(conn, sha, doc) + name = os.path.basename(path) + if doc.status == "extracted": + total_ok += len(doc.blocks) + print(f"✓ {name}: {len(doc.blocks)} block(s) " + f"[{doc.locator_scheme}; DOCX carries no page numbers]") + if doc.parts and doc.parts.skipped: + print(f" · skipped by design: " + f"{', '.join(doc.parts.skipped)}") + if doc.parts and doc.parts.unhandled: + print(f" ⚠ not descended into: " + f"{', '.join(doc.parts.unhandled)}") + else: + total_fail += 1 + print(f"✗ {name}: extraction failed (0 block(s))") + print(f" ⚠ {doc.reason}") + continue + for pg in doc.pages: + conn.execute( + "INSERT OR REPLACE INTO document_pages " + "(document_sha256, page_number, text_sha256, text, status) " + "VALUES (?,?,?,?,?)", + (sha, pg.number, pg.text_sha256, pg.text, pg.status)) + if pg.status == "extracted": + total_ok += 1 + else: + total_fail += 1 + n = len(doc.pages) + mark = "✓" if doc.pages and doc.pages[0].status == "extracted" else "✗" + print(f"{mark} {os.path.basename(path)}: {n} page(s) " + f"({total_ok} ok / {total_fail} failed so far)") + for pg in doc.pages: + if pg.status == "extraction-failed": + print(f" ⚠ page {pg.number}: {pg.reason}") + conn.commit() + print(f"located units: {total_ok} extracted · {total_fail} failed " + f"(never silently empty)") + + def cmd_auth_stage(args): """Stage-validated authority status change (RFC 0001 §2.2c).""" conn = store.connect(args.case_dir) @@ -349,6 +420,9 @@ def main(): p.add_argument("--status", choices=["extracted", "extraction-failed"]) p.set_defaults(fn=cmd_cite) + p = sub.add_parser("pages"); p.add_argument("case_dir"); p.add_argument("path", nargs="?") + p.set_defaults(fn=cmd_pages) + p = sub.add_parser("stage"); p.add_argument("case_dir"); p.add_argument("authority_id"); p.add_argument("status") p.add_argument("--via", default=None); p.add_argument("--evidence", default=None) p.set_defaults(fn=cmd_auth_stage) diff --git a/matterkit/extract.py b/matterkit/extract.py new file mode 100644 index 0000000..aa13370 --- /dev/null +++ b/matterkit/extract.py @@ -0,0 +1,1065 @@ +"""Document text extraction with source locators (RFC 0004 + RFC 0005). + +Stdlib-only by default. Honest failure: a PDF whose text stream we cannot +decode (LZW, CID/Type0 fonts, image-only scans) returns an explicit +`extraction_failed` on that page — never silently empty text, which is the +single failure mode that loses evidence. + +Heavier deps (pdfplumber, tesseract OCR) are OPTIONAL: importable only when +installed, never required. This module must stay importable with stdlib only. + +======================================================================== +LOCATOR SCHEMES +======================================================================== + +Every extraction declares exactly one `locator_scheme`, and the scheme +determines both what a locator means and where it is stored. The schemes are +never mixed and one is never coerced into another. + +``pdf-page`` (RFC 0004) PDF. Locator is a 1-based page number recovered + from the file's own page/content-stream structure. + Stored in ``document_pages``. +``docx-structural`` (RFC 0005) DOCX. Locator is a structural path through the + OOXML block tree. Stored in ``document_blocks``. +``whole-file`` (RFC 0004) ``.txt``/``.md`` and unextractable kinds. The whole + file is one unit; there is no intra-file locator. + +**A DOCX never carries a page number.** Pagination in WordprocessingML is +produced by the rendering engine at layout time (fonts, printer metrics, +field results, hyphenation) and is not present in the file. An explicit +``w:br w:type="page"`` is a *hint to the renderer*, not an authoritative page +boundary: content still reflows around it. Any page number attached to DOCX +content would therefore be manufactured by this tool, not read from the +source — a fabricated legal-grade citation. So: + +* ``extract_docx`` returns ``pages == []`` in every case, success or failure. +* ``document_blocks`` has no ``page_number`` column, and a CHECK constraint + refuses any ``locator_scheme`` other than ``docx-structural``. +* the ``matter pages`` CLI writes no ``document_pages`` row for a ``.docx``. + +======================================================================== +THE DOCX LOCATOR CONTRACT (RFC 0005) +======================================================================== + +A citation is the triple ``(document_sha256, part, locator)``. + +``document_sha256`` The sha256 of the exact source bytes. Locators are only + valid against the byte-identical file; editing a DOCX + produces a different sha256 and thus a different + citation space. Sources are opened read-only and are + never rewritten. +``part`` The OOXML part the locator addresses. v1 addresses + exactly one: ``word/document.xml``. +``locator`` ``#//...`` — a path of integer + coordinates through the part's *block tree*, rendered + with a one-letter type prefix that cycles ``b`` (block + child), ``r`` (table row), ``c`` (table cell). + +Path construction. Within a container (the body, or a table cell) the +recognised block-level children — ``w:p`` and ``w:tbl``, in document order, +ignoring property/bookmark elements that carry no text — are numbered from 0. +A ``w:p`` at index *i* is addressed ``(…, i)``. To reach content inside a +``w:tbl`` at index *i*, append the 0-based row and cell indexes and recurse +into that cell: ``(…, i, r, c)``. Hence every locator path has length +``3n + 1`` and the addressed unit is always a paragraph: + + word/document.xml#b3 4th block child of the body + word/document.xml#b3/r1/c0/b2 3rd paragraph of row 1, cell 0 of + the table at body index 3 + word/document.xml#b0/r0/c0/b1/r2/c1/b0 nested one table deeper + +**Uniqueness.** A path is a coordinate in a tree: two distinct blocks differ +at their first divergent index, so no two blocks in a part can share a path. +Collisions across parts are impossible because ``part`` is part of the +citation, and collisions across documents are impossible because +``document_sha256`` is. Identical *text* in two places is therefore still two +distinct citations — the locator identifies a position, not a string. The +storage primary key ``(document_sha256, part, locator)`` enforces this. + +**Offsets.** ``DocumentText.text`` is the extracted text of the part: the +block texts joined by a single ``\n`` in reading order. Each block records +``char_start``/``char_end`` into that string, with the invariant + + doc.text[block.char_start:block.char_end] == block.text + +so a quoted passage re-derives from the stored offsets exactly. Offsets index +the *extracted text*, never the raw XML. The joiner is not a delimiter: a +``w:br`` inside a paragraph also renders as ``\n``, so ``doc.text`` must not +be re-split on newlines to recover blocks. The stored offsets are the only +authoritative block boundaries. + +**Why character offsets and not run indexes.** A ``w:r`` run boundary is a +formatting artefact: Word re-splits runs on spell-check, revision tracking, +or an unrelated edit, and a run carries no semantic boundary. Run indexes are +therefore unstable identifiers for a passage. ``run_count`` is recorded as a +diagnostic (it shows the run traversal happened) and is never part of a +citation. + +**What counts as text.** Only ``w:t`` is visible text. ``w:delText`` +(tracked-change deletions) and ``w:instrText`` (field instruction codes) are +never extracted — quoting struck-out or machine-internal text as document +content would misstate the evidence. ``w:tab``, ``w:br``/``w:cr`` and +``w:noBreakHyphen`` render as ``\t``, ``\n`` and ``-``. Element *tails* are +ignored, so pretty-printing whitespace in the XML never enters the text. + +**Hostile input.** The extension is checked against the leading bytes before +`zipfile` is handed the file, so a renamed ``.txt``/``.pdf`` — and a polyglot +whose zip is appended to another format — is refused rather than re-routed. +A single part is capped twice (declared size, then a bounded read) so a +forged header cannot get past the zip-bomb guard. A DOCTYPE is refused +outright: ``xml.etree.ElementTree`` expands internal entities and the C +parser exposes no expat handle to turn that off, and a conforming OOXML part +has no DTD. That check is a byte scan, so a UTF-8 BOM is stripped first and a +part that does not then begin as UTF-8 XML is refused rather than parsed +blind. Two entries under the same part name are refused as ambiguous, +because which one a reader resolves to is reader-dependent and the citation +could not be pinned. + +**Failure vocabulary.** ``status`` is ``extracted`` or ``extraction-failed`` +at the document level, with a human-readable ``reason`` on failure. Failure +yields zero blocks; it is never a zero-length success. A well-formed document +containing no paragraphs is the distinct case ``extracted`` with zero blocks +and ``reason is None``. + +**Part-set policy (v1).** Only ``word/document.xml`` is read. +Headers, footers, footnotes, endnotes and comments are text-bearing parts +that v1 deliberately does not extract; they are listed in +``PartInventory.skipped`` so a reviewer can distinguish "skipped by design" +from "missed". Every name in the archive is listed in ``PartInventory.seen``. +Block-level elements encountered in a container but not descended into (for +example ``w:sdt`` content controls) are named in ``PartInventory.unhandled`` +so no content is dropped silently. + +v1 addresses the main part by its conventional name rather than by resolving +the ``officeDocument`` package relationship, so a producer that names the +main part something other than ``word/document.xml`` fails with "no +word/document.xml part" instead of being read. That is a capability gap, not +a silent one; see RFC 0005 for the v2 item. +""" +from __future__ import annotations + +import hashlib +import json +import re +import zipfile +import zlib +from dataclasses import dataclass, field +from pathlib import Path +from xml.etree import ElementTree + +LOCATOR_SCHEME_PDF_PAGE = "pdf-page" +LOCATOR_SCHEME_DOCX = "docx-structural" +LOCATOR_SCHEME_WHOLE_FILE = "whole-file" + + +@dataclass +class Page: + number: int + text: str + status: str = "extracted" # extracted | extraction-failed + reason: str | None = None + text_sha256: str | None = None + + +@dataclass(frozen=True) +class DocxLocator: + """A structural coordinate inside one OOXML part. See the module docstring.""" + + part: str + path: tuple[int, ...] + + _PREFIXES = "brc" + + @property + def container_type(self) -> str: + if not self.path: + return "part" # the whole part; used only for failure rows + return "paragraph" if len(self.path) == 1 else "table-cell" + + def canonical(self) -> str: + segs = [f"{self._PREFIXES[i % 3]}{n}" for i, n in enumerate(self.path)] + return f"{self.part}#{'/'.join(segs)}" + + @classmethod + def parse(cls, text: str) -> "DocxLocator": + part, sep, rest = text.partition("#") + if not sep: + raise ValueError(f"not a docx locator (no '#'): {text!r}") + path: list[int] = [] + if rest: + for i, seg in enumerate(rest.split("/")): + want = cls._PREFIXES[i % 3] + if len(seg) < 2 or seg[0] != want or not seg[1:].isdigit(): + raise ValueError(f"bad locator segment {seg!r} in {text!r}") + path.append(int(seg[1:])) + if len(path) % 3 != 1: + raise ValueError(f"locator path must have length 3n+1: {text!r}") + return cls(part=part, path=tuple(path)) + + +@dataclass +class Block: + """One addressable paragraph of DOCX content.""" + + locator: DocxLocator + index: int # reading-order ordinal within the part + text: str + char_start: int # offsets into DocumentText.text + char_end: int + run_count: int # diagnostic only, never part of a citation + text_sha256: str + status: str = "extracted" + + +@dataclass(frozen=True) +class PartInventory: + """What the archive held, what we read, and what we knowingly did not.""" + + seen: tuple[str, ...] = () + extracted: tuple[str, ...] = () + skipped: tuple[str, ...] = () + unhandled: tuple[str, ...] = () + + +@dataclass +class DocumentText: + path: str + kind: str # pdf | text | docx | unknown + pages: list[Page] = field(default_factory=list) + blocks: list[Block] = field(default_factory=list) + locator_scheme: str = LOCATOR_SCHEME_WHOLE_FILE + status: str = "extracted" # extracted | extraction-failed + reason: str | None = None + text: str = "" # full extracted part text; block offsets index this + text_sha256: str | None = None + parts: PartInventory | None = None + + +# --- minimal PDF text extraction -------------------------------------------- +# Extracts `(...)Tj` / `[...] TJ` text-showing operators out of FlateDecode +# content streams. Handles the common plain-text PDF; everything else fails +# honestly per-page rather than guessing. + +_OBJ_DELIMS = "()<>[]{}/%" +_WS = b" \t\r\n\f\x00" + + +def _skip_ws(b: bytes, i: int) -> int: + while i < len(b) and b[i] in _WS: + i += 1 + return i + + +def _parse_number(b: bytes, i: int): + i = _skip_ws(b, i) + s = i + while i < len(b) and b[i] in b"0123456789+-.": + i += 1 + return b[s:i], i + + +def _parse_name(b: bytes, i: int): + i = _skip_ws(b, i) + if i < len(b) and b[i] == ord("/"): + s = i + 1 + i += 1 + while i < len(b) and b[i] not in _WS and b[i] not in _OBJ_DELIMS.encode(): + i += 1 + return b[s:i], i + return None, i + + +def _parse_string(b: bytes, i: int): + """Literal string; returns decoded bytes (rough latin-1 for text).""" + i = _skip_ws(b, i) + if i >= len(b) or b[i] != ord("("): + return None, i + depth = 1 + j = i + 1 + out = bytearray() + while j < len(b) and depth: + c = b[j] + if c == ord("\\"): + j += 1 + if j < len(b): + nxt = b[j] + if nxt == ord("n"): + out.append(ord("\n")) + elif nxt == ord("r"): + out.append(ord("\r")) + elif nxt == ord("t"): + out.append(ord("\t")) + elif nxt in b"()\\": + out.append(nxt) + elif nxt in b"01234567": + octv = b[j:j + 3] + out.append(int(octv, 8) & 0xFF) + j += 2 + else: + out.append(nxt) + j += 1 + elif c == ord("("): + depth += 1 + out.append(c) + j += 1 + elif c == ord(")"): + depth -= 1 + if depth: + out.append(c) + j += 1 + else: + out.append(c) + j += 1 + return out, j + + +def _extract_text_from_content(stream: bytes) -> str: + i = 0 + parts: list[str] = [] + last_text = "" + while i < len(stream): + c = stream[i] + if c == ord("("): + s, i2 = _parse_string(stream, i) + if s is None: + i += 1 + continue + i = i2 + try: + parts.append(s.decode("latin-1")) + except UnicodeDecodeError: + pass + elif c == ord("T"): + # "Tj" / "ET" text-show end; also literal `TJ` array operator + if i + 1 < len(stream) and stream[i + 1:] and stream[i + 1:i + 2] == b"j": + i += 2 + parts.append("\n") # Tj = show text; line break after + continue + if stream[i:i + 2] == b"TJ": + i += 2 + continue + i += 1 + elif c == ord("'"): + parts.append("\n") + i += 1 + elif c == ord('"'): + # double-quote = newline then show + parts.append("\n") + i += 1 + else: + i += 1 + text = "".join(parts) + return text + + +def _obj_offsets(b: bytes): + """Yield (start,end) of each `N M obj ... endobj`.""" + i = 0 + out = [] + while i < len(b): + idx = b.find(b"obj", i) + if idx < 0: + break + # back up to a boundary + s = idx + # find preceding number(s): sequence "N G obj" or "N G R obj" + # scan backwards for start of line + ls = b.rfind(b"\n", 0, idx) + 1 + head = b[ls:idx].split() + if len(head) >= 2 and head[-2].isdigit() and head[-1].isdigit(): + # obj num = head[-2] + end = b.find(b"endobj", idx) + if end < 0: + break + out.append((idx, end)) + i = end + 6 + else: + i = idx + 3 + return out + + +def _streams_in_obj(obj: bytes) -> list[bytes]: + """Return decoded data of each `stream ... endstream` in an object.""" + out = [] + i = 0 + while True: + s = obj.find(b"stream", i) + if s < 0: + break + # skip to EOL after 'stream' + j = obj.find(b"\n", s) + if j < 0: + j = obj.find(b"\r", s) + if j < 0: + break + start = j + 1 + # account for \r\n + if obj[start:start + 1] == b"\n": + start += 1 + e = obj.find(b"endstream", start) + if e < 0: + break + raw = obj[start:e] + if raw.endswith(b"\n"): + raw = raw[:-1] + if raw.endswith(b"\r"): + raw = raw[:-1] + out.append(raw) + i = e + len(b"endstream") + return out + + +def extract_pdf(path: str) -> DocumentText: + data = Path(path).read_bytes() + out = DocumentText(path=path, kind="pdf", + locator_scheme=LOCATOR_SCHEME_PDF_PAGE) + # Split content text per page: crude page-boundary detection by page objects. + # We first decode FlateDecode streams globally (many simple PDFs use one + # stream per page but we err honest: if no /FlateDecode filter, mark pages + # failed rather than dump binary). + text_blocks = [] + for _s, _e in _obj_offsets(data): + obj = data[_s:_e] + # only content streams that are FlateDecode + if b"/FlateDecode" in obj and b"stream" in obj: + for raw in _streams_in_obj(obj): + try: + decoded = zlib.decompress(raw) + except Exception: + continue + t = _extract_text_from_content(decoded) + if t.strip(): + text_blocks.append(t) + # Assign blocks to pages 1..N as a best-effort; without a real page tree we + # report the count and page numbers we CAN see. If no text at all -> one + # failed page (honest) rather than zero pages. + if not text_blocks: + out.pages.append(Page(number=1, text="", status="extraction-failed", + reason="no decodable text stream (scanned / LZW / CID font)")) + out.status = "extraction-failed" + out.reason = out.pages[0].reason + return out + for idx, t in enumerate(text_blocks, start=1): + out.pages.append(Page( + number=idx, text=t, status="extracted", + text_sha256=hashlib.sha256(t.encode()).hexdigest())) + return out + + +def extract_text(path: str) -> DocumentText: + """Dispatch by extension. Each branch declares its own locator scheme. + + Extension chooses the extractor; the extractor still verifies the content. + A ``.docx`` that is not actually a DOCX fails here — it is never re-routed + to another extractor on the basis of its bytes, because a file whose name + contradicts its content is exactly the provenance problem the kit exists + to refuse. + """ + p = Path(path) + ext = p.suffix.lower() + if ext == ".pdf": + return extract_pdf(path) + if ext == ".docx": + return extract_docx(path) + if ext in (".txt", ".md"): + raw = p.read_text(encoding="utf-8", errors="replace") + digest = hashlib.sha256(raw.encode()).hexdigest() + return DocumentText( + path=path, kind="text", + locator_scheme=LOCATOR_SCHEME_WHOLE_FILE, + text=raw, text_sha256=digest, + pages=[Page(number=1, text=raw, status="extracted", + text_sha256=digest)]) + return DocumentText( + path=path, kind="unknown", + locator_scheme=LOCATOR_SCHEME_WHOLE_FILE, + status="extraction-failed", reason=f"no extractor for {ext}", + pages=[Page(number=1, text="", status="extraction-failed", + reason=f"no extractor for {ext}")]) + + +# --- DOCX structural extraction (RFC 0005) ---------------------------------- + +W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +W = "{%s}" % W_NS + +PRIMARY_PART = "word/document.xml" + +#: Hard ceiling on a single decompressed part. Guards a zip bomb: the declared +#: size is checked first, then the actual read is bounded, so a forged header +#: cannot get past it either. +MAX_PART_BYTES = 32 * 1024 * 1024 + +#: ============================================================================ +#: ARCHIVE BOUNDS (RFC 0005 §4a). Three limits, three labels, and the labels +#: are load-bearing -- do not blur them: +#: +#: MAX_ARCHIVE_MEMBERS and MAX_CD_BYTES are the GUARD. Both are enforced +#: PRE-OPEN, straight off the archive's tail records, BEFORE +#: `zipfile.ZipFile` is constructed -- because `ZipFile.__init__` reads the +#: declared central directory (`fp.read(size_cd)` on CPython 3.11) and +#: materialises one `ZipInfo` per entry before any in-process check could +#: run. A refusal here has allocated no `ZipInfo` objects and read no +#: central-directory bytes. `size_limit:members` / `size_limit:central_directory` +#: failures fire before a ZipFile object exists. +#: +#: These two are a pre-open bound on *honest-or-overstated* tail records +#: (declared count, declared directory bytes, file geometry). A lying +#: archive can understate either field; that is caught post-open (count +#: reconciliation, or CPython raising on a truncated `size_cd` read), not +#: prevented. The byte cap bounds parser *input* and constrains allocation. +#: It is not an exact RAM ceiling and not proof of a pre-open actual-member +#: limit. Allocation between a passing guard and a post-open refusal is +#: bounded only by file size; there is no amplification (local-file threat +#: model). +#: +#: MAX_ARCHIVE_UNCOMPRESSED_BYTES is the ACCEPTANCE LIMIT, NOT a guard. It +#: is enforced POST-OPEN, inside `_extract_docx_open`, after `ZipFile` has +#: already parsed the directory -- per-member uncompressed sizes live in +#: central-directory entries and no tail record sums them, so the sum is +#: unobtainable without paying the parse the guard exists to avoid. It is a +#: declared-size sanity check; nothing post-open can undo the parse. +#: ============================================================================ +MAX_ARCHIVE_MEMBERS = 10_000 +MAX_ARCHIVE_UNCOMPRESSED_BYTES = 256 * 1024 * 1024 + +#: Cap on the declared central-directory size, enforced pre-open off the same +#: tail records as MAX_ARCHIVE_MEMBERS. CPython 3.11 `zipfile._RealGetContents` +#: does `data = fp.read(size_cd)` -- this is that input, not a heap ceiling. +#: +#: 32 MiB is tens of megabytes: a real Word `.docx` directory is kilobytes +#: (tens of parts, short names), and 32 MiB matches MAX_PART_BYTES so a +#: hostile archive cannot force a larger parser input than we already accept +#: for the one part we decompress. An honest 10,000-member directory with +#: long names/comments can approach this order; a 6 MB lying-count archive +#: is *under* the cap and is detected post-open. That residual is load-bearing. +MAX_CD_BYTES = 32 * 1024 * 1024 + +#: Local-file-header and end-of-central-directory signatures. Checked before +#: handing the file to `zipfile`, so a renamed .txt/.pdf is refused on its own +#: bytes rather than on a downstream parse error. +ZIP_MAGICS = (b"PK\x03\x04", b"PK\x05\x06") + +#: Text-bearing parts v1 deliberately does not extract. Recorded, not silent. +_SKIPPED_PART_RE = re.compile( + r"^word/(header\d*\.xml|footer\d*\.xml|footnotes\.xml|endnotes\.xml" + r"|comments\w*\.xml)$") + +#: Elements that may appear at a block position and carry no citable text. +#: Anything at a block position that is neither recognised nor listed here is +#: reported in PartInventory.unhandled rather than dropped silently. +_INERT_BLOCK_TAGS = frozenset({ + "pPr", "sectPr", "tblPr", "tblPrEx", "tblGrid", "trPr", "tcPr", + "bookmarkStart", "bookmarkEnd", "proofErr", "permStart", "permEnd", + "commentRangeStart", "commentRangeEnd", + "customXmlInsRangeStart", "customXmlInsRangeEnd", + "customXmlDelRangeStart", "customXmlDelRangeEnd", + "customXmlMoveFromRangeStart", "customXmlMoveFromRangeEnd", + "customXmlMoveToRangeStart", "customXmlMoveToRangeEnd", + "moveFromRangeStart", "moveFromRangeEnd", + "moveToRangeStart", "moveToRangeEnd", +}) + +#: Table nesting beyond this is pathological rather than a real document; +#: descending stops and the fact is recorded instead of silently dropped. +_MAX_TABLE_DEPTH = 24 + +#: Zip tail-record signatures (APPNOTE): End Of Central Directory, ZIP64 EOCD +#: locator, ZIP64 EOCD record. +_SIG_EOCD = b"PK\x05\x06" +_SIG_Z64_LOCATOR = b"PK\x06\x07" +_SIG_Z64_EOCD = b"PK\x06\x06" + +#: Fixed record lengths, in bytes. +_EOCD_LEN = 22 +_Z64_LOCATOR_LEN = 20 +_Z64_EOCD_FIXED_LEN = 56 + +#: Tail window read by the GUARD: the widest comment the EOCD can declare +#: (65,535) plus the EOCD itself (22) plus the ZIP64 locator (20), so the +#: locator always sits inside the single read. The guard's whole worst case is +#: 4 head bytes + this window + one 56-byte ZIP64-EOCD read = 65,637 bytes; +#: not one central-directory byte is touched. +_TAIL_WINDOW = 65_535 + _EOCD_LEN + _Z64_LOCATOR_LEN # 65,577 + +#: Minimum fixed length of one central-directory file header (APPNOTE 4.3.12). +#: An entry can carry extra name/extra/comment bytes on top of this, never +#: fewer, so `entries_total * 46 > cd_size` is a provable structural lie. +_CD_HEADER_MIN = 46 + + +def _tail_record_error(detail: str) -> str: + """Every pre-open structural lie exits as corrupt/truncated (§3 style).""" + return f"corrupt or truncated zip archive: {detail}" + + +def _read_zip_tail_records(p: Path, file_size: int): + """Pre-open reader for the GUARD. Returns (entries_total, cd_size, + cd_offset) or a reason string; a str return means refuse, before any + ZipFile exists. + + Reads only the file's tail: at most _TAIL_WINDOW (65,577) bytes, scanned + right-to-left for the EOCD signature. A candidate is accepted only if its + comment-length field puts the record's end exactly at EOF -- we do not go + fishing for another record. A saturated classic field (0xFFFF / 0xFFFFFFFF) + means "consult ZIP64" (APPNOTE 4.4.1.4); the sentinels fire INDEPENDENTLY + PER FIELD, so an archive with entries_total == 0xFFFF and a perfectly + normal 32-bit cd_size is a real case, not a hypothetical one -- each + saturated field is replaced from the ZIP64 record and each unsaturated + field is kept as-is. Every structural lie found here is refused as + corrupt/truncated rather than passed on to `zipfile` to interpret. + """ + if file_size < _EOCD_LEN: + return _tail_record_error( + f"file is {file_size} bytes, too short to hold a {_EOCD_LEN}-byte " + f"end-of-central-directory record") + try: + with p.open("rb") as fh: + fh.seek(file_size - _TAIL_WINDOW if file_size > _TAIL_WINDOW else 0) + tail = fh.read(_TAIL_WINDOW) + tail_start = file_size - len(tail) + # One candidate, the rightmost: if its comment length does not put + # the record's end exactly at EOF, the archive is corrupt and we + # refuse -- we do not go fishing for another record. + eocd_off = tail.rfind(_SIG_EOCD) + if (eocd_off < 0 or eocd_off + _EOCD_LEN > len(tail) + or (tail_start + eocd_off + _EOCD_LEN + + int.from_bytes(tail[eocd_off + 20:eocd_off + 22], + "little")) != file_size): + return _tail_record_error( + "no end-of-central-directory record ends exactly at EOF") + entries16 = int.from_bytes(tail[eocd_off + 10:eocd_off + 12], "little") + cd_size = int.from_bytes(tail[eocd_off + 12:eocd_off + 16], "little") + cd_offset = int.from_bytes(tail[eocd_off + 16:eocd_off + 20], "little") + + # Sentinels fire INDEPENDENTLY PER FIELD: any relevant field + # saturated on its own forces the ZIP64 consult (an archive can + # have entries_total == 0xFFFF while cd_offset is a perfectly + # normal 32-bit value). Once consulted, the ZIP64 record carries + # the actual values: total entries at +32, cd_size at +40; a + # saturated cd_offset is replaced from +48, an unsaturated + # 32-bit cd_offset stays as the EOCD declared it. + entries_sentinel = entries16 == 0xFFFF + size_sentinel = cd_size == 0xFFFFFFFF + offset_sentinel = cd_offset == 0xFFFFFFFF + entries_total = entries16 + if entries_sentinel or size_sentinel or offset_sentinel: + loc_off = eocd_off - _Z64_LOCATOR_LEN + if loc_off < 0 or tail[loc_off:loc_off + 4] != _SIG_Z64_LOCATOR: + return _tail_record_error( + "zip64 sentinel field in the EOCD but no zip64 EOCD " + "locator immediately before it") + locator_start = tail_start + loc_off + z64_offset = int.from_bytes(tail[loc_off + 8:loc_off + 16], "little") + if z64_offset < 0 or z64_offset + _Z64_EOCD_FIXED_LEN > file_size: + return _tail_record_error( + f"zip64 EOCD locator points at offset {z64_offset}, " + f"outside the archive") + z64_off = z64_offset - tail_start + if 0 <= z64_off and z64_off + _Z64_EOCD_FIXED_LEN <= len(tail): + rec = tail[z64_off:z64_off + _Z64_EOCD_FIXED_LEN] + else: + # ZIP64 record sits before the tail window (only possible + # behind a near-maximal EOCD comment): one extra bounded + # read of exactly the 56-byte fixed portion. + fh.seek(z64_offset) + rec = fh.read(_Z64_EOCD_FIXED_LEN) + if rec[:4] != _SIG_Z64_EOCD: + return _tail_record_error( + f"zip64 EOCD locator points at offset {z64_offset}, " + f"which does not begin a zip64 EOCD record") + # record-size field (ZIP64 EOCD +4): `value + 12` is the whole + # record, and the record must end exactly at the locator. + rec_size = int.from_bytes(rec[4:12], "little") + if z64_offset + rec_size + 12 != locator_start: + return _tail_record_error( + f"zip64 EOCD record declares size {rec_size}; size+12 " + f"does not end at the locator") + if entries_sentinel: + entries_total = int.from_bytes(rec[32:40], "little") + if size_sentinel: + cd_size = int.from_bytes(rec[40:48], "little") + if offset_sentinel: + cd_offset = int.from_bytes(rec[48:56], "little") + + if cd_offset + cd_size > file_size: + return _tail_record_error( + f"central directory claims offset {cd_offset} + size " + f"{cd_size}, past end of file ({file_size} bytes)") + if entries_total * _CD_HEADER_MIN > cd_size: + return _tail_record_error( + f"central directory claims {entries_total} entries but is " + f"only {cd_size} bytes; at {_CD_HEADER_MIN} bytes minimum " + f"per entry the count is a structural lie") + return entries_total, cd_size, cd_offset + except OSError as exc: + return _tail_record_error(f"cannot read archive tail: {exc}") + + +def _local(tag: str) -> str: + return tag.rpartition("}")[2] + + +def _block_children(container, unhandled: set): + """Yield (block_index, element, localname) for w:p / w:tbl in document order.""" + idx = 0 + for child in container: + if child.tag in (W + "p", W + "tbl"): + yield idx, child, _local(child.tag) + idx += 1 + elif _local(child.tag) not in _INERT_BLOCK_TAGS: + unhandled.add(_local(child.tag)) + + +def _named_children(container, name: str, unhandled: set): + want = W + name + for child in container: + if child.tag == want: + yield child + elif _local(child.tag) not in _INERT_BLOCK_TAGS: + unhandled.add(_local(child.tag)) + + +def _collect_paragraphs(container, prefix: tuple, unhandled: set, out: list, + depth: int = 0) -> None: + """Depth-first walk collecting (path, w:p element) pairs in reading order.""" + for idx, child, local in _block_children(container, unhandled): + path = prefix + (idx,) + if local == "p": + out.append((path, child)) + continue + if depth >= _MAX_TABLE_DEPTH: + unhandled.add("tbl(nesting-depth-exceeded)") + continue + for r, row in enumerate(_named_children(child, "tr", unhandled)): + for c, cell in enumerate(_named_children(row, "tc", unhandled)): + _collect_paragraphs(cell, path + (r, c), unhandled, out, depth + 1) + + +def _paragraph_text(el, acc: list, runs: list) -> None: + """Append a paragraph's visible text to `acc`; count w:r runs into `runs`. + + Only ``w:t`` is visible text. ``w:delText`` (tracked deletion) and + ``w:instrText`` (field instruction codes such as a HYPERLINK target) are + not visible content and are never extracted. ``w:pPr`` is skipped whole, + because ``w:pPr/w:tabs/w:tab`` is a tab-stop *definition* and would + otherwise inject a tab character that is not in the document. + """ + for child in el: + tag = child.tag + if tag in (W + "pPr", W + "rPr", W + "del", W + "moveFrom", + W + "instrText", W + "delText", W + "delInstrText", + W + "fldChar"): + continue + if tag == W + "r": + runs[0] += 1 + elif tag == W + "t": + acc.append(child.text or "") + continue + elif tag == W + "tab": + acc.append("\t") + continue + elif tag in (W + "br", W + "cr"): + acc.append("\n") + continue + elif tag == W + "noBreakHyphen": + acc.append("-") + continue + elif tag == W + "softHyphen": + continue + _paragraph_text(child, acc, runs) + + +def _prolog_has_doctype(raw: bytes) -> bool: + """True if a DOCTYPE appears in the XML prolog. + + `xml.etree.ElementTree` expands internal general entities, so a DTD is an + entity-expansion (billion-laughs) vector, and the C parser exposes no expat + handle to disable it. A conforming OOXML part has no DTD at all, so the + honest guard is to refuse one. + + The scan walks the prolog per the XML grammar -- + ``prolog ::= XMLDecl? Misc* (doctypedecl Misc*)?`` -- stopping at the root + element's ``<``. It therefore cannot be fooled by the characters + ``= n or raw[i] != 0x3C: # not a '<' -> prolog is over + return False + if raw.startswith(b"", i) + if j < 0: + return False + i = j + 2 + continue + if raw.startswith(b"", i) + if j < 0: + return False + i = j + 3 + continue + return False # root element reached + return False + + +def _docx_failed(path: str, reason: str, + parts: PartInventory | None = None) -> DocumentText: + """Every DOCX failure exit goes through here: zero blocks, zero pages, a reason.""" + return DocumentText(path=path, kind="docx", + locator_scheme=LOCATOR_SCHEME_DOCX, + status="extraction-failed", reason=reason, + parts=parts or PartInventory()) + + +def extract_docx(path: str, *, max_part_bytes: int = MAX_PART_BYTES) -> DocumentText: + """Extract `word/document.xml` text with structural locators. Never paginates. + + Returns a DocumentText whose `blocks` carry `DocxLocator`s and whose + `pages` is empty in every case, success or failure. Every zip/XML defect + is an explicit `extraction-failed` with a reason — never an empty-string + success. The source is opened read-only and is never modified. + """ + p = Path(path) + try: + with p.open("rb") as fh: + head = fh.read(4) + file_size = p.stat().st_size + except OSError as exc: + return _docx_failed(path, f"cannot read file: {exc}") + if head not in ZIP_MAGICS: + return _docx_failed( + path, f"not a zip archive: leading bytes {head!r} are not a zip " + f"signature, so the .docx extension does not match the content") + + # THE GUARD (RFC 0005 §4a): the member bound and the declared-directory + # byte cap run HERE, before `zipfile.ZipFile` is constructed, because + # `ZipFile.__init__` reads the declared central directory and materialises + # one `ZipInfo` per entry before any in-process check could run. A + # `size_limit:members` or `size_limit:central_directory` refusal means no + # ZipFile object ever existed. Do not move this below the `with`. + tail = _read_zip_tail_records(p, file_size) + if isinstance(tail, str): # a structural lie: refuse + return _docx_failed(path, tail) + preopen_entries, cd_size, _cd_offset = tail + if preopen_entries > MAX_ARCHIVE_MEMBERS: + return _docx_failed( + path, f"size_limit:members — archive declares {preopen_entries} " + f"entries, over the {MAX_ARCHIVE_MEMBERS}-member guard; " + f"refused pre-open, before any ZipFile was constructed") + if cd_size > MAX_CD_BYTES: + return _docx_failed( + path, f"size_limit:central_directory — archive declares a " + f"{cd_size}-byte central directory, over the " + f"{MAX_CD_BYTES}-byte cap; refused pre-open, before any " + f"ZipFile was constructed") + + try: + with zipfile.ZipFile(p, "r") as zf: + return _extract_docx_open(path, zf, max_part_bytes, preopen_entries) + except zipfile.BadZipFile as exc: + return _docx_failed(path, f"corrupt or truncated zip archive: {exc}") + except (EOFError, zlib.error) as exc: + return _docx_failed(path, f"corrupt zip member data: {exc}") + except RecursionError: + return _docx_failed(path, "document nesting too deep to traverse safely") + except OSError as exc: + return _docx_failed(path, f"cannot read zip archive: {exc}") + + +def _extract_docx_open(path: str, zf: zipfile.ZipFile, + max_part_bytes: int, preopen_entries: int) -> DocumentText: + names = tuple(zf.namelist()) + skipped = tuple(n for n in names if _SKIPPED_PART_RE.match(n)) + inv = PartInventory(seen=names, skipped=skipped) + + # Honesty check, NOT a resource guard — the guard already ran pre-open + # (`size_limit:members`, before ZipFile existed). No field of the tail + # records is authenticated, so an internally consistent but false count is + # not provable pre-open; this reconciliation is the honest follow-up. The + # directory `ZipFile` actually materialised must equal the pre-open count. + if len(names) != preopen_entries: + return _docx_failed( + path, f"corrupt or truncated zip archive: end-of-central-directory " + f"declares {preopen_entries} entries but the central " + f"directory holds {len(names)}; the tail records lie about " + f"the archive they describe", inv) + + # ACCEPTANCE LIMIT, not a guard (RFC 0005 §4a): `ZipFile.__init__` has + # already parsed the whole directory before this line runs, and nothing + # post-open can undo that — the pre-open guard is what bounded it. This is + # a checked sum of every member's declared uncompressed size, before any + # part is read; a declared-size sanity check, not per-member enforcement + # (the declared-plus-bounded-read pair on the one part we open stays that). + total_declared = sum(zi.file_size for zi in zf.infolist()) + if total_declared > MAX_ARCHIVE_UNCOMPRESSED_BYTES: + return _docx_failed( + path, f"size_limit:aggregate — members declare {total_declared} " + f"uncompressed bytes in total, over the " + f"{MAX_ARCHIVE_UNCOMPRESSED_BYTES}-byte acceptance limit; " + f"refused after the directory was parsed, which is why " + f"this is an acceptance limit and not a guard", inv) + + if PRIMARY_PART not in names: + return _docx_failed( + path, f"archive is a zip but has no {PRIMARY_PART} part " + f"({len(names)} member(s) seen)", inv) + if names.count(PRIMARY_PART) > 1: + # Two entries under one name: `zipfile` resolves to the last, another + # reader may take the first. Which bytes the citation pins to would be + # reader-dependent, so there is no honest answer -- refuse. + return _docx_failed( + path, f"archive contains {names.count(PRIMARY_PART)} duplicate " + f"{PRIMARY_PART} entries; which one a reader sees is " + f"ambiguous, so the content cannot be pinned", inv) + + info = zf.getinfo(PRIMARY_PART) + if info.flag_bits & 0x1: + return _docx_failed( + path, f"{PRIMARY_PART} is encrypted; refusing to guess at a password", + inv) + if info.file_size > max_part_bytes: + return _docx_failed( + path, f"{PRIMARY_PART} declares {info.file_size} decompressed bytes, " + f"too large (cap {max_part_bytes})", inv) + try: + with zf.open(info) as fh: + raw = fh.read(max_part_bytes + 1) + except RuntimeError as exc: # zipfile signals encrypted members this way + return _docx_failed(path, f"cannot read {PRIMARY_PART}: {exc}", inv) + if len(raw) > max_part_bytes: + return _docx_failed( + path, f"{PRIMARY_PART} decompressed past the {max_part_bytes}-byte cap, " + f"too large (the declared size was understated)", inv) + if not raw.strip(): + return _docx_failed(path, f"{PRIMARY_PART} is present but empty", inv) + + # The DOCTYPE guard below is a byte scan, so the part must actually be + # UTF-8. A UTF-16/32 part would scan as unrecognisable and slip past it, + # and Word does not produce one; refuse rather than parse blind. + scan = raw[3:] if raw.startswith(b"\xef\xbb\xbf") else raw + lead = scan.lstrip(b" \t\r\n") + if lead[:1] != b"<" or lead[1:2] == b"\x00": + return _docx_failed( + path, f"{PRIMARY_PART} has an unsupported encoding: it does not " + f"begin as UTF-8 XML (leading bytes {raw[:4]!r}); only UTF-8 " + f"parts can be checked for a DOCTYPE before parsing", inv) + + if _prolog_has_doctype(scan): + return _docx_failed( + path, f"{PRIMARY_PART} carries a DOCTYPE declaration; a conforming " + f"OOXML part has no DTD, and entity expansion is refused " + f"rather than evaluated", inv) + + try: + root = ElementTree.fromstring(raw) + except ElementTree.ParseError as exc: + return _docx_failed( + path, f"malformed xml in {PRIMARY_PART}: {exc}; refusing to salvage a " + f"partial parse into a success", inv) + + ns, local = (root.tag[1:].split("}", 1) if root.tag.startswith("{") + else ("", root.tag)) + if ns != W_NS: + return _docx_failed( + path, f"unsupported wordprocessingml namespace {ns!r} in {PRIMARY_PART} " + f"(expected transitional OOXML {W_NS!r}); strict OOXML and " + f"non-Word packages are not supported", inv) + if local != "document": + return _docx_failed( + path, f"unexpected root element {local!r} in {PRIMARY_PART} " + f"(expected 'document')", inv) + body = root.find(W + "body") + if body is None: + return _docx_failed(path, f"{PRIMARY_PART} has no w:body element", inv) + + unhandled: set = set() + found: list = [] + _collect_paragraphs(body, (), unhandled, found) + + blocks: list[Block] = [] + pieces: list[str] = [] + cursor = 0 + for i, (bpath, p_el) in enumerate(found): + acc: list[str] = [] + runs = [0] + _paragraph_text(p_el, acc, runs) + text = "".join(acc) + blocks.append(Block( + locator=DocxLocator(part=PRIMARY_PART, path=bpath), + index=i, text=text, + char_start=cursor, char_end=cursor + len(text), + run_count=runs[0], + text_sha256=hashlib.sha256(text.encode("utf-8")).hexdigest())) + pieces.append(text) + cursor += len(text) + 1 # +1 for the "\n" joiner used below + + full = "\n".join(pieces) + return DocumentText( + path=path, kind="docx", + locator_scheme=LOCATOR_SCHEME_DOCX, + status="extracted", reason=None, + blocks=blocks, text=full, + text_sha256=hashlib.sha256(full.encode("utf-8")).hexdigest(), + parts=PartInventory(seen=names, extracted=(PRIMARY_PART,), + skipped=skipped, unhandled=tuple(sorted(unhandled)))) + + +_BLOCK_COLUMNS = ( + "document_sha256", "locator_scheme", "part", "locator", "block_index", + "container_type", "path", "char_start", "char_end", "text", "text_sha256", + "status", "reason", +) + + +def record_blocks(conn, document_sha256: str, doc: DocumentText) -> int: + """Write a DOCX extraction into `document_blocks` on `conn`. Returns rows written. + + The caller owns the transaction — this function never commits, so the + write can be rolled back with the rest of the caller's unit of work. + + A failed extraction writes exactly one row — status `extraction-failed`, + no text, the reason kept — so the store records that the document was + examined and could not be read. Writing nothing would read as "not yet + extracted"; writing an empty string would read as "no content". Both are + lies about the evidence. + + Re-extracting the same bytes rewrites the same rows: the primary key is + (document_sha256, part, locator). + """ + if doc.locator_scheme != LOCATOR_SCHEME_DOCX: + raise ValueError( + f"record_blocks handles {LOCATOR_SCHEME_DOCX} only, got " + f"{doc.locator_scheme!r}; page-scheme extractions belong in " + f"document_pages") + rows = [] + whole_part = DocxLocator(PRIMARY_PART, ()).canonical() + if doc.status == "extraction-failed": + rows.append((document_sha256, LOCATOR_SCHEME_DOCX, PRIMARY_PART, + whole_part, -1, "part", "[]", + None, None, None, None, "extraction-failed", doc.reason)) + elif not doc.blocks: + # A well-formed document with no paragraphs. Writing nothing would be + # indistinguishable from "never extracted", so record the result. + rows.append((document_sha256, LOCATOR_SCHEME_DOCX, PRIMARY_PART, + whole_part, -1, "part", "[]", 0, 0, "", + hashlib.sha256(b"").hexdigest(), "extracted", + "document is well-formed and contains no paragraphs")) + else: + for b in doc.blocks: + rows.append((document_sha256, LOCATOR_SCHEME_DOCX, b.locator.part, + b.locator.canonical(), b.index, b.locator.container_type, + json.dumps(list(b.locator.path)), + b.char_start, b.char_end, b.text, b.text_sha256, + b.status, None)) + placeholders = ",".join("?" * len(_BLOCK_COLUMNS)) + conn.executemany( + f"INSERT OR REPLACE INTO document_blocks ({','.join(_BLOCK_COLUMNS)}) " + f"VALUES ({placeholders})", rows) + return len(rows) diff --git a/matterkit/store.py b/matterkit/store.py index f524288..947f5e3 100644 --- a/matterkit/store.py +++ b/matterkit/store.py @@ -59,6 +59,38 @@ status TEXT NOT NULL CHECK (status IN ('extracted','extraction-failed')), created_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS document_pages ( + document_sha256 TEXT NOT NULL, + page_number INTEGER NOT NULL, + text_sha256 TEXT, + text TEXT, + status TEXT NOT NULL CHECK (status IN ('extracted','extraction-failed')), + PRIMARY KEY (document_sha256, page_number) +); +-- RFC 0005: DOCX content, addressed by STRUCTURAL locator. +-- Deliberately a sibling of document_pages rather than a reuse of it: DOCX +-- has no rendered page numbers (pagination is produced by the renderer, not +-- stored in the file), so a page_number here would be a fabricated citation. +-- There is therefore no page_number column, and locator_scheme is constrained +-- to the one scheme this table can honestly hold. Purely additive: existing +-- stores gain the table on next connect; no data migration, and nothing in +-- document_pages is read or rewritten. +CREATE TABLE IF NOT EXISTS document_blocks ( + document_sha256 TEXT NOT NULL, + locator_scheme TEXT NOT NULL CHECK (locator_scheme IN ('docx-structural')), + part TEXT NOT NULL, -- OOXML part, e.g. word/document.xml + locator TEXT NOT NULL, -- canonical path, e.g. word/document.xml#b3/r1/c0/b0 + block_index INTEGER NOT NULL, -- reading-order ordinal; -1 on a failure row + container_type TEXT NOT NULL + CHECK (container_type IN ('paragraph','table-cell','part')), + path TEXT NOT NULL, -- JSON array of the locator's integer path + char_start INTEGER, char_end INTEGER, -- offsets into the extracted part text + text TEXT, + text_sha256 TEXT, + status TEXT NOT NULL CHECK (status IN ('extracted','extraction-failed')), + reason TEXT, + PRIMARY KEY (document_sha256, part, locator) +); """ VALID_STATUS = {"DOCUMENTED FACT", "ALLEGATION", "INFERENCE", "HYPOTHESIS", "UNKNOWN"} diff --git a/rfcs/0004-document-extraction-viewer-gaps.md b/rfcs/0004-document-extraction-viewer-gaps.md new file mode 100644 index 0000000..668af1c --- /dev/null +++ b/rfcs/0004-document-extraction-viewer-gaps.md @@ -0,0 +1,39 @@ +# Document extraction & read-only viewer — gap inventory (bounded) + +**Date:** 2026-09-11 · **Owner:** @flash (chair assignment) · **Repo:** `teakesdev/corpus-open` (`/Users/ty/dev/matter-kit`, HEAD `f35efb2`) + +## What exists today (verified against the tree) + +| capability | state | evidence | +|---|---|---| +| Ingest hashes + classifies files | **yes** | `ingest.py`: `sha256_file`, `classify` (ext→kind + dir-hint), `ingest_paths`, `verify_integrity`. Sources never modified. | +| PDF/DOCX are *recognized* | **yes, as blobs** | `EXT_KINDS`: `.pdf→filing`, `.docx→filing` — registered and content-hashed only. | +| **PDF/DOCX text extraction** | **NO** | grep for pypdf/pdfplumber/fitz/mupdf/docx/tesseract → nothing except a comment in citations.py. `ingest.py` has no `extract`/`text`/`page` symbol. | +| Page locators on passages | **NO** | no `passages` table; `documents` is hash+path only. | +| Assertion → source pinning | **yes, coarse** | `assertion_sources(document_id, locator)` — locator is a free string, no page/offset schema, no click-through. | +| Read-only viewer | **NO** | no viewer/browse/inspect module; `packet.py` renders markdown from DB rows only. | +| Timeline / evidence matrix | **NO** | `events` + `issues` tables exist (empty scaffolds); no chronology renderer, no issue↔evidence join UI. | +| MCP surface | facts/deadlines/authorities/cite_extract + adapters | no document/read/view tools. | + +## The smallest missing vertical slice (what I am implementing) + +**Local PDF text extraction with page locators** — the one capability that turns the kit from "hash store" into "document intelligence," and the documented flagship workflow's first real step. + +Design (zero-dependency, chair's "lightweight default"): +- **README/text paths unchanged.** `.txt`/`.md` already ingest as-is. +- **New module `matterkit/extract.py`** — stdlib-only primary path, **no pypdf dep by default**: + - `extract_pdf(path) -> list[Page]`: a minimal, honest stdlib PDF text extractor covering the common text-stream/FlateDecode case (Tj/TJ operators, `/FlateDecode` via `zlib`). Returns per-page text + page number. **Not a full PDF engine** — streams using `/LZWDecode`, Type0/CID fonts, or scanned (image-only) pages return `extraction_failed` on that page, never silent empty text. + - `extract_text(path)` dispatches by extension (`.pdf`, plain, else null -> caller decides). +- **Optional heavier deps** (pdfplumber/tesseract) are a documented *optional* install (`extras_require`) — not required, not imported unless present. Keeps default install stdlib. +- **Page-locator storage:** `cite_extract`-style provenance is already the pattern; extraction writes to a new `document_pages` table scoped by `document_sha256` + page number + `text_sha256` (offsets always relative to the exact per-page text). + +## Out of scope (this slice) +OCR for scanned pages, docx unpacking, full tables/layout, the read-only viewer UI, timeline/matrix join, any model-produced facts. + +## Vertical-slice acceptance (synthetic + public fixtures) +1. A synthetic 2-page PDF (text-stream/FlateDecode) extracts both pages, correct page numbers, non-empty, `text_sha256` stable. +2. A scanned/image-only PDF returns `extraction_failed` honestly (no silent empty). +3. A `.txt` file extracts identically to raw read. +4. Extracted page text reads back against source offsets (`text[start:end]` matches). +5. No new hard dependency: `import matterkit.extract` works with stdlib only. +6. Existing 88 tests stay green. \ No newline at end of file diff --git a/rfcs/0005-docx-structural-extraction.md b/rfcs/0005-docx-structural-extraction.md new file mode 100644 index 0000000..c9f87b0 --- /dev/null +++ b/rfcs/0005-docx-structural-extraction.md @@ -0,0 +1,496 @@ +# RFC 0005 — DOCX text extraction with structural locators + +**Date:** 2026-09-13 · **Repo:** `teakesdev/corpus-open` (`/Users/ty/dev/matter-kit`) +**Branches from:** `main` @ `ca9f0d6` · **Depends on:** RFC 0004 (`matterkit/extract.py`, `document_pages`, the honest-`extraction_failed` precedent) + +Closes the last "recognized as blob" gap for text-bearing documents in the RFC 0004 +inventory. `.docx` has been a first-class `EXT_KINDS` entry (`ingest.py`) since the +beginning; until now it was hashed and never read. + +--- + +## 1. The pagination problem, and why this RFC does not solve it + +WordprocessingML stores **no page numbers**. Pagination is produced by the rendering +engine at layout time from the font metrics, printer/page setup, field results and +hyphenation dictionary available on the machine doing the rendering. The same `.docx` +paginates differently in Word 2019 and Word 365, on macOS and Windows, and in +LibreOffice. An explicit `` is a *hint to the renderer*, not an +authoritative boundary: content still reflows around it. + +So there is no page number to read out of the file. Any page number this kit attached +to DOCX content would be one **this tool invented** — a fabricated legal-grade +citation, which is precisely the failure mode the kit exists to refuse. RFC 0004 +established the parallel principle for PDFs: never a silent empty string, because a +silent empty loses evidence. The same principle here forbids a confident-looking +locator that no one can check against the source. + +**Decision: DOCX locators are structural, never paginal.** Concretely, and enforced in +three places rather than by convention: + +| Enforcement | Where | +|---|---| +| `extract_docx` returns `pages == []` in every case, success and failure | `matterkit/extract.py` | +| `document_blocks` has **no** `page_number` column, and a CHECK constrains `locator_scheme` to `docx-structural` | `matterkit/store.py` | +| `record_blocks` raises on any non-`docx-structural` extraction, so the block store cannot be used to launder a page locator | `matterkit/extract.py` | +| `matter pages` routes `.docx` to `document_blocks` and writes no `document_pages` row | `matter.py` | + +### 1a. Storage: why a sibling table, not a `locator_scheme` column on `document_pages` + +The brief allowed either. A sibling table was chosen because it makes the invariant +*structural* instead of *procedural*: with a shared table, "never put a paragraph index +in `page_number`" is a rule someone has to keep remembering, and a single careless +`INSERT` breaks it silently. With a separate table there is no `page_number` column to +misuse, and the CHECK constraint means the database itself refuses the mistake. + +The migration is therefore **purely additive**: `CREATE TABLE IF NOT EXISTS +document_blocks` runs on the next `store.connect()`. No existing row is read, rewritten +or backfilled; `document_pages` is untouched; an older kit reading a newer store simply +does not see the table. + +--- + +## 2. The locator contract + +A citation is the triple **`(document_sha256, part, locator)`**. + +**`document_sha256`** — sha256 of the exact source bytes. A locator is valid only +against the byte-identical file. Editing a DOCX yields a different sha256 and therefore +a different citation space; a stale citation cannot silently re-point at moved content. +Sources are opened read-only and are never rewritten (RFC 0004's immutability +discipline, covered by a test that re-hashes the file after extraction). + +**`part`** — the OOXML part the locator addresses. v1 addresses exactly one: +`word/document.xml`. + +**`locator`** — `#//…`, a path of integer coordinates through +the part's *block tree*, with a one-letter type prefix cycling `b` (block child), +`r` (table row), `c` (table cell). + +### 2a. Path construction + +Within a container — the body, or a table cell — the recognised block-level children +(`w:p` and `w:tbl`, in document order, skipping property and bookmark elements that +carry no text) are numbered from 0. + +- a `w:p` at index *i* is addressed `(…, i)` +- to reach content inside a `w:tbl` at index *i*, append the 0-based row and cell + indexes and recurse into that cell: `(…, i, r, c)` + +Every path therefore has length `3n + 1`, and **the addressed unit is always a +paragraph** — the smallest block that has a stable identity in the format: + +``` +word/document.xml#b3 4th block child of the body +word/document.xml#b3/r1/c0/b2 3rd paragraph of row 1, cell 0 + of the table at body index 3 +word/document.xml#b0/r0/c0/b1/r2/c1/b0 nested one table deeper +``` + +`container_type` is derived, not stored independently: `paragraph` when the path has +length 1, `table-cell` otherwise (the paragraph lives inside a cell), `part` for the +whole-part rows described in §3. + +### 2b. Uniqueness and collision properties + +*(this is the property the review gate is aimed at)* + +- **Within a part:** a path is a coordinate in a tree. Two distinct blocks have distinct + paths, because at the first index where their traversals diverge the integers differ. + No two blocks in a part can collide. +- **Across parts:** impossible, because `part` is part of the citation. +- **Across documents:** impossible, because `document_sha256` is. +- **Identical text is not identical identity.** Two paragraphs reading "Same text." have + the same `text_sha256` and two different locators. The locator identifies a *position*; + the hash describes the *content*. Conflating them is what lets a citation drift. +- The storage primary key `(document_sha256, part, locator)` enforces all of the above, + and makes re-extraction of the same bytes idempotent rather than duplicative. + +The canonical string form is total and injective: `DocxLocator.parse(loc.canonical()) +== loc` for every locator, and `parse` rejects malformed segments and any path whose +length is not `3n + 1`. Tests cover round-tripping, the four cells of a 2×2 table, a +nested table, and a whole mixed document (10 blocks, 10 distinct locators). + +### 2c. Offsets + +`DocumentText.text` is the extracted text of the part: block texts joined by a single +`\n` in reading order. Each block carries `char_start`/`char_end` into that string: + +``` +doc.text[block.char_start:block.char_end] == block.text +``` + +so a quoted passage re-derives exactly from the stored offsets. **The joiner is not a +delimiter**: a `w:br` inside a paragraph also renders as `\n`, so `doc.text` must not be +re-split on newlines to recover blocks — the stored offsets are the only authoritative +boundaries. Offsets index the extracted text, never the raw XML. + +### 2d. Why character offsets rather than run indexes + +The brief's sketch was `{part, container_type, container_index, run_start, run_end}`. +This implementation keeps the first three and replaces the run pair with character +offsets, because a `w:r` boundary is a formatting artefact, not a semantic one: Word +re-splits runs on spell-check, on revision tracking, and on edits elsewhere in the +paragraph. "Run 3 of paragraph 7" names a different substring after a trivial edit, +which is exactly the instability a citation must not have. Character offsets into the +extracted text are stable under the same conditions and are directly checkable by a +reader. + +`run_count` is still recorded — it demonstrates the run traversal happened and is useful +when diagnosing a paragraph — but it is **never part of a citation**. + +### 2e. What counts as text + +Only `w:t` is visible text. + +- `w:delText` (tracked-change deletions) is **never** extracted. Quoting struck-out text + as document content misstates the evidence. +- `w:instrText` (field instruction codes, e.g. a `HYPERLINK` target) is never extracted; + the field's visible result is a normal `w:t` and is. +- `w:ins` content *is* extracted — it is text present in this version of the document. +- `w:tab` → `\t`, `w:br`/`w:cr` → `\n`, `w:noBreakHyphen` → `-`, `w:softHyphen` → nothing. +- `w:pPr` is skipped whole, because `w:pPr/w:tabs/w:tab` is a tab-stop *definition* and + a naive descendant walk would inject a tab character that is not in the document. +- Element *tails* are ignored, so pretty-printing whitespace in the XML never enters the + extracted text. + +An empty paragraph is an addressable block with empty text, not a dropped one — it holds +a real structural position that later blocks are numbered against. + +--- + +## 3. Failure vocabulary + +`status` is `extracted` or `extraction-failed` at the **document** level, with a +human-readable `reason` on failure. There is no partial success: a failure yields zero +blocks and is never a zero-length success. + +| Input | Result | Reason contains | +|---|---|---| +| Not a zip (renamed `.txt`, `.pdf`, anything without a zip signature) | failed | `not a zip archive` | +| Polyglot — another format with a zip appended | failed | signature is checked on the leading bytes, before `zipfile` | +| Truncated / corrupt archive | failed | `corrupt or truncated zip archive` | +| CRC mismatch in the document part | failed | `corrupt or truncated zip archive: Bad CRC-32` | +| Encrypted member | failed | `is encrypted; refusing to guess at a password` | +| Part exceeds the decompressed cap (declared **or** actual) | failed | `too large` | +| No `word/document.xml` | failed | `has no word/document.xml part` | +| Two `word/document.xml` entries | failed | `duplicate … entries … ambiguous` | +| `word/document.xml` present but blank | failed | `present but empty` | +| Non-UTF-8 part (UTF-16/32, with or without a BOM) | failed | `has an unsupported encoding` | +| DOCTYPE present (including behind a UTF-8 BOM) | failed | `carries a DOCTYPE declaration` | +| Malformed XML | failed | `malformed xml … refusing to salvage a partial parse` | +| Strict-OOXML or other namespace | failed | `unsupported wordprocessingml namespace` | +| Root element is not `w:document` | failed | `unexpected root element` | +| No `w:body` | failed | `has no w:body element` | +| **Well-formed, no paragraphs** | **extracted**, 0 blocks, `reason is None` | — | + +### 3a. Extension-vs-magic mismatch is refused, not re-dispatched + +A `.docx` whose bytes are a PDF fails. It is *not* re-routed to `extract_pdf`. A file +whose name contradicts its content is exactly the provenance problem the kit exists to +surface; quietly reading it under a different extractor would record a document that +does not match how anyone else on the matter refers to it. + +### 3b. Three things the store must be able to tell apart + +1. **never extracted** — no row. +2. **extracted, could not be read** — one row, `status='extraction-failed'`, `text IS NULL`, + `reason` preserved. +3. **extracted, genuinely contains nothing** — one row, `status='extracted'`, + `container_type='part'`, `text=''`, with a reason noting the document is well-formed + and has no paragraphs. + +Writing nothing in case 3 would collapse it into case 1; writing an empty string without +a row would collapse cases 2 and 3. Both are lies about the evidence, so both are tested. + +--- + +## 4. Hostile-input policy + +- **Extension/magic check first.** The leading bytes must be a zip signature before the + file reaches `zipfile`, which otherwise locates a central directory anywhere in the + file and would happily open a polyglot. +- **Zip-bomb guard, checked twice.** `MAX_PART_BYTES` (32 MiB, overridable per call via + `max_part_bytes=`) is enforced against the *declared* decompressed size and then again + against a bounded read, so a forged header does not get past it. +- **DOCTYPE refused.** `xml.etree.ElementTree` expands internal general entities — the + billion-laughs vector — and the C parser in CPython exposes no expat handle to disable + it. A conforming OOXML part has no DTD, so refusing one costs nothing and removes the + class of attack. The check walks the XML prolog per the grammar + (`prolog ::= XMLDecl? Misc* (doctypedecl Misc*)?`) and stops at the root element, so it + cannot be fooled by the characters ` cd_size` is a lie). What is *not* provable pre-open: an internally +consistent but false count, or a downward lie about `cd_size` that still fits +`count × 46`. No field is authenticated, and we will not claim otherwise. So one +cheap post-open reconciliation is added — `len(zf.namelist())` must equal the +pre-open count; a mismatch is refused as corrupt, never silently trusted — labelled +as what it is: an honesty check, not a resource guard. The guard already ran. + +**`MAX_ARCHIVE_MEMBERS = 10_000`.** A real Word `.docx` is ~10–40 parts (a minimal +package is ~10; media- and annotation-heavy documents reach the hundreds). 10,000 is +two to three orders of magnitude of headroom. It bounds the declared count on +honest-or-overstated tails; a hostile archive that *understates* the count still +reaches `ZipFile`. + +**`MAX_CD_BYTES = 32 MiB`.** Same order as `MAX_PART_BYTES`. A real Word directory is +kilobytes; 32 MiB is tens of megabytes of headroom and is the number CPython 3.11 +`zipfile._RealGetContents` will `fp.read()` if we let construction start. It bounds +parser input for honest-or-overstated `cd_size`. It does not bound RAM, and it does +not bound the actual member count. A 6 MB directory under a lying count of 5 is +under this cap and is detected post-open. + +#### Aggregate uncompressed bytes — an acceptance limit, not a guard + +**No: an aggregate-uncompressed bound is not achievable pre-open.** Per-member +uncompressed sizes live in the central-directory entries; the EOCD gives only the +directory's *size* and *count*, and no tail record sums member sizes. Obtaining the sum +requires parsing the directory — the very cost the guard exists to avoid. The honest +split, and the labels must not be blurred: + +- **GUARD (pre-open):** the member-count bound and the declared-directory byte cap + above. Refuses before any `ZipFile` exists, and only for honest-or-overstated tails. +- **ACCEPTANCE LIMIT (post-open):** `MAX_ARCHIVE_UNCOMPRESSED_BYTES = 256 MiB` + (8 × `MAX_PART_BYTES`), a checked sum of every member's declared `file_size` in + `_extract_docx_open`, before any part read. It refuses *after* the directory was + already parsed, and it is a declared-size sanity check: we never decompress members + other than `word/document.xml`, so their real inflation is never realised + in-process, and the existing declared-plus-bounded-read pair stays the only + per-member enforcement. + +#### Residual exposure + +- A `members` or `central_directory` refusal reads ≤ `4` head bytes + ≤ `65,557` tail + bytes, plus ≤ `20` (locator — widen the tail window to `65,577` so it is always + inside) and ≤ `56` (ZIP64 EOCD) when sentinels are present: **worst case ≤ 65,637 + bytes, zero central-directory bytes, zero `ZipFile` constructor attempts.** +- A lying tail (understated count with an honest `cd_size` still under `MAX_CD_BYTES`; + or a downward `cd_size` lie that still passes `count × 46`) is not preventable + pre-open. `ZipFile.__init__` then `fp.read`s the declared `cd_size` (CPython 3.11 + `_RealGetContents`) and either materialises the directory or raises `BadZipFile` + mid-construction. On the measured downward-`cd_size` fixture the message is + `"Bad magic number for central directory"`: patching size but not offset shifts + CPython's `concat`, so the bounded read is taken from the wrong place. A short + file can instead raise `"Truncated central directory"`. Either way the declared + size bounds parser input; zero successful constructor returns is not proof of + zero allocation. Allocation on that path is typical, bounded by file size, not + an enforced ceiling. Post-open detection (count reconciliation, or the + `except BadZipFile` handler) does not undo it. +- An `aggregate` refusal cannot prevent what already happened: the directory parse + the guard allowed. The acceptance limit does not remove it; nothing post-open can. +- Unavoidable on every accepted path: the tail window itself, and `zipfile`'s own + re-parse of the EOCD after our guard passes. + +#### Parser agreement (tests, not a cross-version fact) + +The pre-open reader and CPython `zipfile`'s `_EndRecData` / `_EndRecData64` are +checked against each other by test: honest EOCD, and ZIP64 locator+record with every +classic field saturated. Agreement means the same effective +`(entries_total, cd_size, cd_offset)`. These checks are pinned to the CPython 3.11 +implementation they were read against (`_EndRecData` takes `rfind(stringEndArchive)` +— rightmost — and assumes the magic does not appear in the comment). They carry +forward as tests, not as a settled property of every Python. + +Documented divergences, all fail-closed on our side, none a defect: + +- **Magic in the EOCD comment (`PK\x05\x06`).** CPython selects the in-comment + occurrence. No right-to-left repair scan restores agreement, and none is + implemented (P3-1: withdrawn). We refuse because that occurrence's comment-length + field does not put a record at EOF. Tests assert *our refusal only*. +- **Comment-length field that does not land on EOF.** We refuse. CPython still + returns the classic count/size/offset and `ZipFile` opens. Same class as P3-1. +- **Per-field sentinels vs CPython's all-fields overwrite.** APPNOTE 4.4.1.4 + saturates independently per field; we keep unsaturated classic values. CPython + `_EndRecData64`, once a locator is present, replaces count, size, and offset from + the ZIP64 record together. Agreement tests use the common all-sentinel ZIP64 + shape; mixed-sentinel fixtures pin *our* APPNOTE behaviour, not CPython's. + +#### Failure vocabulary (§3 style) + +| Condition | Reason starts | Fires | +|---|---|---| +| total-entry count > `MAX_ARCHIVE_MEMBERS` | `size_limit:members` | pre-open — the GUARD; no `ZipFile` constructed | +| declared `cd_size` > `MAX_CD_BYTES` | `size_limit:central_directory` | pre-open — the GUARD; no `ZipFile` constructed | +| Σ declared `file_size` > `MAX_ARCHIVE_UNCOMPRESSED_BYTES` | `size_limit:aggregate` | post-open — acceptance limit; directory already parsed | + +All three exit through `_docx_failed`: `status='extraction-failed'`, zero blocks, zero +pages, reason preserved. A downward `cd_size` lie that reaches `ZipFile` and raises +`BadZipFile` is `corrupt or truncated zip archive: …`, distinguishable from +`size_limit:central_directory`. + +--- + +## 5. Part-set policy + +Only `word/document.xml` is read. `PartInventory` makes the rest legible rather than +leaving a reviewer to guess whether a part was skipped by design or missed: + +| field | meaning | +|---|---| +| `seen` | every member name in the archive | +| `extracted` | `("word/document.xml",)` | +| `skipped` | text-bearing parts present and deliberately not read in v1: `word/header*.xml`, `word/footer*.xml`, `word/footnotes.xml`, `word/endnotes.xml`, `word/comments*.xml` | +| `unhandled` | local names of elements found at a block position and not descended into — e.g. `sdt` (content controls) — so no content is dropped silently | + +`matter pages` prints both `skipped` and `unhandled` for each DOCX. + +--- + +## 6. Storage + +```sql +CREATE TABLE IF NOT EXISTS document_blocks ( + document_sha256 TEXT NOT NULL, + locator_scheme TEXT NOT NULL CHECK (locator_scheme IN ('docx-structural')), + part TEXT NOT NULL, + locator TEXT NOT NULL, + block_index INTEGER NOT NULL, -- reading-order ordinal; -1 on a whole-part row + container_type TEXT NOT NULL + CHECK (container_type IN ('paragraph','table-cell','part')), + path TEXT NOT NULL, -- JSON array of the locator's integer path + char_start INTEGER, char_end INTEGER, + text TEXT, + text_sha256 TEXT, + status TEXT NOT NULL CHECK (status IN ('extracted','extraction-failed')), + reason TEXT, + PRIMARY KEY (document_sha256, part, locator) +); +``` + +No `page_number`. The `locator_scheme` CHECK is deliberately single-valued: adding a +scheme is a deliberate migration with its own review, not an insert someone can make by +accident. + +### Locator schemes in the kit + +| scheme | format | locator means | table | +|---|---|---|---| +| `pdf-page` (RFC 0004) | PDF | 1-based page number read from the file's own structure | `document_pages` | +| `docx-structural` (this RFC) | DOCX | structural path through the OOXML block tree | `document_blocks` | +| `whole-file` (RFC 0004) | `.txt`/`.md`, unextractable kinds | the whole file is one unit | `document_pages` | + +--- + +## 7. Dependencies + +Stdlib only: `zipfile`, `xml.etree.ElementTree`, `hashlib`, `json`, `re`, `zlib`, +`pathlib`. No `python-docx`, no `lxml`. The floor is asserted by a test that re-imports +the module in a subprocess with every `site-packages` path stripped from `sys.path`, not +by a comment. RFC 0004's optional heavier paths (pdfplumber, tesseract) stay optional and +remain unimported. + +--- + +## 8. Acceptance + +57 new tests in `tests/test_extract_docx.py`; 150 total, green under +`python -m unittest discover -s tests` (the CI runner) on Python 3.11 and 3.14. Every fixture is +synthesised in-process from XML strings plus `zipfile` — no binary fixtures committed, no +client documents. + +Mapping to the brief's acceptance list: + +| brief | covered by | +|---|---| +| (a) 2-paragraph DOCX, correct indexes, stable `text_sha256` | `TestLocatorContract` | +| (b) 2×2 table, four cells addressed distinctly | `TestTableLocators` (plus multi-paragraph cells, nesting, a 10-block mixed document) | +| (c) truncated/corrupt zip → failed | `TestHonestFailure`, `TestHostileInput` | +| (d) zip without `word/document.xml` → failed | `TestHonestFailure` | +| (e) renamed `.txt`/PDF refused, not re-dispatched | `TestHonestFailure` (and the polyglot case in `TestHostileInput`) | +| (f) offset round-trip | `TestOffsetRoundTrip`, `TestBlockStorage` (re-sliced out of SQLite) | +| (g) 93 existing tests green; stdlib-only asserted by import check | full suite; `TestStdlibOnlyFloor` | + +Beyond the list: zero-fabricated-pagination assertions at the object, schema and CLI +levels; the empty-vs-failed distinction; tracked-deletion and field-code exclusion; +DOCTYPE (including behind a BOM), non-UTF-8, duplicate-part, CRC and zip-bomb +refusals; source-immutability re-hash. + +--- + +## 9. Out of scope (inherited from RFC 0004 unless noted) + +OCR; rendered-page claims for any format; header/footer/footnote/endnote/comment +extraction; style and formatting preservation; `.doc` (legacy binary — explicitly *not* +routed to the DOCX extractor); full OOXML schema coverage; the viewer UI; timeline and +evidence matrix; any model-produced facts. `matter-mcp.py` is untouched. + +## 10. Named gaps for v2 + +Each of these is visible in the output rather than silent: + +1. **Main-part resolution.** v1 addresses `word/document.xml` by its conventional name + instead of resolving the `officeDocument` package relationship in `_rels/.rels`. A + producer that names the main part otherwise fails with "no word/document.xml part" + rather than being read. +2. **`w:sdt` content controls** are not descended into; they appear in + `PartInventory.unhandled`. Descending needs a path segment for the SDT wrapper, which + changes the locator grammar and so wants its own review. +3. **Headers, footers, footnotes, endnotes, comments** are listed in + `PartInventory.skipped`. Extracting them needs a second `part` value per document, + which the schema already supports (`part` is in the primary key) but the extractor + does not yet produce. +4. **`document_pages` and `.txt`/`.md`.** A plain-text file is stored as + `document_pages` page 1, which is a whole-file locator wearing a page number. It + predates this RFC and is left alone here rather than widened into it, but it is the + same category of claim and should be reconciled when the viewer lands. diff --git a/tests/fixtures/zip_hostile_builders.py b/tests/fixtures/zip_hostile_builders.py new file mode 100644 index 0000000..a55dbb7 --- /dev/null +++ b/tests/fixtures/zip_hostile_builders.py @@ -0,0 +1,122 @@ +"""Hostile-zip builders for matter-kit DOCX guard tests (increment 3 fixtures). + +STAGED artifact — provenance and parameters from the 2026-09-14 flash experiment +(measured: declared-count 5 over a real 6.3 MB central directory / 3,001 entries × +2 KB comments → 1 ZipFile construction, 68 ms, 14.2 MB peak before refusal). +Commit target: tests/fixtures/zip_hostile_builders.py (stdlib-only, no matterkit imports). + +Variants provided: + 1. understated EOCD count, honest cd_size (post-open-detection case) + 2. understated EOCD count AND cd_size (deepseek's downward-lie hypothesis test) + 3. honest oversized count (pre-open members-guard case, 0 constructions) + 4. honest oversized count AND honest large cd_size (the MAX_CD_BYTES target case) + 5. EOCD comment embedding PK\x05\x06 (P3-1 right-to-left-scan case) +""" + +import io +import struct +import zipfile + +_EOCD_SIG = b"PK\x05\x06" + + +def build_docx_zip(n_members=3_000, comment_bytes=2_048, comment_payload=None): + """Build a docx-shaped zip: one real document part + n_members pad entries. + + comment_payload: bytes for pad-entry comments (default b'C'*comment_bytes). + Returns (bytes, real_entries_total, real_cd_size). + """ + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as z: + z.writestr( + "word/document.xml", + '', + ) + for i in range(n_members): + zi = zipfile.ZipInfo(f"pad/{i:05d}.bin") + zi.comment = ( + comment_payload if comment_payload is not None else b"C" * comment_bytes + ) + z.writestr(zi, b"\x00" * 8) + data = buf.getvalue() + i = data.rfind(_EOCD_SIG) + _, _, _, _, _, cd_size, _cd_offset, _clen = struct.unpack_from(" 0xFFFF: + raise ValueError("comment too long") + struct.pack_into(" bytes: + """Minimal PDF with N content streams, each its own object.""" + objs = [] + n = 1 + body = [] + offsets = [] + for i, s in enumerate(streams): + f = (filters[i] if filters else "FlateDecode") + data = zlib.compress(s) if f == "FlateDecode" else s + body.append(f"{n} 0 obj\n<< /Length {len(data)} /Filter /{f} >>\nstream\n") + body.append(data) + body.append(b"\nendstream\nendobj\n") + offsets.append((n, 0)) + n += 1 + pdf = b"%PDF-1.4\n" + pdf += b"".join(b if isinstance(b, bytes) else b.encode() for b in body) + pdf += b"%%EOF\n" + return pdf + + +class TestStdlibOnly(unittest.TestCase): + def test_import_needs_no_third_party(self): + # if this import already succeeded, stdlib-only holds; assert no pypdf. + self.assertNotIn("pypdf", sys.modules) + self.assertNotIn("pdfplumber", sys.modules) + + +class TestPDFExtraction(unittest.TestCase): + def test_flatdecode_two_pages_extract(self): + s1 = b"BT (Page one text.) Tj ET" + s2 = b"BT (Page two text.) Tj ET" + pdf = _make_pdf([s1, s2]) + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(pdf) + p = f.name + try: + doc = extract.extract_pdf(p) + self.assertEqual(doc.kind, "pdf") + statuses = [pg.status for pg in doc.pages] + self.assertTrue(any(s == "extracted" for s in statuses)) + self.assertTrue(any("Page one" in pg.text for pg in doc.pages)) + for pg in doc.pages: + if pg.status == "extracted": + self.assertEqual(pg.text_sha256, + __import__("hashlib").sha256(pg.text.encode()).hexdigest()) + # offset stability: source[0:len] == text + self.assertEqual(pg.text[: len(pg.text)], pg.text) + finally: + os.unlink(p) + + def test_image_only_pdf_fails_honestly(self): + # a binary stream that is NOT FlateDecode-decodable text -> extraction-failed + pdf = _make_pdf([b"\xff\xd8\xff\xe0" + b"\x00" * 40], filters=["None"]) + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(pdf) + p = f.name + try: + doc = extract.extract_pdf(p) + self.assertGreaterEqual(len(doc.pages), 1) + self.assertTrue(any(pg.status == "extraction-failed" for pg in doc.pages)) + # never silently empty on a claimed extracted page + for pg in doc.pages: + if pg.status == "extracted": + self.assertTrue(pg.text.strip()) + finally: + os.unlink(p) + + +class TestPlainText(unittest.TestCase): + def test_txt_extracts_identically(self): + import tempfile, os + body = "Under 15 U.S.C. \u00a7 1681a, the statute applies.\nSecond line." + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False, mode="w") as f: + f.write(body) + p = f.name + try: + doc = extract.extract_text(p) + self.assertEqual(doc.kind, "text") + self.assertEqual(doc.pages[0].text, body) + self.assertEqual(doc.pages[0].status, "extracted") + finally: + os.unlink(p) + + def test_unknown_ext_fails_honestly(self): + doc = extract.extract_text("does/not/matter.bin") + self.assertEqual(doc.kind, "unknown") + self.assertEqual(doc.pages[0].status, "extraction-failed") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_extract_docx.py b/tests/test_extract_docx.py new file mode 100644 index 0000000..bc8299b --- /dev/null +++ b/tests/test_extract_docx.py @@ -0,0 +1,1491 @@ +"""RFC 0005 — DOCX text extraction with STRUCTURAL locators. + +Every fixture is synthesised in-process from XML strings + `zipfile`; nothing +binary is committed and no client document is touched. + +The central invariant under test: a DOCX extraction never produces, stores, or +implies a page number. DOCX pagination is a property of the rendering engine, +not of the file, so a page number here would be a fabricated citation. +""" +import hashlib +import io +import os +import sqlite3 +import struct +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent / "fixtures")) + +import zip_hostile_builders as hostile # noqa: E402 +from matterkit import extract # noqa: E402 + +W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +STRICT_NS = "http://purl.oclc.org/ooxml/wordprocessingml/main" + +CONTENT_TYPES = ( + '' + '' + '' + '' + "" +) +RELS = ( + '' + '' + '' + "" +) + + +def document_xml(body_inner: str, ns: str = W_NS) -> str: + return ( + '' + f'{body_inner}' + ) + + +def para(text: str) -> str: + return f"{text}" + + +def build_docx_bytes(parts: dict) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in parts.items(): + zf.writestr(name, data) + return buf.getvalue() + + +def standard_parts(body_inner: str, ns: str = W_NS) -> dict: + return { + "[Content_Types].xml": CONTENT_TYPES, + "_rels/.rels": RELS, + "word/document.xml": document_xml(body_inner, ns=ns), + } + + +class DocxFixture: + """Context manager writing raw bytes to a temp file with a chosen suffix.""" + + def __init__(self, data: bytes, suffix: str = ".docx"): + self.data = data + self.suffix = suffix + + def __enter__(self) -> str: + fd, self.path = tempfile.mkstemp(suffix=self.suffix) + with os.fdopen(fd, "wb") as f: + f.write(self.data) + return self.path + + def __exit__(self, *exc): + os.unlink(self.path) + return False + + +def docx_file(body_inner: str, ns: str = W_NS) -> DocxFixture: + return DocxFixture(build_docx_bytes(standard_parts(body_inner, ns=ns))) + + +# -------------------------------------------------------------------------- +# Locator contract +# -------------------------------------------------------------------------- +class TestLocatorContract(unittest.TestCase): + def test_body_paragraph_locators_carry_structural_path(self): + with docx_file(para("First paragraph.") + para("Second paragraph.")) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extracted") + self.assertEqual(doc.locator_scheme, extract.LOCATOR_SCHEME_DOCX) + self.assertEqual(len(doc.blocks), 2) + first, second = doc.blocks + self.assertEqual(first.text, "First paragraph.") + self.assertEqual(second.text, "Second paragraph.") + self.assertEqual(first.locator.part, "word/document.xml") + self.assertEqual(first.locator.container_type, "paragraph") + self.assertEqual(first.locator.path, (0,)) + self.assertEqual(second.locator.path, (1,)) + self.assertEqual(first.index, 0) + self.assertEqual(second.index, 1) + + def test_canonical_locator_round_trips_through_parse(self): + with docx_file(para("Alpha") + para("Beta")) as p: + doc = extract.extract_docx(p) + for block in doc.blocks: + canonical = block.locator.canonical() + self.assertEqual(extract.DocxLocator.parse(canonical), block.locator) + self.assertEqual(extract.DocxLocator.parse(canonical).canonical(), canonical) + + def test_text_sha256_is_the_sha_of_the_block_text(self): + with docx_file(para("Alpha") + para("Beta")) as p: + doc = extract.extract_docx(p) + for block in doc.blocks: + self.assertEqual( + block.text_sha256, + hashlib.sha256(block.text.encode("utf-8")).hexdigest(), + ) + + def test_run_count_reflects_the_runs_traversed_in_the_paragraph(self): + body = ( + "" + "Split " + "across " + "three runs." + "" + ) + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.blocks[0].text, "Split across three runs.") + self.assertEqual(doc.blocks[0].run_count, 3) + # run indexes are diagnostic only -- the citable position is the char span + self.assertEqual(doc.blocks[0].char_end - doc.blocks[0].char_start, + len("Split across three runs.")) + + def test_identical_paragraph_text_still_yields_distinct_locators(self): + with docx_file(para("Same text.") * 1 + para("Same text.")) as p: + doc = extract.extract_docx(p) + self.assertEqual(len(doc.blocks), 2) + self.assertEqual(doc.blocks[0].text, doc.blocks[1].text) + self.assertEqual(doc.blocks[0].text_sha256, doc.blocks[1].text_sha256) + self.assertNotEqual( + doc.blocks[0].locator.canonical(), doc.blocks[1].locator.canonical() + ) + + +# -------------------------------------------------------------------------- +# Tables +# -------------------------------------------------------------------------- +TABLE_2X2 = ( + "" + "" + f"{para('R0C0')}" + f"{para('R0C1')}" + "" + "" + f"{para('R1C0')}" + f"{para('R1C1')}" + "" +) + + +class TestTableLocators(unittest.TestCase): + def test_two_by_two_table_addresses_four_cells_distinctly(self): + with docx_file(TABLE_2X2) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extracted") + self.assertEqual(len(doc.blocks), 4) + by_text = {b.text: b.locator for b in doc.blocks} + self.assertEqual(sorted(by_text), ["R0C0", "R0C1", "R1C0", "R1C1"]) + self.assertEqual(by_text["R0C0"].path, (0, 0, 0, 0)) + self.assertEqual(by_text["R0C1"].path, (0, 0, 1, 0)) + self.assertEqual(by_text["R1C0"].path, (0, 1, 0, 0)) + self.assertEqual(by_text["R1C1"].path, (0, 1, 1, 0)) + for loc in by_text.values(): + self.assertEqual(loc.container_type, "table-cell") + self.assertEqual(len({loc.canonical() for loc in by_text.values()}), 4) + + def test_multi_paragraph_cell_addresses_each_paragraph(self): + tbl = ( + "" + + para("Cell line one") + + para("Cell line two") + + "" + ) + with docx_file(tbl) as p: + doc = extract.extract_docx(p) + self.assertEqual([b.text for b in doc.blocks], ["Cell line one", "Cell line two"]) + self.assertEqual(doc.blocks[0].locator.path, (0, 0, 0, 0)) + self.assertEqual(doc.blocks[1].locator.path, (0, 0, 0, 1)) + + def test_nested_table_paths_remain_unique_and_typed(self): + inner = "" + para("Nested") + "" + outer = "" + para("Outer") + inner + "" + with docx_file(outer) as p: + doc = extract.extract_docx(p) + paths = [b.locator.path for b in doc.blocks] + self.assertEqual(paths, [(0, 0, 0, 0), (0, 0, 0, 1, 0, 0, 0)]) + self.assertEqual(len(set(paths)), len(paths)) + self.assertTrue(all(len(pth) % 3 == 1 for pth in paths)) + + def test_every_locator_in_a_mixed_document_is_unique(self): + body = para("Intro") + TABLE_2X2 + para("Outro") + TABLE_2X2 + with docx_file(body) as p: + doc = extract.extract_docx(p) + canonicals = [b.locator.canonical() for b in doc.blocks] + self.assertEqual(len(canonicals), 10) + self.assertEqual(len(set(canonicals)), len(canonicals)) + + +# -------------------------------------------------------------------------- +# Offsets +# -------------------------------------------------------------------------- +class TestOffsetRoundTrip(unittest.TestCase): + def test_char_offsets_slice_the_exact_block_text(self): + body = para("Intro") + TABLE_2X2 + para("Outro") + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertTrue(doc.text) + for block in doc.blocks: + self.assertEqual(doc.text[block.char_start:block.char_end], block.text) + self.assertEqual( + doc.text_sha256, hashlib.sha256(doc.text.encode("utf-8")).hexdigest() + ) + + def test_offsets_are_monotonic_and_non_overlapping(self): + body = para("One") + para("Two") + para("Three") + with docx_file(body) as p: + doc = extract.extract_docx(p) + prev_end = -1 + for block in doc.blocks: + self.assertGreater(block.char_start, prev_end) + self.assertEqual(block.char_end, block.char_start + len(block.text)) + prev_end = block.char_end + + def test_tabs_and_breaks_render_into_the_block_text(self): + body = ( + "ABC" + ) + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.blocks[0].text, "A\tB\nC") + self.assertEqual(doc.text[doc.blocks[0].char_start:doc.blocks[0].char_end], + "A\tB\nC") + + def test_tracked_deletion_text_is_not_extracted(self): + body = ( + "" + "Kept. " + "Struck out." + "Inserted." + "" + ) + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.blocks[0].text, "Kept. Inserted.") + + def test_field_instruction_codes_are_not_extracted(self): + body = ( + "" + " HYPERLINK \"http://example.test\" " + "Visible label" + "" + ) + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.blocks[0].text, "Visible label") + + +# -------------------------------------------------------------------------- +# Honest failure vocabulary +# -------------------------------------------------------------------------- +class TestHonestFailure(unittest.TestCase): + def assert_failed(self, doc, needle): + self.assertEqual(doc.status, "extraction-failed") + self.assertEqual(doc.blocks, []) + self.assertEqual(doc.pages, []) + self.assertIsNotNone(doc.reason) + self.assertIn(needle, doc.reason.lower()) + + def test_truncated_zip_fails(self): + data = build_docx_bytes(standard_parts(para("Hello"))) + with DocxFixture(data[: len(data) // 2]) as p: + self.assert_failed(extract.extract_docx(p), "zip") + + def test_renamed_text_file_is_refused_not_redispatched(self): + with DocxFixture(b"This is plain text, not a docx.\n") as p: + doc = extract.extract_docx(p) + self.assert_failed(doc, "zip") + self.assertEqual(extract.extract_text(p).status, "extraction-failed") + + def test_renamed_pdf_is_refused(self): + with DocxFixture(b"%PDF-1.4\n1 0 obj\n<< >>\nendobj\n%%EOF\n") as p: + self.assert_failed(extract.extract_docx(p), "zip") + + def test_zip_without_document_xml_fails(self): + data = build_docx_bytes({"[Content_Types].xml": CONTENT_TYPES, "_rels/.rels": RELS}) + with DocxFixture(data) as p: + self.assert_failed(extract.extract_docx(p), "word/document.xml") + + def test_blank_document_xml_fails(self): + parts = standard_parts(para("x")) + parts["word/document.xml"] = " \n " + with DocxFixture(build_docx_bytes(parts)) as p: + self.assert_failed(extract.extract_docx(p), "empty") + + def test_malformed_xml_fails_without_partial_salvage(self): + parts = standard_parts(para("x")) + parts["word/document.xml"] = ( + '' + f'' + "Salvageable looking text" + ) # never closed + with DocxFixture(build_docx_bytes(parts)) as p: + doc = extract.extract_docx(p) + self.assert_failed(doc, "xml") + self.assertNotIn("Salvageable", doc.text) + + def test_wrong_root_element_fails(self): + parts = standard_parts(para("x")) + parts["word/document.xml"] = f'' + with DocxFixture(build_docx_bytes(parts)) as p: + self.assert_failed(extract.extract_docx(p), "root") + + def test_missing_body_fails(self): + parts = standard_parts(para("x")) + parts["word/document.xml"] = f'' + with DocxFixture(build_docx_bytes(parts)) as p: + self.assert_failed(extract.extract_docx(p), "body") + + def test_ooxml_strict_namespace_fails_rather_than_returning_empty(self): + with docx_file(para("Strict namespace text"), ns=STRICT_NS) as p: + self.assert_failed(extract.extract_docx(p), "namespace") + + def test_encrypted_part_fails(self): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("word/document.xml", document_xml(para("secret"))) + raw = bytearray(buf.getvalue()) + # set the "encrypted" general-purpose bit on the local header + central dir + for magic in (b"PK\x03\x04", b"PK\x01\x02"): + idx = raw.find(magic) + while idx != -1: + flag_at = idx + (6 if magic == b"PK\x03\x04" else 8) + raw[flag_at] |= 0x01 + idx = raw.find(magic, idx + 1) + with DocxFixture(bytes(raw)) as p: + self.assert_failed(extract.extract_docx(p), "encrypt") + + def test_oversized_part_is_refused(self): + body = "".join(para("padding " * 64) for _ in range(200)) + with docx_file(body) as p: + doc = extract.extract_docx(p, max_part_bytes=1024) + self.assert_failed(doc, "too large") + + def test_empty_but_wellformed_document_succeeds_with_zero_blocks(self): + with docx_file("") as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extracted") + self.assertEqual(doc.blocks, []) + self.assertEqual(doc.text, "") + self.assertIsNone(doc.reason) + + def test_empty_paragraph_is_an_addressable_block_not_a_dropped_one(self): + with docx_file(para("Before") + "" + para("After")) as p: + doc = extract.extract_docx(p) + self.assertEqual([b.text for b in doc.blocks], ["Before", "", "After"]) + self.assertEqual(doc.blocks[1].locator.path, (1,)) + self.assertEqual(doc.blocks[1].status, "extracted") + + +# -------------------------------------------------------------------------- +# No fabricated pagination, anywhere +# -------------------------------------------------------------------------- +class TestNoFabricatedPagination(unittest.TestCase): + def test_docx_extraction_yields_no_pages(self): + body = para("Intro") + TABLE_2X2 + "" + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.pages, []) + self.assertEqual(doc.locator_scheme, extract.LOCATOR_SCHEME_DOCX) + for block in doc.blocks: + self.assertFalse(hasattr(block.locator, "page_number")) + self.assertNotIn("page", block.locator.canonical().lower()) + + def test_explicit_page_break_does_not_create_a_page_index(self): + body = ( + para("Before break") + + "After break" + ) + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual(len(doc.blocks), 2) + self.assertEqual(doc.blocks[1].locator.path, (1,)) + self.assertEqual(doc.blocks[1].text, "\nAfter break") + + +# -------------------------------------------------------------------------- +# Part-set policy +# -------------------------------------------------------------------------- +class TestPartInventory(unittest.TestCase): + def test_inventory_separates_extracted_skipped_and_seen(self): + parts = standard_parts(para("Body text")) + parts["word/header1.xml"] = document_xml(para("Header text")) + parts["word/footer1.xml"] = document_xml(para("Footer text")) + parts["word/footnotes.xml"] = document_xml(para("Footnote text")) + parts["word/theme/theme1.xml"] = "" + with DocxFixture(build_docx_bytes(parts)) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extracted") + self.assertEqual(doc.parts.extracted, ("word/document.xml",)) + self.assertEqual( + sorted(doc.parts.skipped), + ["word/footer1.xml", "word/footnotes.xml", "word/header1.xml"], + ) + self.assertIn("word/theme/theme1.xml", doc.parts.seen) + self.assertEqual([b.text for b in doc.blocks], ["Body text"]) + + def test_unhandled_block_level_elements_are_named_not_silently_dropped(self): + body = para("Visible") + "" + para("Inside control") + "" + with docx_file(body) as p: + doc = extract.extract_docx(p) + self.assertEqual([b.text for b in doc.blocks], ["Visible"]) + self.assertIn("sdt", doc.parts.unhandled) + + def test_clean_document_reports_no_unhandled_elements(self): + with docx_file(para("Just a paragraph.") + TABLE_2X2) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.parts.unhandled, ()) + + +# -------------------------------------------------------------------------- +# Immutability + dispatch + dependency floor +# -------------------------------------------------------------------------- +class TestSourceImmutability(unittest.TestCase): + def test_source_bytes_unchanged_by_extraction(self): + data = build_docx_bytes(standard_parts(para("Hello") + TABLE_2X2)) + before = hashlib.sha256(data).hexdigest() + with DocxFixture(data) as p: + extract.extract_docx(p) + extract.extract_text(p) + after = hashlib.sha256(Path(p).read_bytes()).hexdigest() + mtime_stable = os.path.getsize(p) + self.assertEqual(before, after) + self.assertEqual(mtime_stable, len(data)) + + +class TestDispatch(unittest.TestCase): + def test_extract_text_routes_docx_to_the_structural_extractor(self): + with docx_file(para("Dispatched.")) as p: + doc = extract.extract_text(p) + self.assertEqual(doc.kind, "docx") + self.assertEqual(doc.locator_scheme, extract.LOCATOR_SCHEME_DOCX) + self.assertEqual([b.text for b in doc.blocks], ["Dispatched."]) + + def test_uppercase_extension_still_routes(self): + data = build_docx_bytes(standard_parts(para("Shouty."))) + with DocxFixture(data, suffix=".DOCX") as p: + doc = extract.extract_text(p) + self.assertEqual(doc.kind, "docx") + + def test_legacy_doc_is_not_treated_as_docx(self): + data = build_docx_bytes(standard_parts(para("Not legacy."))) + with DocxFixture(data, suffix=".doc") as p: + doc = extract.extract_text(p) + self.assertEqual(doc.status, "extraction-failed") + self.assertNotEqual(doc.kind, "docx") + + def test_unknown_extension_uses_the_whole_file_scheme(self): + doc = extract.extract_text("nonexistent/whatever.bin") + self.assertEqual(doc.locator_scheme, extract.LOCATOR_SCHEME_WHOLE_FILE) + self.assertEqual(doc.status, "extraction-failed") + + def test_pdf_path_keeps_the_page_locator_scheme(self): + import zlib + + stream = zlib.compress(b"BT (Page one text.) Tj ET") + pdf = ( + b"%PDF-1.4\n1 0 obj\n<< /Length " + + str(len(stream)).encode() + + b" /Filter /FlateDecode >>\nstream\n" + + stream + + b"\nendstream\nendobj\n%%EOF\n" + ) + with DocxFixture(pdf, suffix=".pdf") as p: + doc = extract.extract_text(p) + self.assertEqual(doc.kind, "pdf") + self.assertEqual(doc.locator_scheme, extract.LOCATOR_SCHEME_PDF_PAGE) + self.assertEqual(doc.blocks, []) + + +class TestStdlibOnlyFloor(unittest.TestCase): + def test_no_third_party_module_is_imported(self): + self.assertNotIn("docx", sys.modules) + self.assertNotIn("lxml", sys.modules) + self.assertNotIn("pypdf", sys.modules) + self.assertNotIn("pdfplumber", sys.modules) + + def test_module_imports_with_third_party_paths_removed(self): + import subprocess + + repo = str(Path(__file__).resolve().parent.parent) + code = ( + "import sys, sysconfig\n" + f"sys.path = [p for p in sys.path if 'site-packages' not in p]\n" + f"sys.path.insert(0, {repo!r})\n" + "import matterkit.extract as e\n" + "assert hasattr(e, 'extract_docx')\n" + "print('ok')\n" + ) + out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertIn("ok", out.stdout) + + +# -------------------------------------------------------------------------- +# Storage +# -------------------------------------------------------------------------- +class TestBlockStorage(unittest.TestCase): + def setUp(self): + from matterkit import store + + self.tmp = tempfile.mkdtemp() + self.conn = store.connect(self.tmp) + + def tearDown(self): + self.conn.close() + + def test_document_blocks_table_has_no_page_number_column(self): + cols = [r[1] for r in self.conn.execute("PRAGMA table_info(document_blocks)")] + self.assertTrue(cols, "document_blocks table missing") + self.assertNotIn("page_number", cols) + self.assertIn("locator", cols) + self.assertIn("locator_scheme", cols) + self.assertIn("char_start", cols) + + def test_record_blocks_persists_sliceable_locators(self): + body = para("Intro") + TABLE_2X2 + para("Outro") + with docx_file(body) as p: + doc = extract.extract_docx(p) + sha = hashlib.sha256(Path(p).read_bytes()).hexdigest() + n = extract.record_blocks(self.conn, sha, doc) + self.assertEqual(n, len(doc.blocks)) + rows = self.conn.execute( + "SELECT * FROM document_blocks WHERE document_sha256=? ORDER BY block_index", + (sha,), + ).fetchall() + self.assertEqual(len(rows), 6) + for row, block in zip(rows, doc.blocks): + self.assertEqual(row["locator"], block.locator.canonical()) + self.assertEqual(row["locator_scheme"], extract.LOCATOR_SCHEME_DOCX) + self.assertEqual(row["text"], block.text) + self.assertEqual(row["text_sha256"], block.text_sha256) + self.assertEqual(doc.text[row["char_start"]:row["char_end"]], row["text"]) + + def test_record_blocks_is_idempotent_on_reextraction(self): + with docx_file(para("A") + para("B")) as p: + doc = extract.extract_docx(p) + sha = hashlib.sha256(Path(p).read_bytes()).hexdigest() + extract.record_blocks(self.conn, sha, doc) + extract.record_blocks(self.conn, sha, doc) + count = self.conn.execute( + "SELECT COUNT(*) FROM document_blocks WHERE document_sha256=?", (sha,) + ).fetchone()[0] + self.assertEqual(count, 2) + + def test_record_blocks_does_not_commit_caller_transaction(self): + """The caller owns the transaction: record_blocks never commits.""" + with docx_file(para("A") + para("B")) as p: + doc = extract.extract_docx(p) + sha = hashlib.sha256(Path(p).read_bytes()).hexdigest() + self.conn.execute("BEGIN") + n = extract.record_blocks(self.conn, sha, doc) + self.assertEqual(n, len(doc.blocks)) + self.conn.rollback() + count = self.conn.execute( + "SELECT COUNT(*) FROM document_blocks" + ).fetchone()[0] + self.assertEqual(count, 0) + + def test_failed_extraction_records_one_failure_row_and_no_content(self): + with DocxFixture(b"not a zip at all") as p: + doc = extract.extract_docx(p) + sha = hashlib.sha256(Path(p).read_bytes()).hexdigest() + extract.record_blocks(self.conn, sha, doc) + rows = self.conn.execute( + "SELECT * FROM document_blocks WHERE document_sha256=?", (sha,) + ).fetchall() + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["status"], "extraction-failed") + self.assertIsNone(rows[0]["text"]) + self.assertIsNotNone(rows[0]["reason"]) + + def test_record_blocks_refuses_a_page_scheme_extraction(self): + """The block store cannot be used to launder a page locator.""" + doc = extract.DocumentText( + path="x.pdf", kind="pdf", + locator_scheme=extract.LOCATOR_SCHEME_PDF_PAGE) + with self.assertRaises(ValueError): + extract.record_blocks(self.conn, "deadbeef", doc) + + def test_status_column_rejects_invented_statuses(self): + with self.assertRaises(sqlite3.IntegrityError): + self.conn.execute( + "INSERT INTO document_blocks (document_sha256, locator_scheme, part, " + "locator, block_index, container_type, path, status) " + "VALUES ('x','docx-structural','word/document.xml','w#b0',0," + "'paragraph','[0]','probably-fine')" + ) + + def test_locator_scheme_column_rejects_a_page_scheme_for_blocks(self): + with self.assertRaises(sqlite3.IntegrityError): + self.conn.execute( + "INSERT INTO document_blocks (document_sha256, locator_scheme, part, " + "locator, block_index, container_type, path, status) " + "VALUES ('x','pdf-page','word/document.xml','w#b0',0," + "'paragraph','[0]','extracted')" + ) + + +# -------------------------------------------------------------------------- +# Hostile archives and hostile XML +# -------------------------------------------------------------------------- +class TestHostileInput(unittest.TestCase): + def assert_failed(self, doc, needle): + self.assertEqual(doc.status, "extraction-failed") + self.assertEqual(doc.blocks, []) + self.assertEqual(doc.pages, []) + self.assertIn(needle, doc.reason.lower()) + + def test_doctype_declaration_is_refused(self): + """An entity-expansion bomb never reaches the parser: OOXML has no DTD.""" + parts = standard_parts(para("x")) + parts["word/document.xml"] = ( + '' + '' + '' + '' + ']>' + f'' + "&c;" + "" + ) + with DocxFixture(build_docx_bytes(parts)) as p: + self.assert_failed(extract.extract_docx(p), "doctype") + + def test_doctype_behind_a_utf8_bom_is_still_refused(self): + """The prolog scan must not be defeated by a byte-order mark.""" + parts = standard_parts(para("x")) + xml = ( + '' + ']>' + f'' + "&a;" + "" + ) + parts["word/document.xml"] = b"\xef\xbb\xbf" + xml.encode("utf-8") + with DocxFixture(build_docx_bytes(parts)) as p: + self.assert_failed(extract.extract_docx(p), "doctype") + + def test_plain_utf8_bom_document_still_extracts(self): + parts = standard_parts(para("Byte order marked.")) + parts["word/document.xml"] = ( + b"\xef\xbb\xbf" + parts["word/document.xml"].encode("utf-8")) + with DocxFixture(build_docx_bytes(parts)) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extracted") + self.assertEqual([b.text for b in doc.blocks], ["Byte order marked."]) + + def test_utf16_document_part_is_refused_rather_than_scanned_blind(self): + """A UTF-16 part cannot be byte-scanned for a DOCTYPE, so it is refused.""" + parts = standard_parts(para("x")) + parts["word/document.xml"] = document_xml(para("x")).replace( + 'encoding="UTF-8"', 'encoding="UTF-16"').encode("utf-16") + with DocxFixture(build_docx_bytes(parts)) as p: + self.assert_failed(extract.extract_docx(p), "encoding") + + def test_duplicate_document_part_is_refused_as_ambiguous(self): + import warnings + + buf = io.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("word/document.xml", document_xml(para("Decoy"))) + zf.writestr("word/document.xml", document_xml(para("Payload"))) + with DocxFixture(buf.getvalue()) as p: + self.assert_failed(extract.extract_docx(p), "duplicate") + + def test_crc_mismatch_in_the_document_part_fails(self): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf: + for name, data in standard_parts(para("AAAAAAAA")).items(): + zf.writestr(name, data) + raw = buf.getvalue() + self.assertIn(b"AAAAAAAA", raw) + tampered = raw.replace(b"AAAAAAAA", b"BBBBBBBB") # same length, wrong CRC + with DocxFixture(tampered) as p: + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extraction-failed") + self.assertEqual(doc.blocks, []) + self.assertNotIn("BBBBBBBB", doc.text) + + def test_zip_signature_check_precedes_any_parsing(self): + """A polyglot that only *ends* in a zip is still refused on its head.""" + inner = build_docx_bytes(standard_parts(para("Hidden"))) + with DocxFixture(b"%PDF-1.4\n" + inner) as p: + self.assert_failed(extract.extract_docx(p), "zip") + + +# -------------------------------------------------------------------------- +# Resource bounds (RFC 0005 §4a): the pre-open GUARD vs the post-open +# ACCEPTANCE LIMIT. The spy below is the whole point of these tests: +# `size_limit:members` and `size_limit:central_directory` must refuse with +# ZERO ZipFile constructions (the guard ran before `ZipFile.__init__` +# materialised a ZipInfo per entry), while `size_limit:aggregate` must refuse +# with AT LEAST ONE (the acceptance limit runs post-open, after the directory +# was already parsed). `constructions` counts attempts, `completions` counts +# constructors that returned; a mid-__init__ BadZipFile is 1/0. If the labels +# were swapped, the construction counts would swap with them. +# -------------------------------------------------------------------------- +class ZipFileSpy: + """Replaces `zipfile.ZipFile` AS SEEN BY `matterkit.extract` and counts + constructions, delegating to the real class so post-open paths still run. + + Captures the real class at construction time -- build the spy BEFORE the + patch is active, and build any fixture bytes with `zipfile` before too. + """ + + def __init__(self): + self._real = zipfile.ZipFile + self.constructions = 0 + self.completions = 0 + + def __call__(self, *args, **kwargs): + # Two tallies, and the difference is load-bearing: `constructions` + # counts ATTEMPTS (incremented before `__init__` runs), `completions` + # counts constructors that RETURNED. `ZipFile.__init__` reads the + # central directory, so a lie it catches raises mid-construction -- + # attempts 1, completions 0. A guard that fires pre-open leaves both 0. + self.constructions += 1 + zf = self._real(*args, **kwargs) + self.completions += 1 + return zf + + +def hand_eocd(entries: int, cd_size: int, cd_offset: int) -> bytes: + """Hand-built End Of Central Directory record, 22 bytes (APPNOTE 4.3.16). + Entries total at +10 (2B), cd_size at +12 (4B), cd_offset at +16 (4B).""" + return (b"PK\x05\x06" + + struct.pack(" bytes: + """Hand-built ZIP64 EOCD locator, 20 bytes; record offset at +8 (8B).""" + return b"PK\x06\x07" + struct.pack(" bytes: + """Hand-built ZIP64 EOCD record, 56-byte fixed portion (APPNOTE 4.3.14). + Record size at +4 (8B), total entries at +32 (8B), cd_size at +40 (8B), + cd_offset at +48 (8B).""" + return (b"PK\x06\x06" + + struct.pack(" bytes: + """A file whose head is a zip signature and whose tail is hand-built. + + The pre-open reader sees only head magic + tail records, so synthetic + bytes test the guard faithfully -- no need for 65,536+ real members + (`zipfile` will not emit ZIP64 for few entries, and a real ZIP64 fixture + would be ~6 MB and slow). Every fixture here is refused pre-open or + parsed by the reader directly, so the filler is never interpreted. + """ + return b"PK\x03\x04" + b"\x00" * filler_len + tail + + +def many_member_zip(n: int) -> bytes: + """A real (non-ZIP64) zip with `n` tiny members. ~0.05 s for 10k.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf: + for i in range(n): + zf.writestr(f"m{i:05d}", b"x") + return buf.getvalue() + + +def zip64_declared_sizes(raw: bytes, declared: int) -> bytes: + """Rewrite every central-directory entry to declare `declared` uncompressed + bytes the ZIP64 way: the classic 4-byte field saturated to 0xFFFFFFFF and + the true 64-bit size carried in an extra field (header 0x0001, APPNOTE + 4.5.3) -- exactly how a member over 4 GiB declares itself. + + Only the directory is rewritten; no member is remotely that large, which is + the point: the acceptance limit sums DECLARED sizes. A reader that took the + saturated 32-bit field at face value would compute a different (smaller) + total, so the asserted total is what proves the ZIP64 field was honoured. + """ + eocd = raw.rfind(b"PK\x05\x06") + count, = struct.unpack(" sentinel + extra = struct.pack("= 46 bytes, so the count is provably false. + tail = hand_eocd(100, 1_000, 0) + spy = ZipFileSpy() + with DocxFixture(synthetic_zip(tail, 3_000)) as p: + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assert_failed(doc, "corrupt or truncated") + self.assert_failed(doc, "structural lie") + self.assertEqual(spy.constructions, 0) + + def test_cd_offset_plus_size_past_eof_is_corrupt_spy_untouched(self): + tail = hand_eocd(1, 100_000, 0) # 0 + 100,000 > file size + spy = ZipFileSpy() + with DocxFixture(synthetic_zip(tail, 100)) as p: + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assert_failed(doc, "corrupt or truncated") + self.assertEqual(spy.constructions, 0) + + +class TestCentralDirectoryByteCap(unittest.TestCase): + """The BYTE CAP: `size_limit:central_directory`, pre-open, no ZipFile. + + The member guard bounds the DECLARED entry count, which a hostile archive + can simply understate (measured: declared 5 over a real 3,001-entry, + 6.32 MB directory -> 14.35 MB peak before the post-open reconciliation + notices). The declared directory SIZE is the field CPython actually reads + against -- `_RealGetContents` does `data = fp.read(size_cd)` -- so capping + it bounds the parser's input where capping the count cannot. + + Variant 4 is the RED-against-pre-cap demonstration: honest oversized + `cd_size`, member count well under MAX_ARCHIVE_MEMBERS. Removing the cap + would make these tests fail (ZipFile would be constructed). Variant 2 is + not this evidence -- see TestUnderstatedCdSizeTruncation. + """ + + def assert_failed(self, doc, needle): + self.assertEqual(doc.status, "extraction-failed") + self.assertEqual(doc.blocks, []) + self.assertEqual(doc.pages, []) + self.assertIsNotNone(doc.reason) + self.assertIn(needle, doc.reason.lower()) + + def test_honest_oversized_directory_refused_pre_open_spy_untouched(self): + # Fixture variant 4: every tail record honest, member count well under + # the 10,000-member guard, central directory over MAX_CD_BYTES. + # Nothing here is a lie -- the member guard has nothing to catch, and + # only a byte cap refuses it. ZIP comment max is 65,535, so ~520 + # members with max comments is the smallest real zip over a 32 MiB cap. + cap = extract.MAX_CD_BYTES + data, real, cd_size = hostile.oversized_cd_honest_case( + n_members=520, comment_bytes=65_535) + self.assertLess(real, extract.MAX_ARCHIVE_MEMBERS) # guard cannot fire + self.assertGreater(cd_size, cap) + spy = ZipFileSpy() + with DocxFixture(data) as p: + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assertIsNotNone(doc.reason) + self.assertTrue(doc.reason.startswith("size_limit:central_directory"), + doc.reason) + self.assert_failed(doc, "size_limit:central_directory") + # THE POINT: refused before `zipfile.ZipFile` was even *attempted*. + # constructions is attempts; completions is successful returns. + self.assertEqual(spy.constructions, 0) + self.assertEqual(spy.completions, 0) + + def test_one_byte_over_the_cap_is_refused_spy_untouched(self): + cap = extract.MAX_CD_BYTES + # Declared directory of cap+1 bytes that genuinely fits in the file, + # so the geometry checks pass and the cap is the only thing left. + tail = hand_eocd(1, cap + 1, 0) + spy = ZipFileSpy() + with DocxFixture(synthetic_zip(tail, cap + 8)) as p: + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assertIsNotNone(doc.reason) + self.assertTrue(doc.reason.startswith("size_limit:central_directory"), + doc.reason) + self.assertEqual(spy.constructions, 0) + self.assertEqual(spy.completions, 0) + + def test_directory_exactly_at_the_cap_passes_the_cap(self): + # Boundary: the cap is `>`, not `>=`. At exactly the limit the archive + # proceeds to `ZipFile`, which then rejects the filler as a directory. + # This is the off-by-one guard on the comparison itself. + cap = extract.MAX_CD_BYTES + tail = hand_eocd(1, cap, 0) + spy = ZipFileSpy() + with DocxFixture(synthetic_zip(tail, cap + 8)) as p: + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assertIsNotNone(doc.reason) + self.assertFalse(doc.reason.startswith("size_limit:"), doc.reason) + self.assertEqual(spy.constructions, 1) + self.assertEqual(spy.completions, 0) + + def test_a_real_word_sized_directory_is_nowhere_near_the_cap(self): + # The cap must be invisible to real documents. A standard three-part + # package has a directory of a few hundred bytes. + data = build_docx_bytes(standard_parts(para("Hello"))) + eocd = data.rfind(b"PK\x05\x06") + cd_size, = struct.unpack("64). + The extractor path under test is the `except BadZipFile` handler. + """ + + def test_direct_zipfile_raises_bad_zipfile(self): + data, _real, _cd = hostile.understated_cd_size_case( + n_members=80, comment_bytes=200, declared_count=1) + with self.assertRaises(zipfile.BadZipFile) as ctx: + zipfile.ZipFile(io.BytesIO(data), "r") + self.assertIn("Bad magic number for central directory", str(ctx.exception)) + + def test_extractor_structured_failure_is_not_the_cap_reason(self): + data, _real, _cd = hostile.understated_cd_size_case( + n_members=80, comment_bytes=200, declared_count=1) + spy = ZipFileSpy() + with DocxFixture(data) as p: + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assertEqual(doc.status, "extraction-failed") + self.assertEqual(doc.blocks, []) + self.assertEqual(doc.pages, []) + self.assertIsNotNone(doc.reason) + self.assertTrue(doc.reason.startswith("corrupt or truncated zip archive:"), + doc.reason) + self.assertIn("Bad magic number for central directory", doc.reason) + self.assertFalse(doc.reason.startswith("size_limit:central_directory"), + doc.reason) + self.assertFalse(doc.reason.startswith("size_limit:"), doc.reason) + # Attempted, did not return: CPython raised mid-__init__. + self.assertGreaterEqual(spy.constructions, 1) + self.assertEqual(spy.completions, 0) + + +class TestMagicInEocdCommentRefusal(unittest.TestCase): + """P3-1 / fixture variant 5: `PK\\x05\\x06` in the EOCD comment. + + Our reader takes the rightmost signature and requires the comment-length + field to put that record at EOF. The in-comment occurrence fails that + check, so we refuse. CPython `_EndRecData` also rfinds (rightmost) and + assumes the magic does not appear in the comment -- it may select the + in-comment occurrence rather than fail. No repair scan is implemented: + tests assert *our refusal only*. Encoding "the right record" here would + encode a repair that must not exist. + """ + + def test_magic_in_comment_is_our_refusal_spy_untouched(self): + data = hostile.comment_signature_case() + self.assertIsInstance(data, bytes) + spy = ZipFileSpy() + with DocxFixture(data) as p: + our = extract._read_zip_tail_records(Path(p), os.path.getsize(p)) + with mock.patch.object(extract.zipfile, "ZipFile", spy): + doc = extract.extract_docx(p) + self.assertIsInstance(our, str, "must refuse, not pick a repaired record") + self.assertIn("end-of-central-directory", our) + self.assertEqual(doc.status, "extraction-failed") + self.assertEqual(doc.blocks, []) + self.assertTrue(doc.reason.startswith("corrupt or truncated"), doc.reason) + self.assertFalse(doc.reason.startswith("size_limit:"), doc.reason) + self.assertEqual(spy.constructions, 0) + self.assertEqual(spy.completions, 0) + + +def _cpython_endrec_fields(data: bytes): + """CPython zipfile's tail-record view: (entries, cd_size, cd_offset). + + Returns None if `_EndRecData` finds no record, or the BadZipFile it + raises. Private on purpose -- this is the parser-agreement test, pinned + to CPython's implementation, not a public API we wrap. + """ + fp = io.BytesIO(data) + try: + rec = zipfile._EndRecData(fp) + except zipfile.BadZipFile as exc: + return exc + if not rec: + return None + return rec[zipfile._ECD_ENTRIES_TOTAL], rec[zipfile._ECD_SIZE], rec[zipfile._ECD_OFFSET] + + +class TestParserAgreement(unittest.TestCase): + """Our pre-open reader and CPython zipfile must select the same records, + or refuse together. Pinned as tests against CPython 3.11's `_EndRecData` + (rfind, magic-not-in-comment assumption), not a cross-version fact. + + Variant 5 (magic-in-comment) is deliberately absent from the agreement + matrix -- see TestMagicInEocdCommentRefusal. + """ + + def read_ours(self, data: bytes): + with DocxFixture(data) as p: + return extract._read_zip_tail_records(Path(p), os.path.getsize(p)) + + def zipfile_open_fails(self, data: bytes) -> bool: + try: + with zipfile.ZipFile(io.BytesIO(data), "r"): + return False + except zipfile.BadZipFile: + return True + + def test_honest_eocd_agrees_on_count_size_offset(self): + data = build_docx_bytes(standard_parts(para("Hello"))) + ours = self.read_ours(data) + theirs = _cpython_endrec_fields(data) + self.assertIsInstance(ours, tuple, ours) + self.assertEqual(ours, theirs) + with zipfile.ZipFile(io.BytesIO(data), "r") as zf: + self.assertEqual(len(zf.infolist()), ours[0]) + self.assertEqual(zf.start_dir, ours[2]) + + def test_zip64_all_sentinels_agree_on_count_size_offset(self): + # Common ZIP64 shape: every classic field saturated, values live in + # the ZIP64 record. cd_offset + cd_size lands on the ZIP64 EOCD so + # CPython's `_EndRecData64` locator arithmetic accepts the record. + filler = 2_000 + z64_off = 4 + filler + cd_size = filler + cd_offset = 4 + tail = (hand_z64_eocd(3, cd_size, cd_offset) + + hand_z64_locator(z64_off) + + hand_eocd(0xFFFF, 0xFFFFFFFF, 0xFFFFFFFF)) + data = synthetic_zip(tail, filler) + ours = self.read_ours(data) + theirs = _cpython_endrec_fields(data) + self.assertEqual(ours, (3, cd_size, cd_offset)) + self.assertEqual(ours, theirs) + + def test_comment_length_not_landing_on_eof_is_our_refusal(self): + # Comment-length field claims 8 bytes; the file ends at the EOCD. + # We refuse (record does not end at EOF). CPython `_EndRecData` does + # not require that match and still returns the classic count/size/ + # offset; ZipFile then opens. Fail-closed non-agreement, documented, + # not a defect -- same class as P3-1, not an agreed refusal. + data = bytearray(build_docx_bytes(standard_parts(para("Hello")))) + eocd = data.rfind(b"PK\x05\x06") + data[eocd + 20:eocd + 22] = struct.pack(" the 256 MiB aggregate limit. The members on disk are + # tiny: only the declared uncompressed-size fields are forged, which + # is exactly what the acceptance limit sums. + declared = 31 * 1024 * 1024 + parts = { + "[Content_Types].xml": "", + "_rels/.rels": "", + "word/document.xml": document_xml(para("tiny")), + } + for i in range(6): + parts[f"word/media/img{i}.bin"] = "x" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in parts.items(): + zf.writestr(name, data) + raw = bytearray(buf.getvalue()) + for sig, size_off in ((b"PK\x03\x04", 22), (b"PK\x01\x02", 24)): + idx = raw.find(sig) + while idx != -1: + raw[idx + size_off:idx + size_off + 4] = struct.pack("") as p: + doc = extract.extract_docx(p) + sha = hashlib.sha256(Path(p).read_bytes()).hexdigest() + self.assertEqual(doc.status, "extracted") + self.assertEqual(doc.blocks, []) + + n = extract.record_blocks(self.conn, sha, doc) + self.assertEqual(n, 1) + rows = self.conn.execute( + "SELECT * FROM document_blocks WHERE document_sha256=?", (sha,)).fetchall() + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["status"], "extracted") + self.assertEqual(rows[0]["container_type"], "part") + self.assertEqual(rows[0]["text"], "") + self.assertIsNotNone(rows[0]["reason"]) + + +# -------------------------------------------------------------------------- +# CLI: `matter pages` must never write a page row for a DOCX +# -------------------------------------------------------------------------- +KIT = str(Path(__file__).resolve().parent.parent) + + +class TestPagesCLIOnDocx(unittest.TestCase): + def setUp(self): + import shutil + import subprocess + + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + subprocess.run( + [sys.executable, os.path.join(KIT, "matter.py"), "init", self.dir], + capture_output=True, text=True, check=True) + + def _run(self, *args): + import subprocess + + return subprocess.run( + [sys.executable, os.path.join(KIT, "matter.py"), *args], + capture_output=True, text=True, cwd=self.dir) + + def test_docx_writes_blocks_and_leaves_document_pages_untouched(self): + from matterkit import store + + src = os.path.join(self.dir, "exhibit.docx") + Path(src).write_bytes(build_docx_bytes(standard_parts(para("Intro") + TABLE_2X2))) + sha = hashlib.sha256(Path(src).read_bytes()).hexdigest() + + r = self._run("pages", self.dir, src) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("block", r.stdout.lower()) + + conn = store.connect(self.dir) + blocks = conn.execute( + "SELECT * FROM document_blocks WHERE document_sha256=? ORDER BY block_index", + (sha,)).fetchall() + pages = conn.execute( + "SELECT * FROM document_pages WHERE document_sha256=?", (sha,)).fetchall() + conn.close() + self.assertEqual(len(blocks), 5) + self.assertEqual(pages, [], "a DOCX must never produce a document_pages row") + self.assertEqual(blocks[0]["locator"], "word/document.xml#b0") + + def test_corrupt_docx_reports_failure_and_writes_no_content(self): + from matterkit import store + + src = os.path.join(self.dir, "broken.docx") + Path(src).write_bytes(b"definitely not a zip") + sha = hashlib.sha256(Path(src).read_bytes()).hexdigest() + + r = self._run("pages", self.dir, src) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("failed", r.stdout.lower()) + + conn = store.connect(self.dir) + rows = conn.execute( + "SELECT * FROM document_blocks WHERE document_sha256=?", (sha,)).fetchall() + pages = conn.execute( + "SELECT * FROM document_pages WHERE document_sha256=?", (sha,)).fetchall() + conn.close() + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["status"], "extraction-failed") + self.assertIsNone(rows[0]["text"]) + self.assertEqual(pages, []) + + +if __name__ == "__main__": + unittest.main()