feat(extract): DOCX text extraction with structural locators (RFC 0005)

README.md

@@ -55,6 +55,28 @@ python3 matter.py packet ~/cases/my-case --layer 3min --out COUNSEL_PACKET.md
 Tools: `matter.status` · `matter.fact_add` · `matter.fact_list` ·
 `matter.deadline_list` · `matter.packet_render` · `matter.cite_extract`.
 
+## Document extraction and locators (RFC 0004, RFC 0005)
+
+```bash
+python3 matter.py pages ~/cases/my-case                       # every ingested document
+python3 matter.py pages ~/cases/my-case Filings/motion.docx    # one file
+```
+
+Stdlib only — no pypdf, no python-docx. Each format gets the locator its own
+bytes can support, and the two never mix:
+
+| scheme | format | locator | stored in |
+|---|---|---|---|
+| `pdf-page` | PDF | 1-based page number read from the file's structure | `document_pages` |
+| `docx-structural` | DOCX | path through the OOXML block tree, e.g. `word/document.xml#b3/r1/c0/b0` | `document_blocks` |
+| `whole-file` | `.txt` / `.md` | the file is one unit | `document_pages` |
+
+**DOCX never gets a page number.** Pagination is produced by the renderer at
+layout time and is not in the file, so one would have to be invented — see
+RFC 0005. Extraction that cannot be done honestly fails and says why
+(`extraction-failed` with a reason); it is never a silent empty string.
+Sources are read-only and are never rewritten.
+
 ## Citations and stages (v0.2, RFC 0001)
 
 ```bash

matter.py

@@ -164,6 +164,77 @@ def cmd_cite(args):
                   f"(text {r['text_sha256'][:12]}, parser {r['parser']})")
 
 
+def cmd_pages(args):
+    """pages <dir> [path] — extract document text with source locators.
+
+    Stdlib-only. Scanned/LZW/CID-font PDFs fail honestly per page; nothing is
+    ever silently empty. PDF/text results write to document_pages keyed by
+    sha256+page (RFC 0004).
+
+    DOCX (RFC 0005) is routed to structural locators and written to
+    document_blocks instead. A DOCX never produces a document_pages row: the
+    format carries no rendered page numbers, so one would have to be invented.
+    """
+    conn = store.connect(args.case_dir)
+    from matterkit import extract, ingest
+    import hashlib
+    if args.path:
+        files = [(args.path, ingest.sha256_file(args.path))]
+    else:
+        docs = conn.execute("SELECT id, sha256, rel_path FROM documents").fetchall()
+        files = []
+        for d in docs:
+            # rel_path is relative to the cwd at ingest time; the id is a hash,
+            # not a path. Resolve rel_path against cwd first, then fall back to
+            # the matter dir (covers both the demo layout and older ingests).
+            full = os.path.abspath(d["rel_path"])
+            if not os.path.isfile(full):
+                full = os.path.join(args.case_dir, d["rel_path"])
+            if os.path.isfile(full):
+                files.append((full, d["sha256"]))
+    total_ok = total_fail = 0
+    for path, sha in files:
+        doc = extract.extract_text(path)
+        if doc.locator_scheme == extract.LOCATOR_SCHEME_DOCX:
+            extract.record_blocks(conn, sha, doc)
+            name = os.path.basename(path)
+            if doc.status == "extracted":
+                total_ok += len(doc.blocks)
+                print(f"✓ {name}: {len(doc.blocks)} block(s) "
+                      f"[{doc.locator_scheme}; DOCX carries no page numbers]")
+                if doc.parts and doc.parts.skipped:
+                    print(f"   · skipped by design: "
+                          f"{', '.join(doc.parts.skipped)}")
+                if doc.parts and doc.parts.unhandled:
+                    print(f"   ⚠ not descended into: "
+                          f"{', '.join(doc.parts.unhandled)}")
+            else:
+                total_fail += 1
+                print(f"✗ {name}: extraction failed (0 block(s))")
+                print(f"   ⚠ {doc.reason}")
+            continue
+        for pg in doc.pages:
+            conn.execute(
+                "INSERT OR REPLACE INTO document_pages "
+                "(document_sha256, page_number, text_sha256, text, status) "
+                "VALUES (?,?,?,?,?)",
+                (sha, pg.number, pg.text_sha256, pg.text, pg.status))
+            if pg.status == "extracted":
+                total_ok += 1
+            else:
+                total_fail += 1
+        n = len(doc.pages)
+        mark = "✓" if doc.pages and doc.pages[0].status == "extracted" else "✗"
+        print(f"{mark} {os.path.basename(path)}: {n} page(s) "
+              f"({total_ok} ok / {total_fail} failed so far)")
+        for pg in doc.pages:
+            if pg.status == "extraction-failed":
+                print(f"   ⚠ page {pg.number}: {pg.reason}")
+    conn.commit()
+    print(f"located units: {total_ok} extracted · {total_fail} failed "
+          f"(never silently empty)")
+
+
 def cmd_auth_stage(args):
     """Stage-validated authority status change (RFC 0001 §2.2c)."""
     conn = store.connect(args.case_dir)
@@ -349,6 +420,9 @@ def main():
     p.add_argument("--status", choices=["extracted", "extraction-failed"])
     p.set_defaults(fn=cmd_cite)
 
+    p = sub.add_parser("pages"); p.add_argument("case_dir"); p.add_argument("path", nargs="?")
+    p.set_defaults(fn=cmd_pages)
+
     p = sub.add_parser("stage"); p.add_argument("case_dir"); p.add_argument("authority_id"); p.add_argument("status")
     p.add_argument("--via", default=None); p.add_argument("--evidence", default=None)
     p.set_defaults(fn=cmd_auth_stage)