diff --git a/bench/icdar2013/__pycache__/score.cpython-313.pyc b/bench/icdar2013/__pycache__/score.cpython-313.pyc new file mode 100644 index 0000000..678893b Binary files /dev/null and b/bench/icdar2013/__pycache__/score.cpython-313.pyc differ diff --git a/bench/icdar2013/diag.py b/bench/icdar2013/diag.py index c8d5dc8..ff715e3 100644 --- a/bench/icdar2013/diag.py +++ b/bench/icdar2013/diag.py @@ -43,7 +43,7 @@ def main() -> int: for pdf, xml in pairs: gt = gt_relations(xml) out = subprocess.run( - [exe, "-strategy", "lines", pdf], capture_output=True, timeout=120 + [exe, "-strategy", os.environ.get("DIAG_STRATEGY","lines"), pdf], capture_output=True, timeout=120 ).stdout tables = json.loads(out or b"[]") detected_tables_total += len(tables) diff --git a/bench/icdar2013/extract.go b/bench/icdar2013/extract.go index ba010bd..255cd04 100644 --- a/bench/icdar2013/extract.go +++ b/bench/icdar2013/extract.go @@ -39,12 +39,24 @@ func main() { lines := mk(pdftable.StrategyLines, pdftable.StrategyLines) text := mk(pdftable.StrategyText, pdftable.StrategyText) + // "mixed" is the booktabs case: horizontal rules give the rows, word + // alignment gives the columns. A table ruled only horizontally has no + // ruling intersections at all, so pure "lines" cannot see it. + mixed := mk(pdftable.StrategyText, pdftable.StrategyLines) + auto := mk(pdftable.StrategyAuto, pdftable.StrategyAuto) + var attempts []pdftable.TableSettings switch *strategy { case "lines": attempts = []pdftable.TableSettings{lines} case "text": attempts = []pdftable.TableSettings{text} + case "mixed": + attempts = []pdftable.TableSettings{mixed} + case "lines-then-mixed": + attempts = []pdftable.TableSettings{lines, mixed} + case "auto": + attempts = []pdftable.TableSettings{auto} default: // fallback: ruled cells first, whitespace alignment if none attempts = []pdftable.TableSettings{lines, text} } diff --git a/bench/icdar2013/score.py b/bench/icdar2013/score.py index 1e3fdb8..9e26dc6 100644 --- a/bench/icdar2013/score.py +++ b/bench/icdar2013/score.py @@ -154,10 +154,9 @@ def main() -> int: systems = { "pdftable (lines)": lambda p: run_pdftable(exe, p, "lines", False), - "pdftable (fallback)": lambda p: run_pdftable(exe, p, "fallback", False), - "pdftable (fallback+merge)": lambda p: run_pdftable(exe, p, "fallback", True), + "pdftable (AUTO)": lambda p: run_pdftable(exe, p, "auto", False), + "pdftable (AUTO +merge)": lambda p: run_pdftable(exe, p, "auto", True), "pdfplumber (lines)": lambda p: run_pdfplumber(p, "lines"), - "pdfplumber (text)": lambda p: run_pdfplumber(p, "text"), } totals = {k: [0, 0, 0] for k in systems} diff --git a/docs/README.md b/docs/README.md index 41c9e96..dceb3ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ release history in [`CHANGELOG.md`](../CHANGELOG.md). | date | subject | headline | | --- | --- | --- | | [2026-08-02](evaluations/2026-08-02-icdar2013-table-structure.md) | ICDAR 2013 table detection + structure | F1 0.362 end-to-end; the bottleneck is **detection**, not cell accuracy | +| [2026-08-02](evaluations/2026-08-02-strategy-auto-negative-result.md) | `StrategyAuto` for one-axis-ruled tables | **negative result** — detection 22%→18% missed, but F1 0.362→0.358. Shipped opt-in only | | [2026-08-02](evaluations/2026-08-02-font-metrics-and-table-fidelity.md) | font metrics and table fidelity | position drift 11.99pt → **0.0000pt**; negative-sign loss 19% → **0%** | ### Conventions diff --git a/docs/evaluations/2026-08-02-strategy-auto-negative-result.md b/docs/evaluations/2026-08-02-strategy-auto-negative-result.md new file mode 100644 index 0000000..ae88e63 --- /dev/null +++ b/docs/evaluations/2026-08-02-strategy-auto-negative-result.md @@ -0,0 +1,105 @@ +# StrategyAuto — improves detection, does not improve accuracy + +**Date:** 2026-08-02 +**Commit:** `05c0c92` + `StrategyAuto` +**Harness:** [`bench/icdar2013`](../../bench/icdar2013/) +**Verdict:** shipped as **opt-in**; does **not** become a default. The +hypothesis it tested is disproved. + +## Hypothesis + +The [ICDAR 2013 evaluation](2026-08-02-icdar2013-table-structure.md) found +28 of 125 documents (22%) where pdftable detected **no table at all**, and +attributed it to tables ruled on one axis only. `lines` builds cells from +*intersecting* rulings, so a horizontally-ruled table yields none. + +Confirmed directly — every zero-detection document had horizontal rules +and zero vertical ones: + +``` +us-017: 218 H, 0 V lines=0 mixed=6 +us-018: 226 H, 0 V lines=0 mixed=7 +us-024: 135 H, 0 V lines=0 mixed=4 +us-025: 225 H, 0 V lines=0 mixed=3 +``` + +So: pick `lines` for the ruled axis and `text` for the unruled one, but +**only when the other axis is ruled** — those rulings being the evidence +that a table is genuinely present. Expectation: recall rises, precision +holds. + +## Result — the hypothesis was wrong + +| system | precision | recall | F1 | +| --- | --- | --- | --- | +| pdftable (`lines`) | 0.865 | 0.229 | **0.362** | +| pdftable (`auto`) | 0.797 | 0.231 | **0.358** | +| pdftable (`auto` + MergeSplitTokens) | 0.826 | 0.230 | 0.359 | +| pdfplumber (`lines`) | 0.868 | 0.235 | 0.370 | + +Recall moved 0.229 → 0.231. Precision fell 0.865 → 0.797. **Net slightly +worse.** + +## Why — and it corrects the earlier conclusion + +Detection did improve, exactly as predicted: + +| | `lines` | `auto` | +| --- | --- | --- | +| documents with no table found | 28 (22%) | **23 (18%)** | +| tables detected | 306 | **331** | +| F1 *on documents where one was found* | 0.556 | **0.400** | + +We now find tables in five more documents. **The tables we find there are +gridded badly.** Once the newly-detected hard documents enter the scored +set, quality on that set falls from 0.556 to 0.400. + +The earlier report concluded "the bottleneck is detection, not +structure", reasoning from precision 0.865 on detected documents. That +was true of the documents `lines` could already see — an easier +population, self-selected by being fully ruled. On the harder ones, +**structure is weak too**: knowing where the table is does not tell you +where its columns are, and word-alignment clustering does not recover +them on these layouts. + +Corrected statement: **detection and structure are both weak on +one-axis-ruled tables. Fixing detection alone converts almost nothing.** + +## What was shipped, and why anything at all + +`StrategyAuto` is available per axis and is **not** a default. It is +correct for its stated case — it finds tables that `lines` cannot see — +and a caller who knows their corpus is booktabs-ruled gets a real +improvement. `TestAutoIsNotTheDefault` pins that it stays opt-in. + +The conservative rule is load-bearing and worth keeping even though the +headline did not move. Auto declines to guess when *neither* axis is +ruled. Without that guard, a naive `lines`→`text` fallback scores 0.223 +precision — prose has word alignment too, and the text strategy reports a +table for it. That is why the fallback row in the earlier report is worse +than `lines` alone. + +## What this implies for the next attempt + +Do not spend more effort on heuristics for finding the table region. The +measurement says the missing piece is **row and column structure** on +layouts where the rules do not supply it. + +That is a different shape of problem, and it maps onto what layout models +actually output. Table Transformer and similar predict rows, columns and +spanning cells — not merely a table bounding box. That is the part +geometry cannot recover here. + +Revised split for the hybrid: + +- **layout/VLM model → rows, columns and spans** (not just "where is the + table") +- **pdftable text layer → cell contents and coordinates**, which stays + exact and keeps citation geometry + +## Reproduce + +```sh +python bench/icdar2013/run.py +DIAG_STRATEGY=auto python bench/icdar2013/diag.py +``` diff --git a/finder.go b/finder.go index 2ae0bd7..73c1b46 100644 --- a/finder.go +++ b/finder.go @@ -869,7 +869,7 @@ func ensureSupportedStrategies(s TableSettings) error { {"horizontal", s.HorizontalStrategy}, } { switch pair.strategy { - case StrategyLines, StrategyLinesStrict, StrategyText, StrategyExplicit: + case StrategyLines, StrategyLinesStrict, StrategyText, StrategyExplicit, StrategyAuto: // ok default: return fmt.Errorf("%w: unknown %s_strategy %q", ErrUnsupported, pair.axis, pair.strategy) diff --git a/page.go b/page.go index 4131535..7974fb9 100644 --- a/page.go +++ b/page.go @@ -460,9 +460,27 @@ func (p *page) findTableEdges(s TableSettings) ([]layout.Edge, error) { pageWidth := p.Width() pageHeight := p.Height() + // Resolve StrategyAuto now that the page's drawn edges are known. + vStrategy := resolveAuto(s.VerticalStrategy, layout.Vertical, lineLikeEdges) + hStrategy := resolveAuto(s.HorizontalStrategy, layout.Horizontal, lineLikeEdges) + + // Auto may have turned an axis into "text" after the fact, so the + // words it needs might not have been fetched above. + if words == nil && (vStrategy == StrategyText || hStrategy == StrategyText) { + opts := DefaultWordOpts() + opts.XTolerance = s.TextTolerance + opts.YTolerance = s.TextTolerance + opts.KeepBlankChars = s.KeepBlankChars + w, err := p.Words(opts) + if err != nil { + return nil, err + } + words = w + } + // Per-axis base edge derivation. - vEdges := p.baseEdges(s.VerticalStrategy, layout.Vertical, lineLikeEdges, words, s) - hEdges := p.baseEdges(s.HorizontalStrategy, layout.Horizontal, lineLikeEdges, words, s) + vEdges := p.baseEdges(vStrategy, layout.Vertical, lineLikeEdges, words, s) + hEdges := p.baseEdges(hStrategy, layout.Horizontal, lineLikeEdges, words, s) // Explicit overrides are added on top of whichever base set was // chosen. With StrategyExplicit the base set is empty so the @@ -493,8 +511,42 @@ func (p *page) findTableEdges(s TableSettings) ([]layout.Edge, error) { // drawn primitives (Lines / Rects / Curves), i.e. whether // findTableEdges needs to call Objects(). Text and explicit // strategies don't. +// +// StrategyAuto counts: it cannot decide anything without seeing what the +// page drew, and it may well resolve to "lines". func isLineLike(s TableStrategy) bool { - return s == StrategyLines || s == StrategyLinesStrict + return s == StrategyLines || s == StrategyLinesStrict || s == StrategyAuto +} + +// minEdgesForAxis is how many rulings an axis needs before Auto treats it +// as genuinely ruled. Two is the floor that can bound a cell; a single +// stray rule (an underline, a header separator, a page border) is not a +// table and must not be read as evidence of one. +const minEdgesForAxis = 2 + +// resolveAuto turns StrategyAuto into a concrete strategy for one axis, +// using the edges the page actually drew. Every other strategy passes +// through untouched. +// +// See StrategyAuto for why the "neither axis is ruled" case deliberately +// resolves to lines rather than text. +func resolveAuto(s TableStrategy, orientation layout.Orientation, lineLikeEdges []layout.Edge) TableStrategy { + if s != StrategyAuto { + return s + } + other := layout.Horizontal + if orientation == layout.Horizontal { + other = layout.Vertical + } + if len(layout.FilterEdgesByOrientation(lineLikeEdges, orientation)) >= minEdgesForAxis { + return StrategyLines + } + if len(layout.FilterEdgesByOrientation(lineLikeEdges, other)) >= minEdgesForAxis { + // The other axis is ruled, so a table is really here — we just + // have to infer this axis from word alignment. + return StrategyText + } + return StrategyLines } // baseEdges returns the per-axis edges produced by the named strategy. diff --git a/strategy_auto_test.go b/strategy_auto_test.go new file mode 100644 index 0000000..42b6c60 --- /dev/null +++ b/strategy_auto_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2026 Halleluyah Oludele +// Licensed under the MIT License. + +package pdftable + +import ( + "testing" + + "github.com/hallelx2/pdftable/internal/layout" +) + +func vEdge(x, y0, y1 float64) layout.Edge { + return layout.Edge{X0: x, X1: x, Y0: y0, Y1: y1, Orientation: layout.Vertical} +} + +func hEdge(y, x0, x1 float64) layout.Edge { + return layout.Edge{X0: x0, X1: x1, Y0: y, Y1: y, Orientation: layout.Horizontal} +} + +// TestResolveAutoPicksPerAxis covers the decision table for StrategyAuto. +// +// The case it exists for is the table ruled on one axis only — booktabs +// style, the house style of most government and academic publishing. +// Such a table has no ruling intersections at all, so "lines" finds +// nothing on either axis. +func TestResolveAutoPicksPerAxis(t *testing.T) { + fullGrid := []layout.Edge{ + vEdge(100, 0, 50), vEdge(200, 0, 50), + hEdge(0, 100, 200), hEdge(50, 100, 200), + } + horizOnly := []layout.Edge{ + hEdge(0, 100, 200), hEdge(25, 100, 200), hEdge(50, 100, 200), + } + vertOnly := []layout.Edge{ + vEdge(100, 0, 50), vEdge(150, 0, 50), vEdge(200, 0, 50), + } + + cases := []struct { + name string + edges []layout.Edge + wantV TableStrategy + wantH TableStrategy + why string + }{ + { + "fully ruled", fullGrid, StrategyLines, StrategyLines, + "both axes are ruled, so neither needs inferring", + }, + { + "horizontal rules only", horizOnly, StrategyText, StrategyLines, + "rows come from the rules; columns must be inferred from words", + }, + { + "vertical rules only", vertOnly, StrategyLines, StrategyText, + "the mirror image: columns from rules, rows inferred", + }, + { + "no rulings at all", nil, StrategyLines, StrategyLines, + "NOT text/text — see below", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + gotV := resolveAuto(StrategyAuto, layout.Vertical, tc.edges) + gotH := resolveAuto(StrategyAuto, layout.Horizontal, tc.edges) + if gotV != tc.wantV || gotH != tc.wantH { + t.Errorf("v=%q h=%q, want v=%q h=%q (%s)", gotV, gotH, tc.wantV, tc.wantH, tc.why) + } + }) + } +} + +// TestResolveAutoDeclinesWithoutEvidence is the property that keeps +// StrategyAuto from wrecking precision. +// +// Falling back to "text" on a page with no rulings is what makes a naive +// lines->text fallback score WORSE than lines alone: measured on ICDAR +// 2013 it drops precision from 0.865 to 0.223, because prose has word +// alignment too and the text strategy will report a table for it. +// +// Rulings on the OTHER axis are the evidence that a table is really +// present. Without that evidence Auto must decline to guess. +func TestResolveAutoDeclinesWithoutEvidence(t *testing.T) { + // A prose page: no rulings anywhere. + if got := resolveAuto(StrategyAuto, layout.Vertical, nil); got != StrategyLines { + t.Errorf("no rulings resolved to %q; must stay %q so prose is not read as a table", + got, StrategyLines) + } + + // A single stray rule — an underline, a header separator, a page + // border — is not a table and must not count as evidence. + one := []layout.Edge{hEdge(0, 100, 200)} + if got := resolveAuto(StrategyAuto, layout.Vertical, one); got != StrategyLines { + t.Errorf("one stray rule resolved to %q; %d edges are needed before an axis counts as ruled", + got, minEdgesForAxis) + } +} + +// TestResolveAutoLeavesOtherStrategiesAlone guards the pass-through. +func TestResolveAutoLeavesOtherStrategiesAlone(t *testing.T) { + for _, s := range []TableStrategy{ + StrategyLines, StrategyLinesStrict, StrategyText, StrategyExplicit, + } { + if got := resolveAuto(s, layout.Vertical, nil); got != s { + t.Errorf("resolveAuto(%q) = %q, want it untouched", s, got) + } + } +} + +// TestAutoIsNotTheDefault pins that opting in is required. Auto improves +// table DETECTION but measurably lowers aggregate F1 on ICDAR 2013 +// (0.362 -> 0.358), so it must never be silently switched on. +func TestAutoIsNotTheDefault(t *testing.T) { + d := DefaultTableSettings() + if d.VerticalStrategy == StrategyAuto || d.HorizontalStrategy == StrategyAuto { + t.Error("StrategyAuto must not be a default — it lowers aggregate benchmark F1") + } +} diff --git a/table.go b/table.go index 1f3f9d9..8c1ee2d 100644 --- a/table.go +++ b/table.go @@ -58,6 +58,34 @@ const ( // The "explicit" strategy on an axis requires at least two // coordinates on that axis; fewer than two produces an error. StrategyExplicit TableStrategy = "explicit" + + // StrategyAuto picks "lines" or "text" for its axis by looking at + // what the page actually drew. It exists for the very common table + // that is ruled on ONE axis only. + // + // A table with horizontal rules and no vertical ones — booktabs + // style, and the house style of most government and academic + // publishing — yields no ruling intersections at all, so "lines" + // finds nothing on either axis. On the ICDAR 2013 competition set + // that accounted for every document where pdftable detected no + // table whatsoever: us-017 has 218 horizontal rules and 0 vertical, + // us-018 has 226 and 0, us-025 has 225 and 0. + // + // The rule, per axis: + // + // - this axis has usable rulings -> "lines" + // - it does not, but the OTHER axis does -> "text" + // - neither axis has rulings -> "lines" (find nothing) + // + // That last case is the important one. Falling back to "text" when + // the page has no rulings at all is what makes a naive lines->text + // fallback score WORSE than "lines" alone: on the same benchmark it + // drops precision from 0.865 to 0.223, because a prose page has + // word alignment too and the text strategy will happily report a + // table for it. Rulings on the other axis are the evidence that a + // table is really there; without that evidence Auto declines to + // guess. + StrategyAuto TableStrategy = "auto" ) // TableSettings controls table finding. Construct via