diff --git a/.github/branch-protection.json b/.github/branch-protection.json new file mode 100644 index 0000000..2d1362c --- /dev/null +++ b/.github/branch-protection.json @@ -0,0 +1,9 @@ +{ + "required_status_checks": { + "strict": false, + "contexts": ["gate"] + }, + "enforce_admins": false, + "required_pull_request_reviews": null, + "restrictions": null +} diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml new file mode 100644 index 0000000..5838aba --- /dev/null +++ b/.github/workflows/gate.yml @@ -0,0 +1,102 @@ +# The merge gate. Every push to main and every pull request runs the tests and +# then replays the three folders of ci-corpus, the committed corpus, which +# ci-baseline.json names. gate-baseline.json names the folders of the private +# corpus, which a runner cannot read, and is replayed by the workbench check. +# The job is a required status check on main, so a branch that breaks a build +# cannot be merged. +name: gate + +on: + push: + branches: [main] + pull_request: + +jobs: + gate: + name: gate + # Pinned rather than ubuntu-latest: the runner's TeX Live decides what + # xelatex writes, and the PDF's bytes are the OCR cache's key. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + # The Dockerfile's package list, which is the one record of what the PDF + # generator's template.latex loads. + - name: Install pandoc, poppler and TeX Live + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + pandoc \ + poppler-utils \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-science \ + texlive-lang-chinese \ + texlive-lang-arabic \ + texlive-bibtex-extra \ + texlive-plain-generic \ + texlive-fonts-recommended \ + lmodern \ + fonts-noto-core \ + fonts-noto-cjk + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install poetry + run: pipx install poetry + + - name: Install the package + run: poetry install --with dev + + # SOURCE_DATE_EPOCH fixes the timestamp xelatex writes into the PDF, so + # the same tex source compiles to the same bytes and the OCR cache key + # does not change between runs. + - name: Compile the CI corpus PDF + run: | + cd ci-corpus/tex + SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ + xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex + rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log + + # Keyed by the PDF's bytes, which is also how ocr_pdf names its entry + # inside the directory. A restored entry for another PDF is unused. + - name: Restore the OCR cache + uses: actions/cache@v4 + with: + path: ~/.cache/in2lambda-agent + key: ocr-${{ hashFiles('ci-corpus/pdf/*.pdf') }} + restore-keys: ocr- + + - name: Tests + run: poetry run pytest -q + + - name: Gate + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: poetry run in2lambda-agent gate ci-baseline.json + + # What this run would record, so that a baseline change is committed from + # what CI saw. Written whether or not the gate passed. + - name: Record what this run did + if: always() + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: | + cp ci-baseline.json "${RUNNER_TEMP}/ci-baseline.json" + poetry run in2lambda-agent gate --record "${RUNNER_TEMP}/ci-baseline.json" + + - name: Upload it + if: always() + uses: actions/upload-artifact@v4 + with: + name: recorded-baseline + path: ${{ runner.temp }}/ci-baseline.json diff --git a/.gitignore b/.gitignore index 45e2c81..ea71284 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ __pycache__/ dist/ # in2lambda's KaTeX converter writes this into the working directory on import. log +# The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex +# in the workflow, so the repository holds the tex source and not the output. +ci-corpus/pdf/*.pdf diff --git a/README.md b/README.md index 2f30217..2fee31f 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ poetry run in2lambda-agent corpus ExampleContents --suffix tex --suffix md In full: ```sh -poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] +poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] [--cache DIR] ``` `ROOT` is the corpus directory and each `PATH` a folder under it to run, defaulting to @@ -285,6 +285,82 @@ review, rejections where its value comes from. One document that fails is one row and not the end of the sweep, and a set whose folder cannot be copied is a row for each of its documents. +`--cache` (default `./.in2lambda-agent`) is where the OCR of each PDF is kept. A sweep +pointed at a cache that an earlier run filled makes no Mathpix call, and needs no +Mathpix credentials. + +## Gate + +Nothing merges without a replay over real documents. `gate` reruns the saved specs +over the folders a baseline file names, and compares what each folder did this run +with what the baseline records: + +```sh +poetry run in2lambda-agent gate BASELINE [--record] [--cache DIR] [--work DIR] +``` + +Every run is `corpus --replay`, so no model call is made. The command prints one line +per folder, and exits 1 when a folder builds fewer documents than the baseline records +or when a single document does worse than the baseline records it doing. The second +check is what a baseline of no builds rests on: a corpus where every document faults +still reports the document that stops being read. + +``` +work /tmp/in2lambda-agent-gate-3f1a +UCL_MechEng built 0 faulted 0 build refused 0 skipped 1 no spec 2 (baseline built 0) + worse UCL_MechEng/Worksheet_2.pdf faulted -> no spec: replay: no model call is allowed +``` + +Run the command from the repository root. The gate reads `specs`, and a folder's `root` +where `root` is relative, from the directory the command runs in. + +The repository holds two baselines, because a clone holds the second corpus and not the +first: + +| File | Corpus | Run by | +| --- | --- | --- | +| `gate-baseline.json` | the three folders of `ExampleContents`, which is private | the workbench check | +| `ci-baseline.json` | the three folders of `ci-corpus`, which is committed | `.github/workflows/gate.yml` | + +Each folder's `root` and `suffixes` are written by hand. `built`, the count of documents +that built, and `documents`, the outcome of each single document, are what `--record` +writes. A folder the file records no `built` for passes on any count, and its line reads +`(not recorded)`. A change to a recorded count or outcome belongs in a pull request that +says why the count or the outcome changed. + +`gate-baseline.json` records 0 built for all three folders. Every document of +`ExampleContents` replays to `faulted`, because pandoc's line wrapping is reported as a +math delimiter error and each document needs a fixing round that a replay does not run. +The recorded outcomes are what the gate defends until a later ticket raises the count. + +`--cache` defaults to `~/.cache/in2lambda-agent`, outside any worktree, so that a PDF +converted on one branch is not converted again on the next. `--work` defaults to a new +directory under the system temp directory, which the gate does not delete: read the +drafts of a folder that failed there. The gate also copies the spec tree into the work +directory and replays the copy, because a sweep appends a record of each run beside the +spec it reads. Neither directory is inside the repository, so `git status` after a gate +run reports no new file. + +`.github/workflows/gate.yml` runs pytest and then `gate ci-baseline.json` on every push +to `main` and every pull request. Mathpix reads `ci-corpus/pdf/sheet-1.pdf` once and the +workflow stores the markdown in the Actions cache under the PDF's hash. The job needs +two repository secrets, `MATHPIX_APP_ID` and `MATHPIX_API_KEY`. + +A pull request from a fork is given neither secret. `actions/cache` restores the cache +of the base branch for a fork, and the cached markdown is what `ci-corpus/pdf` then +replays, so the job passes without the secrets. If the cache is empty — the PDF's bytes +changed, or GitHub evicted the entry — Mathpix cannot be called, `ci-corpus/pdf` builds +0 against a recorded 1, the job fails and the pull request cannot be merged. A +maintainer merges that branch by pushing it to a branch of this repository, where the +secrets are read. + +The `gate` job is a required status check on `main`: + +```sh +gh api -X PUT repos/{owner}/{repo}/branches/main/protection \ + --input .github/branch-protection.json +``` + ## Docker The image carries pandoc, a TeX Live whose xelatex runs the PDF generator's diff --git a/ci-baseline.json b/ci-baseline.json new file mode 100644 index 0000000..ebd5e3b --- /dev/null +++ b/ci-baseline.json @@ -0,0 +1,38 @@ +{ + "specs": "corpus-specs", + "folders": { + "ci-corpus/tex": { + "root": ".", + "suffixes": [ + "tex" + ], + "built": 4, + "documents": { + "ci-corpus/tex/sheet-1.tex": "built", + "ci-corpus/tex/sheet-2.tex": "built", + "ci-corpus/tex/sheet-3.tex": "built", + "ci-corpus/tex/solutions-2.tex": "built" + } + }, + "ci-corpus/docx": { + "root": ".", + "suffixes": [ + "docx" + ], + "built": 1, + "documents": { + "ci-corpus/docx/sheet.docx": "built" + } + }, + "ci-corpus/pdf": { + "root": ".", + "suffixes": [ + "pdf" + ], + "built": 1, + "documents": { + "ci-corpus/pdf/sheet-1.pdf": "built" + } + } + } +} diff --git a/ci-corpus/docx/sheet.docx b/ci-corpus/docx/sheet.docx new file mode 100644 index 0000000..137c8d0 Binary files /dev/null and b/ci-corpus/docx/sheet.docx differ diff --git a/ci-corpus/pdf/.gitkeep b/ci-corpus/pdf/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ci-corpus/tex/figures/ball.png b/ci-corpus/tex/figures/ball.png new file mode 100644 index 0000000..f37764b Binary files /dev/null and b/ci-corpus/tex/figures/ball.png differ diff --git a/ci-corpus/tex/sheet-1.tex b/ci-corpus/tex/sheet-1.tex new file mode 100644 index 0000000..3127901 --- /dev/null +++ b/ci-corpus/tex/sheet-1.tex @@ -0,0 +1,46 @@ +% Synthetic, in the shape of the corpus's problem sheets: a starred section +% heading, an enumerate of questions with lettered parts, and a solutions +% section at the end. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 1: Kinematics} + +\begin{enumerate} + +\item A ball is thrown straight up at $20\,\mathrm{m/s}$. + \begin{enumerate} + \item Find the greatest height it reaches. + \item Find its time of flight. + \end{enumerate} + +\item A block of mass $m$ rests on a slope of angle $\theta$. + \begin{enumerate} + \item Name the three forces acting on the block. + \item Find the least coefficient of friction that holds it still. + \end{enumerate} + +\end{enumerate} + +\section*{Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item $h = v^2 / 2g = 20.4\,\mathrm{m}$ + \item $t = 2v/g = 4.08\,\mathrm{s}$ + \end{enumerate} + +\item + \begin{enumerate} + \item Weight, the normal reaction, and friction along the slope. + \item $\mu = \tan\theta$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-2.tex b/ci-corpus/tex/sheet-2.tex new file mode 100644 index 0000000..f18652d --- /dev/null +++ b/ci-corpus/tex/sheet-2.tex @@ -0,0 +1,28 @@ +% Synthetic: questions with lettered parts and no solutions on the sheet. The +% solutions are in solutions-2.tex, which is a document of its own, as the +% corpus's problem sheets are. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Fields} + +\begin{enumerate} + +\item A dipole sits at the origin. + \begin{enumerate} + \item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. + \item Find the field on the axis at a distance $z$. + \end{enumerate} + +\item A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. + \begin{enumerate} + \item Find the radius of its circular path. + \item Find the period of the motion. + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-3.tex b/ci-corpus/tex/sheet-3.tex new file mode 100644 index 0000000..39d12d8 --- /dev/null +++ b/ci-corpus/tex/sheet-3.tex @@ -0,0 +1,50 @@ +% Synthetic: a sheet whose first question includes a figure, which is the shape +% that first failed a sweep on the path of an image. Nothing here is copied +% from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 3: Statics} + +\begin{enumerate} + +\item The beam below carries a load $W$ at its midpoint. + + \includegraphics[width=0.2\textwidth]{figures/ball.png} + + \begin{enumerate} + \item Find the reaction at each support. + \item Find the bending moment at the midpoint. + \end{enumerate} + +\item A ladder of mass $m$ leans against a smooth wall. + \begin{enumerate} + \item Draw the forces acting on the ladder. + \item Find the least angle at which it does not slip. + \end{enumerate} + +\end{enumerate} + +\section*{Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item $W/2$ at each support. + \item $M = WL/4$ + \end{enumerate} + +\item + \begin{enumerate} + \item The weight at the centre, the normal reaction at the wall, and the + normal reaction and friction at the floor. + \item $\tan\alpha = 1/2\mu$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/solutions-2.tex b/ci-corpus/tex/solutions-2.tex new file mode 100644 index 0000000..0cd053c --- /dev/null +++ b/ci-corpus/tex/solutions-2.tex @@ -0,0 +1,27 @@ +% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the +% shape that first failed a sweep — a document with no questions in it. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item Take the divergence term by term; each pair cancels. + \item $B = \mu_0 m / 2\pi z^3$ + \end{enumerate} + +\item + \begin{enumerate} + \item $r = mv / qB$ + \item $T = 2\pi m / qB$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml new file mode 100644 index 0000000..54afdc5 --- /dev/null +++ b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Para text~'^(STRESS ANALYSIS|Sheet |Note:)|Description automatically generated' +question: ListItem +part: Para text~'^[A-Z]' +solution: Para text~'^\[' +strip: ['^\d+\.\s+', '^\['] +layout: PartsOneSol diff --git a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml new file mode 100644 index 0000000..c4bf288 --- /dev/null +++ b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: Para +solution: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml new file mode 100644 index 0000000..a3c7db3 --- /dev/null +++ b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Para text~'Figure [0-9]:' +question: Para label~'^(1a|Q[0-9])' +part: text~'.' +strip: ['^1a\) ', '^Q[0-9]\s*', '^- ', '^[0-9]+\. ', '^i+\)\s*'] +layout: PartsOneSol diff --git a/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml b/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml new file mode 100644 index 0000000..97c98ce --- /dev/null +++ b/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Header +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml b/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml new file mode 100644 index 0000000..2a668e6 --- /dev/null +++ b/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: ListItem text~'^[A-Z]' +solution: after Header text=Solutions, ListItem +strip: ['^\d+\. '] +layout: PartsOneSol diff --git a/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml b/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml new file mode 100644 index 0000000..1754146 --- /dev/null +++ b/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml @@ -0,0 +1,4 @@ +ignore: Header +question: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/gate-baseline.json b/gate-baseline.json new file mode 100644 index 0000000..52bedac --- /dev/null +++ b/gate-baseline.json @@ -0,0 +1,50 @@ +{ + "specs": "corpus-specs", + "folders": { + "UCL_MechEng": { + "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", + "suffixes": [ + "pdf" + ], + "built": 0, + "documents": { + "UCL_MechEng/Tutorial_2_Solutions.pdf": "skipped", + "UCL_MechEng/Worksheet_1.pdf": "faulted", + "UCL_MechEng/Worksheet_2.pdf": "faulted" + } + }, + "PHYS40002-Mechanics/problem_sheets_and_figures": { + "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", + "suffixes": [ + "tex" + ], + "built": 0, + "documents": { + "PHYS40002-Mechanics/problem_sheets_and_figures/figures/tunnel-potential.tex": "skipped", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS1.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS2.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS3.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS4.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS5.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS6.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS7.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS8.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS9.tex": "faulted" + } + }, + "MECH60014_Stress_analysis_3": { + "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", + "suffixes": [ + "docx" + ], + "built": 0, + "documents": { + "MECH60014_Stress_analysis_3/Stress_Sheet3_Sol(1).docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet4.docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet5 (2).docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet6 (1).docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet6_ Sol (1).docx": "faulted" + } + } + } +} diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index 7664e72..f83bd6b 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -3,10 +3,11 @@ import argparse import getpass import sys +import tempfile from pathlib import Path from typing import Optional, Sequence -from in2lambda_agent import compare, corpus, pipeline +from in2lambda_agent import compare, corpus, gate, pipeline from in2lambda_agent.mathpix import MathpixClient, MathpixError from in2lambda_agent.model import ModelUnavailable, choose_backend from in2lambda_agent.ocr import ocr_pdf @@ -68,8 +69,8 @@ def build_parser() -> argparse.ArgumentParser: """The command line as the design spec describes it. Returns: - A parser with the `run`, `review`, `corpus`, `compare` and `ui` - subcommands. + A parser with the `run`, `review`, `corpus`, `gate`, `compare` and + `ui` subcommands. """ parser = argparse.ArgumentParser( prog="in2lambda-agent", @@ -206,6 +207,37 @@ def build_parser() -> argparse.ArgumentParser: default=corpus.DEFAULT_SPEC_DIR, help="The tree the sets' specs are kept in, mirroring the corpus.", ) + sweep.add_argument( + "--cache", + type=Path, + default=pipeline.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, so a sweep pointed at a cache " + "another run filled converts nothing.", + ) + + check = subcommands.add_parser( + "gate", help="Replay the corpus the baseline names and check it against it." + ) + check.add_argument("baseline", type=Path, help="The committed baseline file.") + check.add_argument( + "--record", + action="store_true", + help="Write this run's counts to the baseline instead of checking them.", + ) + check.add_argument( + "--cache", + type=Path, + default=gate.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, shared between worktrees so " + "that a conversion is paid for once.", + ) + check.add_argument( + "--work", + type=Path, + default=None, + help="Where the folders are copied to be run, under the system temp " + "directory by default so the check writes nothing where it was run.", + ) against = subcommands.add_parser( "compare", help="Check a PDF's OCR against the pages it came from." @@ -265,6 +297,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: specs=args.specs, replay=args.replay, rounds=args.rounds, + cache=args.cache, settings=load_settings(), ) print(f"{len(rows)} documents, written to {args.results}") @@ -273,6 +306,27 @@ def main(argv: Optional[Sequence[str]] = None) -> int: succeeded = {"built", "skipped"} return 0 if rows and all(row.outcome in succeeded for row in rows) else 1 + if args.command == "gate": + baseline = gate.read_baseline(args.baseline) + # The directory is printed and is not deleted, so that the drafts of a + # folder that failed can be read after the run. + work = args.work or Path(tempfile.mkdtemp(prefix="in2lambda-agent-gate-")) + print(f"work {work}") + report = gate.run( + baseline, + record=args.record, + cache=args.cache, + work=work, + settings=load_settings(), + ) + for name, summary in report.folders.items(): + print(gate.folder_line(name, summary)) + if args.record: + gate.write_baseline(baseline, args.baseline) + print(f"recorded {args.baseline}") + return 0 + return 1 if report.failed else 0 + if args.command == "compare": settings = load_settings() try: diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 494d19c..881036b 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -253,6 +253,7 @@ def run_one( settings: Settings, rounds: int = 3, replay: bool = False, + cache: Path = pipeline.DEFAULT_CACHE_DIR, backend: Optional[Backend] = None, ) -> Row: """Runs the pipeline over one document and reads the row off what it did. @@ -269,6 +270,7 @@ def run_one( settings: The environment the run has available. rounds: The round limit, ignored in a replay, which can run none. replay: Run the saved spec and nothing else, making no model call. + cache: Where the OCR of each PDF is kept. backend: The backend to write a spec with, chosen from the settings if absent. @@ -289,6 +291,7 @@ def run_one( # report: the row then says what the saved spec left rather than # that a call could not be made. rounds=0 if replay else rounds, + cache_dir=cache, backend=NoModel() if replay else backend, ) except ModelUnavailable as error: @@ -366,6 +369,7 @@ def sweep( specs: Path = DEFAULT_SPEC_DIR, replay: bool = False, rounds: int = 3, + cache: Path = pipeline.DEFAULT_CACHE_DIR, settings: Optional[Settings] = None, backend: Optional[Backend] = None, ) -> list[Row]: @@ -380,6 +384,8 @@ def sweep( specs: The tree the sets' specs are kept in, mirroring the corpus. replay: Run the saved specs and nothing else, making no model call. rounds: The round limit each run is given. + cache: Where the OCR of each PDF is kept, so that a sweep pointed at a + cache another run filled converts nothing. settings: The environment the runs have available. backend: The backend to write the specs with, chosen from the settings if absent. @@ -475,6 +481,7 @@ def sweep( settings=settings, rounds=rounds, replay=replay, + cache=cache, backend=backend, ) print(f"{row.outcome:<20} {row.source}") diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py new file mode 100644 index 0000000..35ef428 --- /dev/null +++ b/in2lambda_agent/gate.py @@ -0,0 +1,309 @@ +"""The merge gate: a replay over real documents, compared with a baseline. + +Every ticket before this one was tested on synthetic fixtures. The first sweep +over a folder of real PDFs failed on four faults no fixture had: a list-valued +selector, a file of solutions with no questions, an image path, and a wrapped +line. A merge now requires an end-to-end run over real documents. + +The run is `corpus.sweep(replay=True)` over the specs the repository keeps, so +it makes no model call. The baseline file names the folders to run and records +both how many documents each folder built and what each single document did. +The gate fails when a folder builds fewer documents than the baseline records, +or when any one document does worse than it is recorded as doing. The second +check is what gives a baseline of no builds at all teeth: a folder where every +document faults still notices the day one of them stops being read. A folder +the baseline records no count for passes on any count, which is how a folder is +added to the gate before it replays to a build worth defending. The baseline is +committed, and a change to it belongs in a pull request that states why the +counts changed. +""" + +import json +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from in2lambda_agent import corpus +from in2lambda_agent.settings import Settings + +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "in2lambda-agent" +"""Where the gate reads the OCR of each PDF. The directory is outside every +worktree, so a PDF converted on one branch is not converted again on the next. +`corpus` on its own keeps its cache under the directory the user ran from.""" + +RANK = {"built": 0, "build refused": 1, "faulted": 2, "skipped": 3} +"""How bad each outcome is. `skipped` is the worst of them: a document that +faults was at least read, and one that is skipped was not. Anything else is a +document that did not run either — a spec gone, a file unreadable, a document +the sweep no longer finds — and ranks with `skipped`.""" + +MISSING = "missing" +"""What a recorded document the sweep no longer finds is compared as.""" + + +def worse(current: str, recorded: str) -> bool: + """Whether a document did worse this run than the baseline records. + + A document that stops being read is a regression whatever it did before, + which is the case a baseline of nothing but faults rests on: widen what + counts as a solutions file and a document leaves the sweep as `skipped` + without a single count changing. + + Args: + current: What the document did this run. + recorded: What the baseline records it doing. + + Returns: + Whether it is a regression. + """ + return RANK.get(current, 3) > RANK.get(recorded, 3) + + +@dataclass +class Folder: + """One folder of a corpus, as the baseline holds it. + + Attributes: + root: The corpus directory the folder is under. The repository itself + for `ci-corpus`, which is committed; an absolute path for a corpus + outside the repository. + suffixes: The file suffixes that are documents in the folder: `pdf` for + a folder of scans, `tex` or `docx` for sources. + built: How many documents built when the baseline was recorded, or None + where no count is recorded. + documents: What each document did when the baseline was recorded, by + its path under the corpus root. `root` and `suffixes` are written + by hand; `built` and `documents` are what `--record` writes. + """ + + root: Path + suffixes: list[str] + built: Optional[int] = None + documents: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Regression: + """One document that did worse this run than the baseline records. + + Attributes: + document: Its path under the corpus root, as the baseline names it. + recorded: What the baseline records it doing. + current: What it did this run, or `missing` where the sweep no longer + finds it. + reason: What the sweep said about it, where it said anything. + """ + + document: str + recorded: str + current: str + reason: str = "" + + +@dataclass +class Baseline: + """The committed file the gate compares a sweep against. + + Attributes: + specs: The tree the folders' specs are kept in, relative to the + repository. Folder `A/B` reads `/A/B/in2lambda-spec.yaml`. + folders: The folders to run, by their path under their own root. + """ + + specs: Path + folders: dict[str, Folder] + + +@dataclass +class Summary: + """What one folder did this run, beside what the baseline records. + + Attributes: + built: How many documents built. + counts: How many did each other thing, by outcome. + recorded: How many built when the baseline was recorded, or None where + no count is recorded. + regressions: The documents that did worse than the baseline records. + """ + + built: int = 0 + counts: dict[str, int] = field(default_factory=dict) + recorded: Optional[int] = None + regressions: list[Regression] = field(default_factory=list) + + @property + def failed(self) -> bool: + """Whether this folder fails the gate.""" + if self.recorded is not None and self.built < self.recorded: + return True + return bool(self.regressions) + + +@dataclass +class Report: + """The whole run: one summary per folder, in the baseline's order.""" + + folders: dict[str, Summary] = field(default_factory=dict) + + @property + def failed(self) -> bool: + """Whether any folder fails the gate.""" + return any(one.failed for one in self.folders.values()) + + +def read_baseline(path: Path) -> Baseline: + """Reads the committed baseline. + + Args: + path: The JSON file. + + Returns: + The baseline. + """ + written = json.loads(Path(path).read_text(encoding="utf-8")) + return Baseline( + specs=Path(written["specs"]), + folders={ + name: Folder( + root=Path(one["root"]), + suffixes=list(one["suffixes"]), + built=one.get("built"), + documents=dict(one.get("documents", {})), + ) + for name, one in written["folders"].items() + }, + ) + + +def write_baseline(baseline: Baseline, path: Path) -> None: + """Writes the baseline back, counts and all. + + Args: + baseline: What to write. + path: The JSON file, which is overwritten. + """ + written = { + "specs": baseline.specs.as_posix(), + "folders": { + name: { + "root": one.root.as_posix(), + "suffixes": one.suffixes, + "built": one.built, + "documents": one.documents, + } + for name, one in baseline.folders.items() + }, + } + Path(path).write_text(json.dumps(written, indent=2) + "\n", encoding="utf-8") + + +def run( + baseline: Baseline, + *, + record: bool = False, + cache: Path, + work: Path, + settings: Optional[Settings] = None, +) -> Report: + """Replays every folder the baseline names and checks what each one did. + + Nothing is written into the repository or into a corpus: each folder is + swept into its own directory under `work`, and the table is written there. + + Args: + baseline: The folders to run and what to compare against. In record + mode this run's counts and outcomes replace them. + record: Take this run as the new baseline rather than checking it. + cache: Where the OCR of each PDF is kept. The directory is shared + between worktrees, so Mathpix converts each PDF once. + work: Where the folders are copied to be run. + settings: The environment the runs have available. + + Returns: + One summary per folder. + """ + work = Path(work) + # A sweep writes a record of each run beside the spec it read, and the + # record of a gate run is nobody's: it would land in the repository, under + # the specs the gate exists to replay. So the specs are copied under `work` + # and read from there. + specs = work / "specs" + # Fresh each run, so that a spec taken out of the tree is gone from the + # copy the sweep reads rather than left over from the run before. + shutil.rmtree(specs, ignore_errors=True) + if Path(baseline.specs).is_dir(): + shutil.copytree(baseline.specs, specs) + report = Report() + for name, folder in baseline.folders.items(): + rows = corpus.sweep( + folder.root, + paths=[Path(name)], + suffixes=folder.suffixes, + results=work / name / "results.csv", + work=work / "work", + specs=specs, + replay=True, + cache=cache, + settings=settings, + ) + summary = Summary(recorded=folder.built) + outcomes = {row.source: row.outcome for row in rows} + reasons = {row.source: row.reason for row in rows} + for outcome in outcomes.values(): + if outcome == "built": + summary.built += 1 + else: + summary.counts[outcome] = summary.counts.get(outcome, 0) + 1 + if record: + folder.built = summary.built + folder.documents = outcomes + # A recording run reports what this sweep did, so it fails nothing. + summary.recorded = summary.built + else: + for document, was in folder.documents.items(): + now = outcomes.get(document, MISSING) + if worse(now, was): + summary.regressions.append( + Regression(document, was, now, reasons.get(document, "")) + ) + report.folders[name] = summary + return report + + +def folder_line(name: str, summary: Summary) -> str: + """The gate's line for a folder: what it did, beside what is recorded. + + A folder whose documents all did as well as recorded is one line. Each + document that did worse adds an indented line of its own beneath it, so + that a failure names the document rather than only the count. + + Args: + name: The folder, as the baseline names it. + summary: What it did. + + Returns: + The line, and a line per regression beneath it. + """ + counts = " ".join( + f"{outcome} {summary.counts.get(outcome, 0)}" + for outcome in ("faulted", "build refused", "skipped") + ) + other = sorted(set(summary.counts) - {"faulted", "build refused", "skipped"}) + recorded = ( + "not recorded" + if summary.recorded is None + else f"baseline built {summary.recorded}" + ) + return "\n".join( + [ + f"{name:<40} built {summary.built} {counts}" + + "".join(f" {outcome} {summary.counts[outcome]}" for outcome in other) + + f" ({recorded})", + *( + f" worse {one.document} {one.recorded} -> {one.current}" + + (f": {one.reason}" if one.reason else "") + for one in summary.regressions + ), + ] + ) diff --git a/in2lambda_agent/ocr.py b/in2lambda_agent/ocr.py index beee014..c4fb7dc 100644 --- a/in2lambda_agent/ocr.py +++ b/in2lambda_agent/ocr.py @@ -9,6 +9,7 @@ import shutil from dataclasses import dataclass from pathlib import Path +from typing import Optional from in2lambda_agent.mathpix import MathpixClient @@ -29,6 +30,27 @@ class OcrResult: fresh: bool +def cached(pdf: Path, cache_dir: Path) -> Optional[OcrResult]: + """The conversion already in the cache, or None where there is none. + + Asked before a client is built, so that a document whose OCR was fetched + once runs again with no Mathpix credentials at all: a worktree, or a CI job + on a fork, has the cache and not the account. + + Args: + pdf: The PDF whose conversion is wanted. + cache_dir: Holds one entry per document, named by the PDF's hash. + + Returns: + Where the markdown and its media folder are, or None. + """ + entry = Path(cache_dir) / _hash(pdf) + markdown = entry / SOURCE_NAME + if not markdown.exists(): + return None + return OcrResult(markdown, entry / MEDIA_NAME, fresh=False) + + def ocr_pdf( pdf: Path, *, cache_dir: Path, client: MathpixClient, fresh: bool = False ) -> OcrResult: @@ -46,11 +68,12 @@ def ocr_pdf( Raises: MathpixError: If the conversion fails; the entry is left absent. """ + if not fresh and (hit := cached(pdf, cache_dir)) is not None: + return hit + entry = Path(cache_dir) / _hash(pdf) markdown = entry / SOURCE_NAME media = entry / MEDIA_NAME - if markdown.exists() and not fresh: - return OcrResult(markdown, media, fresh=False) # A fresh pass restarts the pipeline for this document, so the whole entry # goes: anything a later stage comes to keep beside source.md — a draft, a diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index fe721d1..d57f05f 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -36,7 +36,7 @@ from in2lambda_agent.fix import RoundResult, fix_round, summary, unrepaired from in2lambda_agent.mathpix import MathpixClient from in2lambda_agent.model import Backend, ModelUnavailable, Usage, choose_backend -from in2lambda_agent.ocr import MEDIA_NAME, ocr_pdf +from in2lambda_agent.ocr import MEDIA_NAME, cached, ocr_pdf from in2lambda_agent.review import RECORD, Question, Review, choose from in2lambda_agent.settings import Settings from in2lambda_agent.spec import RECORD_NAME, record_run, spec_path, write_spec @@ -155,7 +155,8 @@ def run( Raises: MathpixError: If a PDF cannot be converted, MissingCredentials among - them when the run has no Mathpix credentials. + them when a conversion is needed and the run has no Mathpix + credentials. A PDF already in the cache needs none. ModelUnavailable: If a spec must be written and no backend can run. BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. @@ -521,12 +522,18 @@ def _markdown( the OCR stage says about it. Raises: - MathpixError: If the PDF cannot be converted. + MathpixError: If the PDF cannot be converted, MissingCredentials among + them. A PDF already in the cache is not converted and needs none. """ if document.suffix.lower() != ".pdf": return document, None, f"not needed for {document.name}" - client = mathpix or MathpixClient.from_settings(settings) - ocr = ocr_pdf(document, cache_dir=cache_dir, client=client, fresh=fresh) + # The cache is asked before the client is built: a document converted once + # runs again with no credentials, which is what lets a worktree or a fork's + # CI job replay a corpus of PDFs it cannot pay for. + ocr = None if fresh else cached(document, cache_dir) + if ocr is None: + client = mathpix or MathpixClient.from_settings(settings) + ocr = ocr_pdf(document, cache_dir=cache_dir, client=client, fresh=fresh) # A fresh pass is a restart: every stage below reads the new markdown. if ocr.fresh: return ocr.markdown, ocr.media, f"fresh pass, restarting from {ocr.markdown}" diff --git a/tests/test_cli.py b/tests/test_cli.py index e495e44..77805be 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,7 @@ import pytest from conftest import FakeMathpix -from in2lambda_agent import cli, compare, corpus, pipeline +from in2lambda_agent import cli, compare, corpus, gate, pipeline from in2lambda_agent.cli import build_parser, main, reviewer_name from in2lambda_agent.model import Usage from in2lambda_agent.settings import Settings @@ -91,6 +91,7 @@ def test_corpus_defaults(): assert args.results == Path("results.csv") assert args.work == Path(".in2lambda-agent/corpus") assert args.specs == Path("corpus-specs") + assert args.cache == Path(".in2lambda-agent") def test_corpus_every_option(): @@ -113,6 +114,8 @@ def test_corpus_every_option(): "working", "--specs", "saved", + "--cache", + "cached", ] ) @@ -123,6 +126,104 @@ def test_corpus_every_option(): assert args.results == Path("sweep.csv") assert args.work == Path("working") assert args.specs == Path("saved") + assert args.cache == Path("cached") + + +def test_the_corpus_cache_is_handed_to_the_sweep(monkeypatch): + given = {} + + def record(*args, **kwargs): + given.update(kwargs) + return [] + + monkeypatch.setattr(corpus, "sweep", record) + + main(["corpus", "ExampleContents", "--cache", "cached"]) + + assert given["cache"] == Path("cached") + + +def test_gate_defaults(): + args = build_parser().parse_args(["gate", "gate-baseline.json"]) + + assert args.command == "gate" + assert args.baseline == Path("gate-baseline.json") + assert args.record is False + assert args.cache == Path.home() / ".cache" / "in2lambda-agent" + # Chosen when the command runs, so that two runs do not share a directory. + assert args.work is None + + +def test_gate_every_option(): + args = build_parser().parse_args( + [ + "gate", + "saved.json", + "--record", + "--cache", + "cached", + "--work", + "working", + ] + ) + + assert args.record is True + assert args.cache == Path("cached") + assert args.work == Path("working") + + +def test_a_gate_that_passes_exits_zero(tmp_path, monkeypatch, capsys): + path = written_baseline(tmp_path) + report = gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + monkeypatch.setattr(gate, "run", lambda *args, **kwargs: report) + + code = main(["gate", str(path)]) + + assert code == 0 + assert "tex" in capsys.readouterr().out + + +def test_a_gate_that_fails_exits_one_and_says_what_the_folder_built( + tmp_path, monkeypatch, capsys +): + path = written_baseline(tmp_path) + summary = gate.Summary(built=1, counts={"faulted": 1}, recorded=2) + monkeypatch.setattr( + gate, "run", lambda *args, **kwargs: gate.Report(folders={"tex": summary}) + ) + + code = main(["gate", str(path)]) + + assert code == 1 + assert "tex" in capsys.readouterr().out + + +def test_recording_writes_the_baseline_and_exits_zero(tmp_path, monkeypatch): + path = written_baseline(tmp_path) + + def record(baseline, **kwargs): + baseline.folders["tex"].built = 2 + return gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + + monkeypatch.setattr(gate, "run", record) + + code = main(["gate", str(path), "--record"]) + + assert code == 0 + assert gate.read_baseline(path).folders["tex"].built == 2 + + +def written_baseline(tmp_path): + """A baseline file on disk, for the gate command to read.""" + path = tmp_path / "baseline.json" + gate.write_baseline( + gate.Baseline( + specs=Path("corpus-specs"), + folders={"tex": gate.Folder(root=tmp_path / "corpus", suffixes=["tex"])}, + ), + path, + ) + return path def test_compare_defaults(): diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 38d2f46..2124397 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -99,6 +99,22 @@ def test_the_row_says_what_the_spec_made_of_the_document(root, tmp_path): assert sheet.model_seconds == 0.0 and sheet.wall_seconds > 0 +def test_the_sweeps_cache_is_where_each_run_looks_for_its_ocr( + root, tmp_path, monkeypatch +): + given = [] + + def record(source, **kwargs): + given.append(kwargs["cache_dir"]) + raise RuntimeError("as far as this goes") + + monkeypatch.setattr(pipeline, "run", record) + + rows = sweep(root, tmp_path, cache=tmp_path / "shared") + + assert given == [tmp_path / "shared"] * len(rows) + + def test_a_saved_spec_and_its_source_replay_with_no_model_call(root, tmp_path): for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): saved = tmp_path / "specs" / folder / SPEC_NAME diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..6dd7921 --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,316 @@ +"""The merge gate: a replay over a corpus, checked against a recorded baseline.""" + +from pathlib import Path + +import pytest +from test_corpus import make_set +from test_pipeline import SPEC, TEX_SPEC + +from in2lambda_agent import gate +from in2lambda_agent.gate import Baseline, Folder +from in2lambda_agent.settings import Settings +from in2lambda_agent.spec import SPEC_NAME + + +@pytest.fixture +def corpus_root(tmp_path): + """A corpus of two folders, with a saved spec each so a replay builds.""" + made = tmp_path / "corpus" + make_set(made, "sheets", ["sheet.md", "sheet-2.md"]) + make_set(made, "tex", ["tex-sheet.tex", "tex-sheet-2.tex"]) + for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): + saved = tmp_path / "specs" / folder / SPEC_NAME + saved.parent.mkdir(parents=True) + saved.write_text(text) + return made + + +@pytest.fixture +def baseline(corpus_root, tmp_path): + """The baseline as it is written by hand, before anything is recorded.""" + return Baseline( + specs=tmp_path / "specs", + folders={ + "sheets": Folder(root=corpus_root, suffixes=["md"]), + "tex": Folder(root=corpus_root, suffixes=["tex"]), + }, + ) + + +def run(baseline, tmp_path, **kwargs): + """A gate run with its cache and work directory under tmp_path.""" + return gate.run( + baseline, + cache=tmp_path / "cache", + work=tmp_path / "gate", + settings=Settings(), + **kwargs, + ) + + +def test_recording_fills_the_counts_and_leaves_what_was_written_by_hand( + baseline, corpus_root, tmp_path +): + report = run(baseline, tmp_path, record=True) + + assert not report.failed + assert baseline.folders["sheets"].built == 2 + assert baseline.folders["tex"].built == 2 + assert baseline.folders["tex"].documents == { + "tex/tex-sheet.tex": "built", + "tex/tex-sheet-2.tex": "built", + } + # The hand-written half is the run's to read, not to write over. + assert baseline.folders["tex"].root == corpus_root + assert baseline.folders["tex"].suffixes == ["tex"] + + +def test_a_recorded_baseline_passes_the_run_that_recorded_it(baseline, tmp_path): + run(baseline, tmp_path, record=True) + + report = run(baseline, tmp_path) + + assert not report.failed + assert [one.built for one in report.folders.values()] == [2, 2] + + +def test_a_folder_that_builds_fewer_than_recorded_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built += 1 + + report = run(baseline, tmp_path) + + assert report.failed + assert report.folders["tex"].failed and not report.folders["sheets"].failed + + +def test_a_folder_whose_documents_no_longer_build_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # The spec a replay has nothing to replay without. + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert report.failed + summary = report.folders["tex"] + assert (summary.built, summary.recorded) == (0, 2) + assert summary.counts == {"no spec": 2} + assert sorted(one.document for one in summary.regressions) == [ + "tex/tex-sheet-2.tex", + "tex/tex-sheet.tex", + ] + + +def test_a_folder_the_baseline_records_no_count_for_passes(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # A folder added to the gate before it replays to a build worth defending: + # `--record` writes both halves, so neither is there yet. + baseline.folders["tex"].built = None + baseline.folders["tex"].documents = {} + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["tex"].built == 0 + + +def test_a_document_the_baseline_does_not_know_is_not_a_failure( + baseline, corpus_root, tmp_path +): + run(baseline, tmp_path, record=True) + (corpus_root / "sheets" / "sheet-3.md").write_text( + (corpus_root / "sheets" / "sheet.md").read_text() + ) + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["sheets"].built == 3 + + +def test_a_document_the_corpus_no_longer_holds_is_a_regression( + baseline, corpus_root, tmp_path +): + run(baseline, tmp_path, record=True) + (corpus_root / "tex" / "tex-sheet-2.tex").unlink() + + report = run(baseline, tmp_path) + + assert report.failed + regression = report.folders["tex"].regressions[0] + assert (regression.document, regression.current) == ( + "tex/tex-sheet-2.tex", + gate.MISSING, + ) + + +def test_a_baseline_of_no_builds_still_notices_a_document_that_did_worse( + baseline, tmp_path +): + # The check that gives a corpus where nothing builds teeth: the built count + # is 0 on both sides, so only the document's own outcome says anything. + baseline.folders["tex"].built = 0 + baseline.folders["tex"].documents = {"tex/tex-sheet.tex": "faulted"} + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + summary = report.folders["tex"] + assert summary.built == summary.recorded == 0 + assert report.failed + assert [one.document for one in summary.regressions] == ["tex/tex-sheet.tex"] + + +@pytest.mark.parametrize( + ("current", "recorded", "expected"), + [ + ("built", "built", False), + ("built", "faulted", False), + ("skipped", "built", True), + # A document read and faulted, then not read at all. + ("skipped", "faulted", True), + ("build refused", "built", True), + ("faulted", "built", True), + ("faulted", "build refused", True), + ("no spec", "faulted", True), + (gate.MISSING, "built", True), + (gate.MISSING, "no spec", False), + ], +) +def test_what_counts_as_worse(current, recorded, expected): + assert gate.worse(current, recorded) is expected + + +def test_the_folder_line_says_what_each_outcome_came_to(baseline, tmp_path): + run(baseline, tmp_path, record=True) + summary = run(baseline, tmp_path).folders["sheets"] + + line = gate.folder_line("sheets", summary) + + assert line.split() == [ + "sheets", + "built", + "2", + "faulted", + "0", + "build", + "refused", + "0", + "skipped", + "0", + "(baseline", + "built", + "2)", + ] + + +def test_the_folder_line_names_an_outcome_of_its_own(baseline, tmp_path): + summary = gate.Summary(built=1, counts={"no spec": 2}, recorded=3) + + line = gate.folder_line("tex", summary) + + assert "no spec 2" in line and "(baseline built 3)" in line + + +def test_the_folder_line_names_each_document_that_did_worse(): + summary = gate.Summary( + built=0, + counts={"faulted": 1}, + recorded=0, + regressions=[ + gate.Regression("UCL_MechEng/Worksheet_3.pdf", "built", "faulted", "KaTeX"), + gate.Regression("UCL_MechEng/Worksheet_4.pdf", "built", gate.MISSING), + ], + ) + + first, second, third = gate.folder_line("UCL_MechEng", summary).splitlines() + + assert first.startswith("UCL_MechEng") + assert second == " worse UCL_MechEng/Worksheet_3.pdf built -> faulted: KaTeX" + assert third == " worse UCL_MechEng/Worksheet_4.pdf built -> missing" + + +def test_the_folder_line_says_where_no_count_is_recorded(baseline, tmp_path): + line = gate.folder_line("tex", gate.Summary(built=1)) + + assert line.endswith("(not recorded)") + + +def test_the_baseline_survives_being_written_and_read(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built = None + path = tmp_path / "baseline.json" + + gate.write_baseline(baseline, path) + read = gate.read_baseline(path) + + assert read == baseline + + +def test_a_run_writes_nothing_under_the_directory_it_was_run_from( + baseline, tmp_path, monkeypatch +): + ran_from = tmp_path / "empty" + ran_from.mkdir() + monkeypatch.chdir(ran_from) + + run(baseline, tmp_path) + + assert list(ran_from.iterdir()) == [] + + +def test_a_run_leaves_the_committed_spec_tree_as_it_found_it(baseline, tmp_path): + # A sweep writes a record of each run beside the spec it read. The specs + # are the repository's, so the gate reads a copy and the records land there. + specs = tmp_path / "specs" + before = sorted(path.relative_to(specs) for path in specs.rglob("*")) + + run(baseline, tmp_path) + + assert sorted(path.relative_to(specs) for path in specs.rglob("*")) == before + + +def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): + # Shared between worktrees on purpose: OCR already fetched for a PDF is not + # fetched again on the next branch. + assert gate.DEFAULT_CACHE_DIR == Path.home() / ".cache" / "in2lambda-agent" + + +REPOSITORY = Path(__file__).resolve().parent.parent + + +@pytest.mark.parametrize( + ("committed", "named"), + [ + # The private corpus, which the workbench check replays, and the + # committed one, which the workflow replays. Two files, because a + # clone has the second and not the first. + ( + "gate-baseline.json", + { + "UCL_MechEng", + "PHYS40002-Mechanics/problem_sheets_and_figures", + "MECH60014_Stress_analysis_3", + }, + ), + ("ci-baseline.json", {"ci-corpus/tex", "ci-corpus/docx", "ci-corpus/pdf"}), + ], +) +def test_a_committed_baseline_names_its_folders_and_their_specs(committed, named): + baseline = gate.read_baseline(REPOSITORY / committed) + + assert set(baseline.folders) == named + for name, folder in baseline.folders.items(): + # The spec each folder replays, at the path the sweep reads it from. + assert (REPOSITORY / baseline.specs / name / SPEC_NAME).is_file() + # Recorded, however low: a count of none is what the gate defends. + assert folder.built is not None + assert folder.documents + + +def test_the_private_baseline_reads_the_corpus_where_it_is(): + baseline = gate.read_baseline(REPOSITORY / "gate-baseline.json") + + # The corpus is not in the repository and is read at its own path. + assert all(folder.root.is_absolute() for folder in baseline.folders.values()) diff --git a/tests/test_ocr.py b/tests/test_ocr.py index c4fedaa..144c7c5 100644 --- a/tests/test_ocr.py +++ b/tests/test_ocr.py @@ -9,7 +9,7 @@ import pytest from in2lambda_agent.mathpix import MathpixError -from in2lambda_agent.ocr import ocr_pdf +from in2lambda_agent.ocr import cached, ocr_pdf from conftest import PNG, FakeMathpix @@ -108,6 +108,19 @@ def test_the_markdown_is_written_as_utf8_under_any_locale(pdf, tmp_path): assert written.decode("utf-8") == "# Ångström ½\n" +def test_a_document_not_in_the_cache_is_no_hit(pdf, tmp_path): + assert cached(pdf, tmp_path / "cache") is None + + +def test_a_cached_document_is_a_hit_without_a_client(pdf, tmp_path): + written = ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix()) + + hit = cached(pdf, tmp_path / "cache") + + assert hit is not None and not hit.fresh + assert (hit.markdown, hit.media) == (written.markdown, written.media) + + def test_each_document_gets_its_own_entry(pdf, tmp_path): other = tmp_path / "other.pdf" other.write_bytes(b"%PDF-1.4 a different sheet") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index dc8fc32..d07639c 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -15,6 +15,7 @@ from in2lambda_agent import package, pair, pipeline from in2lambda_agent.cli import main from in2lambda_agent.model import ModelUnavailable +from in2lambda_agent.ocr import ocr_pdf from in2lambda_agent.package import SpecRejected from in2lambda_agent.review import ReviewError from in2lambda_agent.settings import Settings @@ -1401,6 +1402,26 @@ def test_a_second_run_over_the_same_pdf_uses_the_cache(pdf, tmp_path): assert len(backend.calls) == 1 +def test_a_cached_pdf_runs_with_no_mathpix_credentials(pdf, tmp_path, monkeypatch): + monkeypatch.delenv("MATHPIX_APP_ID", raising=False) + monkeypatch.delenv("MATHPIX_API_KEY", raising=False) + ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix(SOURCE.read_text())) + + # No client at all: the cache is what the run reads, and building one from + # empty settings would raise before it got there. + result = pipeline.run( + pdf, + out_dir=tmp_path / "out", + settings=Settings(), + cache_dir=tmp_path / "cache", + backend=FakeBackend(SPEC), + ) + ocr = next(stage for stage in result.stages if stage.name == "ocr") + + assert ocr.message.startswith("cached ") + assert result.zip_path.exists() + + def test_a_pdf_without_credentials_exits_one_naming_the_variables( pdf, tmp_path, monkeypatch, capsys ):