diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml new file mode 100644 index 0000000..6b23bf6 --- /dev/null +++ b/.github/workflows/gate.yml @@ -0,0 +1,114 @@ +# The merge gate. Every push to main and every pull request runs the tests and +# then replays the three folders of ci-corpus, the committed corpus, which +# ci-baseline.json names. The job is CI's report on a branch and nothing +# merges on it: the workbench merges with `gh pr merge` as soon as its own +# check passes, and that check replays the private corpus, whose folders a +# runner cannot read, on the machine that holds it. +name: gate + +on: + push: + branches: [main] + pull_request: + +jobs: + gate: + name: gate + # Pinned rather than ubuntu-latest: the runner's TeX Live decides what + # xelatex writes, and the PDF's bytes are the OCR cache's key. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + # Pandoc reads every source document, so its version decides the blocks + # a spec selects. ubuntu-24.04 packages pandoc 3.1.3, under which every + # ci-corpus document faults on a block no selector reaches, and the + # tests fail with it. This release is the one the baselines were + # recorded under. + - name: Install pandoc + run: | + curl -fsSL -o "${RUNNER_TEMP}/pandoc.deb" \ + https://github.com/jgm/pandoc/releases/download/3.9.0.2/pandoc-3.9.0.2-1-amd64.deb + sudo dpkg -i "${RUNNER_TEMP}/pandoc.deb" + pandoc --version | head -1 + + # The Dockerfile's package list, which is the one record of what the PDF + # generator's template.latex loads. + - name: Install poppler and TeX Live + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + poppler-utils \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-science \ + texlive-lang-chinese \ + texlive-lang-arabic \ + texlive-bibtex-extra \ + texlive-plain-generic \ + texlive-fonts-recommended \ + lmodern \ + fonts-noto-core \ + fonts-noto-cjk + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install poetry + run: pipx install poetry + + - name: Install the package + run: poetry install --with dev + + # SOURCE_DATE_EPOCH fixes the timestamp xelatex writes into the PDF, so + # the same tex source compiles to the same bytes and the OCR cache key + # does not change between runs. + - name: Compile the CI corpus PDF + run: | + cd ci-corpus/tex + SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ + xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex + rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log + + # Keyed by the PDF's bytes, which is also how ocr_pdf names its entry + # inside the directory. A restored entry for another PDF is unused. + - name: Restore the OCR cache + uses: actions/cache@v4 + with: + path: ~/.cache/in2lambda-agent + key: ocr-${{ hashFiles('ci-corpus/pdf/*.pdf') }} + restore-keys: ocr- + + - name: Tests + run: poetry run pytest -q + + - name: Gate + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: poetry run in2lambda-agent gate ci-baseline.json + + # What this run would record, so that a baseline change is committed from + # what CI saw. Written whether or not the gate passed. The file is + # recorded where it is checked out, because a baseline names its specs + # relative to its own directory and a copy under RUNNER_TEMP would find + # none; the job throws the checkout away. + - name: Record what this run did + if: always() + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: poetry run in2lambda-agent gate --record ci-baseline.json + + - name: Upload it + if: always() + uses: actions/upload-artifact@v4 + with: + name: recorded-baseline + path: ci-baseline.json diff --git a/.gitignore b/.gitignore index 45e2c81..8a51ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,12 @@ __pycache__/ *.egg-info/ .venv/ dist/ +# The specs for ExampleContents quote the headings of private documents, and +# gate-baseline.json beside them records those documents' file names and the +# absolute path of the corpus on one machine. +corpus-specs/ # in2lambda's KaTeX converter writes this into the working directory on import. log +# The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex +# in the workflow, so the repository holds the tex source and not the output. +ci-corpus/pdf/*.pdf diff --git a/README.md b/README.md index 2f30217..0d85aa1 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ poetry run in2lambda-agent corpus ExampleContents --suffix tex --suffix md In full: ```sh -poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] +poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] [--cache DIR] ``` `ROOT` is the corpus directory and each `PATH` a folder under it to run, defaulting to @@ -285,6 +285,117 @@ review, rejections where its value comes from. One document that fails is one row and not the end of the sweep, and a set whose folder cannot be copied is a row for each of its documents. +`--cache` is where the OCR of each PDF is kept. It defaults to `./.in2lambda-agent`, +the directory `run` caches into, so a sweep over PDFs that `run` has already converted +makes no Mathpix call and needs no Mathpix credentials. + +## Gate + +Nothing merges without a replay over real documents. `gate` reruns the saved specs +over the folders a baseline file names, and compares what each folder did this run +with what the baseline records: + +```sh +poetry run in2lambda-agent gate BASELINE [--record] [--cache DIR] [--work DIR] +``` + +Every run is `corpus --replay`, so no model call is made. The command prints one line +per folder, and exits 1 when a folder builds fewer documents than the baseline records +or when a single document does worse than the baseline records it doing. The second +check is what a baseline of no builds rests on: a corpus where every document faults +still reports the document that stops being read. + +``` +work /tmp/in2lambda-agent-gate-3f1a +tex built 2 faulted 0 build refused 0 skipped 0 no spec 2 (baseline built 4) + worse tex/sheet-3.tex built -> no spec: replay: no model call is allowed +``` + +The gate reads `specs`, and a folder's `root` where `root` is relative, from the +directory `BASELINE` is in, so the command gives the same run from any directory. + +There are two corpora and a baseline for each: + +| Baseline | Corpus | Run by | +| --- | --- | --- | +| `corpus-specs/gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | +| `ci-baseline.json` | the three folders of `ci-corpus` | `.github/workflows/gate.yml` | + +The repository holds `ci-baseline.json` and the specs it names, under +`ci-corpus/specs/`, because `ci-corpus` is synthetic. The repository holds neither the +specs for `ExampleContents` nor the baseline that names them: the specs quote the +headings of private documents, and the baseline records those documents' file names and +the path of the corpus on one machine. `.gitignore` lists `corpus-specs/`, and +`gate-baseline.json` sits in that directory beside the specs it reads, with `"specs": +"."`. + +`ci-corpus` is synthetic, so the repository holds its documents — every one but the +PDF, which xelatex compiles from `ci-corpus/tex/sheet-1.tex`. Run the command the +workflow runs before `gate ci-baseline.json`, because the PDF's bytes are the key the +OCR cache reads under: + +```sh +cd ci-corpus/tex +SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ + xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex +rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log +``` + +Without the PDF the `pdf` folder holds no document, builds 0 against a recorded 1, +and the gate exits 1. + +`ExampleContents` is a set of private documents and is never in the repository: the +gate reads it at the absolute `root` that `gate-baseline.json` gives, which is a path +on the machine the check runs on. + +Each folder's `root` and `suffixes` are written by hand. `built`, the count of documents +that built, and `documents`, the outcome of each single document, are what `--record` +writes. A folder the file records no `built` for passes on any count, and its line reads +`(not recorded)`. A change to a recorded count or outcome belongs in a pull request that +says why the count or the outcome changed. + +Over `ExampleContents` today, every document replays to `faulted` and the baseline +records 0 built for all three folders: pandoc's line wrapping is reported as a math +delimiter error, and each document needs a fixing round that a replay does not run. The +recorded outcomes are what the gate defends until a later ticket raises the count. + +`--cache` defaults to `~/.cache/in2lambda-agent`, which is outside every worktree, +because the gate runs in a worktree of its own: a PDF converted on one branch is +converted again on the next if the cache sits in the branch's directory. `--work` +defaults to a new directory under the system temp +directory, which the gate does not delete: read the +drafts of a folder that failed there. The gate also copies the spec tree into the work +directory and replays the copy, because a sweep appends a record of each run beside the +spec it reads. Neither directory is inside the repository, so `git status` after a gate +run reports no new file. + +`.github/workflows/gate.yml` runs pytest and then `gate ci-baseline.json` on every push +to `main` and every pull request. Mathpix reads `ci-corpus/pdf/sheet-1.pdf` once and the +workflow stores the markdown in the Actions cache under the PDF's hash. The job needs +two repository secrets, `MATHPIX_APP_ID` and `MATHPIX_API_KEY`. + +A pull request from a fork is given neither secret. `actions/cache` restores the cache +of the base branch for a fork, and the cached markdown is what the `pdf` folder then +replays, so the job passes without the secrets. If the cache is empty — the PDF's bytes +changed, or GitHub evicted the entry — Mathpix cannot be called and the `pdf` folder +builds 0 against a recorded 1, so the job fails. Push that branch to a branch of this +repository, where the secrets are read, and the job runs Mathpix once. + +The job is CI's report on a branch and no merge waits for it. The workbench merges with +`gh pr merge` as soon as its own check passes, and `gh pr merge` cannot wait for a +GitHub check, so requiring the job on `main` would refuse every merge the workbench +makes. + +The workbench check runs the gate over `ExampleContents`, which is the larger corpus of +the two: + +```sh +poetry install -q --with dev && poetry run pytest -q && poetry run in2lambda-agent gate /Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/corpus-specs/gate-baseline.json +``` + +The path is absolute because the check runs in a worktree and the worktree holds +neither the baseline nor the specs it names. + ## Docker The image carries pandoc, a TeX Live whose xelatex runs the PDF generator's diff --git a/ci-baseline.json b/ci-baseline.json new file mode 100644 index 0000000..3311388 --- /dev/null +++ b/ci-baseline.json @@ -0,0 +1,37 @@ +{ + "specs": "ci-corpus/specs", + "folders": { + "tex": { + "root": "ci-corpus", + "suffixes": [ + "tex" + ], + "built": 3, + "documents": { + "tex/sheet-1.tex": "built", + "tex/sheet-2.tex": "built", + "tex/sheet-3.tex": "built" + } + }, + "docx": { + "root": "ci-corpus", + "suffixes": [ + "docx" + ], + "built": 1, + "documents": { + "docx/sheet.docx": "built" + } + }, + "pdf": { + "root": "ci-corpus", + "suffixes": [ + "pdf" + ], + "built": 1, + "documents": { + "pdf/sheet-1.pdf": "built" + } + } + } +} diff --git a/ci-corpus/docx/sheet.docx b/ci-corpus/docx/sheet.docx new file mode 100644 index 0000000..137c8d0 Binary files /dev/null and b/ci-corpus/docx/sheet.docx differ diff --git a/ci-corpus/pdf/.gitkeep b/ci-corpus/pdf/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ci-corpus/specs/docx/in2lambda-spec.yaml b/ci-corpus/specs/docx/in2lambda-spec.yaml new file mode 100644 index 0000000..97c98ce --- /dev/null +++ b/ci-corpus/specs/docx/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Header +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/ci-corpus/specs/pdf/in2lambda-spec.yaml b/ci-corpus/specs/pdf/in2lambda-spec.yaml new file mode 100644 index 0000000..3e4997f --- /dev/null +++ b/ci-corpus/specs/pdf/in2lambda-spec.yaml @@ -0,0 +1,10 @@ +# The spec for the pdf set, which is sheet-1.tex compiled by xelatex and read +# back by Mathpix. The OCR writes the sheet's shape back: the question as a +# paragraph, its parts as a numbered list, and the solutions under the +# `Solutions` heading. The selectors are the tex set's for that reason. +ignore: Header +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/ci-corpus/specs/tex/in2lambda-spec.yaml b/ci-corpus/specs/tex/in2lambda-spec.yaml new file mode 100644 index 0000000..c95f041 --- /dev/null +++ b/ci-corpus/specs/tex/in2lambda-spec.yaml @@ -0,0 +1,10 @@ +# The spec for the tex set: each question a paragraph, its parts the items of +# the list under it, and its solutions under a `Solutions` heading — in the +# same file for sheet-1 and sheet-3, and in sheet-2-solutions.tex for sheet-2. +# A solution begins with its label, `1(a)`, so no solution matches `question`. +ignore: Header +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/ci-corpus/tex/figures/ball.png b/ci-corpus/tex/figures/ball.png new file mode 100644 index 0000000..f37764b Binary files /dev/null and b/ci-corpus/tex/figures/ball.png differ diff --git a/ci-corpus/tex/sheet-1.tex b/ci-corpus/tex/sheet-1.tex new file mode 100644 index 0000000..d62b66d --- /dev/null +++ b/ci-corpus/tex/sheet-1.tex @@ -0,0 +1,36 @@ +% Synthetic, in the shape of the corpus's problem sheets: a starred section +% heading, each question a paragraph with its parts as a lettered list, and a +% solutions section at the end. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 1: Kinematics} + +A ball is thrown straight up at $20\,\mathrm{m/s}$. + +\begin{enumerate} +\item Find the greatest height it reaches. +\item Find its time of flight. +\end{enumerate} + +A block of mass $m$ rests on a slope of angle $\theta$. + +\begin{enumerate} +\item Name the three forces acting on the block. +\item Find the least coefficient of friction that holds it still. +\end{enumerate} + +\section*{Solutions} + +1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ + +1(b) $t = 2v/g = 4.08\,\mathrm{s}$ + +2(a) Weight, the normal reaction, and friction along the slope. + +2(b) $\mu = \tan\theta$ + +\end{document} diff --git a/ci-corpus/tex/sheet-2-solutions.tex b/ci-corpus/tex/sheet-2-solutions.tex new file mode 100644 index 0000000..9ba70bf --- /dev/null +++ b/ci-corpus/tex/sheet-2-solutions.tex @@ -0,0 +1,24 @@ +% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the +% shape that first failed a sweep. The name is what pairs the two: `pair` reads +% the stem before the `-solutions` ending, finds sheet-2.tex beside it, and the +% run freezes the questions first and these second. The spec reads every block +% under the `Solutions` heading as a solution, so this file adds no question. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Answers} + +\subsection*{Solutions} + +1(a) Take the divergence term by term; each pair cancels. + +1(b) $B = \mu_0 m / 2\pi z^3$ + +2(a) $r = mv / qB$ + +2(b) $T = 2\pi m / qB$ + +\end{document} diff --git a/ci-corpus/tex/sheet-2.tex b/ci-corpus/tex/sheet-2.tex new file mode 100644 index 0000000..aee9b53 --- /dev/null +++ b/ci-corpus/tex/sheet-2.tex @@ -0,0 +1,26 @@ +% Synthetic: questions with lettered parts and no solutions on the sheet. The +% solutions are in sheet-2-solutions.tex, which `pair` freezes as the draft's +% second source. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Fields} + +A dipole sits at the origin. + +\begin{enumerate} +\item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. +\item Find the field on the axis at a distance $z$. +\end{enumerate} + +A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. + +\begin{enumerate} +\item Find the radius of its circular path. +\item Find the period of the motion. +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-3.tex b/ci-corpus/tex/sheet-3.tex new file mode 100644 index 0000000..e19e16d --- /dev/null +++ b/ci-corpus/tex/sheet-3.tex @@ -0,0 +1,38 @@ +% Synthetic: a sheet whose first question includes a figure, which is the shape +% that first failed a sweep on the path of an image. Nothing here is copied +% from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 3: Statics} + +The beam below carries a load $W$ at its midpoint. +\includegraphics[width=0.2\textwidth]{figures/ball.png} + +\begin{enumerate} +\item Find the reaction at each support. +\item Find the bending moment at the midpoint. +\end{enumerate} + +A ladder of mass $m$ leans against a smooth wall. + +\begin{enumerate} +\item Draw the forces acting on the ladder. +\item Find the least angle at which it does not slip. +\end{enumerate} + +\section*{Solutions} + +1(a) $W/2$ at each support. + +1(b) $M = WL/4$ + +2(a) The weight at the centre, the normal reaction at the wall, and the normal +reaction and friction at the floor. + +2(b) $\tan\alpha = 1/2\mu$ + +\end{document} diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index 7664e72..f83bd6b 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -3,10 +3,11 @@ import argparse import getpass import sys +import tempfile from pathlib import Path from typing import Optional, Sequence -from in2lambda_agent import compare, corpus, pipeline +from in2lambda_agent import compare, corpus, gate, pipeline from in2lambda_agent.mathpix import MathpixClient, MathpixError from in2lambda_agent.model import ModelUnavailable, choose_backend from in2lambda_agent.ocr import ocr_pdf @@ -68,8 +69,8 @@ def build_parser() -> argparse.ArgumentParser: """The command line as the design spec describes it. Returns: - A parser with the `run`, `review`, `corpus`, `compare` and `ui` - subcommands. + A parser with the `run`, `review`, `corpus`, `gate`, `compare` and + `ui` subcommands. """ parser = argparse.ArgumentParser( prog="in2lambda-agent", @@ -206,6 +207,37 @@ def build_parser() -> argparse.ArgumentParser: default=corpus.DEFAULT_SPEC_DIR, help="The tree the sets' specs are kept in, mirroring the corpus.", ) + sweep.add_argument( + "--cache", + type=Path, + default=pipeline.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, so a sweep pointed at a cache " + "another run filled converts nothing.", + ) + + check = subcommands.add_parser( + "gate", help="Replay the corpus the baseline names and check it against it." + ) + check.add_argument("baseline", type=Path, help="The committed baseline file.") + check.add_argument( + "--record", + action="store_true", + help="Write this run's counts to the baseline instead of checking them.", + ) + check.add_argument( + "--cache", + type=Path, + default=gate.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, shared between worktrees so " + "that a conversion is paid for once.", + ) + check.add_argument( + "--work", + type=Path, + default=None, + help="Where the folders are copied to be run, under the system temp " + "directory by default so the check writes nothing where it was run.", + ) against = subcommands.add_parser( "compare", help="Check a PDF's OCR against the pages it came from." @@ -265,6 +297,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: specs=args.specs, replay=args.replay, rounds=args.rounds, + cache=args.cache, settings=load_settings(), ) print(f"{len(rows)} documents, written to {args.results}") @@ -273,6 +306,27 @@ def main(argv: Optional[Sequence[str]] = None) -> int: succeeded = {"built", "skipped"} return 0 if rows and all(row.outcome in succeeded for row in rows) else 1 + if args.command == "gate": + baseline = gate.read_baseline(args.baseline) + # The directory is printed and is not deleted, so that the drafts of a + # folder that failed can be read after the run. + work = args.work or Path(tempfile.mkdtemp(prefix="in2lambda-agent-gate-")) + print(f"work {work}") + report = gate.run( + baseline, + record=args.record, + cache=args.cache, + work=work, + settings=load_settings(), + ) + for name, summary in report.folders.items(): + print(gate.folder_line(name, summary)) + if args.record: + gate.write_baseline(baseline, args.baseline) + print(f"recorded {args.baseline}") + return 0 + return 1 if report.failed else 0 + if args.command == "compare": settings = load_settings() try: diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 494d19c..881036b 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -253,6 +253,7 @@ def run_one( settings: Settings, rounds: int = 3, replay: bool = False, + cache: Path = pipeline.DEFAULT_CACHE_DIR, backend: Optional[Backend] = None, ) -> Row: """Runs the pipeline over one document and reads the row off what it did. @@ -269,6 +270,7 @@ def run_one( settings: The environment the run has available. rounds: The round limit, ignored in a replay, which can run none. replay: Run the saved spec and nothing else, making no model call. + cache: Where the OCR of each PDF is kept. backend: The backend to write a spec with, chosen from the settings if absent. @@ -289,6 +291,7 @@ def run_one( # report: the row then says what the saved spec left rather than # that a call could not be made. rounds=0 if replay else rounds, + cache_dir=cache, backend=NoModel() if replay else backend, ) except ModelUnavailable as error: @@ -366,6 +369,7 @@ def sweep( specs: Path = DEFAULT_SPEC_DIR, replay: bool = False, rounds: int = 3, + cache: Path = pipeline.DEFAULT_CACHE_DIR, settings: Optional[Settings] = None, backend: Optional[Backend] = None, ) -> list[Row]: @@ -380,6 +384,8 @@ def sweep( specs: The tree the sets' specs are kept in, mirroring the corpus. replay: Run the saved specs and nothing else, making no model call. rounds: The round limit each run is given. + cache: Where the OCR of each PDF is kept, so that a sweep pointed at a + cache another run filled converts nothing. settings: The environment the runs have available. backend: The backend to write the specs with, chosen from the settings if absent. @@ -475,6 +481,7 @@ def sweep( settings=settings, rounds=rounds, replay=replay, + cache=cache, backend=backend, ) print(f"{row.outcome:<20} {row.source}") diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py new file mode 100644 index 0000000..e55e30e --- /dev/null +++ b/in2lambda_agent/gate.py @@ -0,0 +1,326 @@ +"""The merge gate: a replay over real documents, compared with a baseline. + +Every ticket before this one was tested on synthetic fixtures. The first sweep +over a folder of real PDFs failed on four faults no fixture had: a list-valued +selector, a file of solutions with no questions, an image path, and a wrapped +line. A merge now requires an end-to-end run over real documents. + +The run is `corpus.sweep(replay=True)` over the specs the repository keeps, so +it makes no model call. The baseline file names the folders to run and records +both how many documents each folder built and what each single document did. +The gate fails when a folder builds fewer documents than the baseline records, +or when any one document does worse than it is recorded as doing. The second +check is what gives a baseline of no builds at all teeth: a folder where every +document faults still notices the day one of them stops being read. A folder +the baseline records no count for passes on any count, which is how a folder is +added to the gate before it replays to a build worth defending. A change to a +recorded count belongs in a pull request that states why the count changed. + +A baseline names its specs, and a corpus of its own, relative to the directory +the file is in. `ci-baseline.json` is at the root of the repository and names +`ci-corpus/specs`; the baseline for the private corpus is at +`corpus-specs/gate-baseline.json`, beside the specs it names, and the +repository holds neither file. +""" + +import json +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from in2lambda_agent import corpus +from in2lambda_agent.settings import Settings + +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "in2lambda-agent" +"""Where the gate reads the OCR of each PDF. The directory is outside every +worktree, so a PDF converted on one branch is not converted again on the next. +`corpus` on its own keeps its cache under the directory the user ran from.""" + +RANK = {"built": 0, "build refused": 1, "faulted": 2, "skipped": 3} +"""How bad each outcome is. `skipped` is the worst of them: a document that +faults was at least read, and one that is skipped was not. Anything else is a +document that did not run either — a spec gone, a file unreadable, a document +the sweep no longer finds — and ranks with `skipped`.""" + +MISSING = "missing" +"""What a recorded document the sweep no longer finds is compared as.""" + + +def worse(current: str, recorded: str) -> bool: + """Whether a document did worse this run than the baseline records. + + A document that stops being read is a regression whatever it did before, + which is the case a baseline of nothing but faults rests on: widen what + counts as a solutions file and a document leaves the sweep as `skipped` + without a single count changing. + + Args: + current: What the document did this run. + recorded: What the baseline records it doing. + + Returns: + Whether it is a regression. + """ + return RANK.get(current, 3) > RANK.get(recorded, 3) + + +@dataclass +class Folder: + """One folder of a corpus, as the baseline holds it. + + Attributes: + root: The corpus directory the folder is under, read relative to the + directory the baseline file is in. `ci-corpus` for the committed + corpus; an absolute path for a corpus outside the repository. + suffixes: The file suffixes that are documents in the folder: `pdf` for + a folder of scans, `tex` or `docx` for sources. + built: How many documents built when the baseline was recorded, or None + where no count is recorded. + documents: What each document did when the baseline was recorded, by + its path under the corpus root. `root` and `suffixes` are written + by hand; `built` and `documents` are what `--record` writes. + """ + + root: Path + suffixes: list[str] + built: Optional[int] = None + documents: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Regression: + """One document that did worse this run than the baseline records. + + Attributes: + document: Its path under the corpus root, as the baseline names it. + recorded: What the baseline records it doing. + current: What it did this run, or `missing` where the sweep no longer + finds it. + reason: What the sweep said about it, where it said anything. + """ + + document: str + recorded: str + current: str + reason: str = "" + + +@dataclass +class Baseline: + """The file the gate compares a sweep against. + + Attributes: + specs: The tree the folders' specs are kept in, read relative to the + directory the baseline file is in. Folder `A/B` reads + `/A/B/in2lambda-spec.yaml`. + folders: The folders to run, by their path under their own root. + directory: The directory the baseline file was read from. `specs` and + a relative `root` are read from there, so the command gives the + same run whichever directory it is run in. The paths themselves are + held as they are written, so `--record` writes them back unchanged. + """ + + specs: Path + folders: dict[str, Folder] + directory: Path = Path(".") + + +@dataclass +class Summary: + """What one folder did this run, beside what the baseline records. + + Attributes: + built: How many documents built. + counts: How many did each other thing, by outcome. + recorded: How many built when the baseline was recorded, or None where + no count is recorded. + regressions: The documents that did worse than the baseline records. + """ + + built: int = 0 + counts: dict[str, int] = field(default_factory=dict) + recorded: Optional[int] = None + regressions: list[Regression] = field(default_factory=list) + + @property + def failed(self) -> bool: + """Whether this folder fails the gate.""" + if self.recorded is not None and self.built < self.recorded: + return True + return bool(self.regressions) + + +@dataclass +class Report: + """The whole run: one summary per folder, in the baseline's order.""" + + folders: dict[str, Summary] = field(default_factory=dict) + + @property + def failed(self) -> bool: + """Whether any folder fails the gate.""" + return any(one.failed for one in self.folders.values()) + + +def read_baseline(path: Path) -> Baseline: + """Reads a baseline file. + + Args: + path: The JSON file. The paths it names are read from the directory it + is in. + + Returns: + The baseline. + """ + written = json.loads(Path(path).read_text(encoding="utf-8")) + return Baseline( + directory=Path(path).parent, + specs=Path(written["specs"]), + folders={ + name: Folder( + root=Path(one["root"]), + suffixes=list(one["suffixes"]), + built=one.get("built"), + documents=dict(one.get("documents", {})), + ) + for name, one in written["folders"].items() + }, + ) + + +def write_baseline(baseline: Baseline, path: Path) -> None: + """Writes the baseline back, counts and all. + + Args: + baseline: What to write. + path: The JSON file, which is overwritten. + """ + written = { + "specs": baseline.specs.as_posix(), + "folders": { + name: { + "root": one.root.as_posix(), + "suffixes": one.suffixes, + "built": one.built, + "documents": one.documents, + } + for name, one in baseline.folders.items() + }, + } + Path(path).write_text(json.dumps(written, indent=2) + "\n", encoding="utf-8") + + +def run( + baseline: Baseline, + *, + record: bool = False, + cache: Path, + work: Path, + settings: Optional[Settings] = None, +) -> Report: + """Replays every folder the baseline names and checks what each one did. + + Nothing is written into the repository or into a corpus: each folder is + swept into its own directory under `work`, and the table is written there. + + Args: + baseline: The folders to run and what to compare against. Its `specs` + and each relative `root` are read from the directory it was read + from. In record mode this run's counts and outcomes replace them. + record: Take this run as the new baseline rather than checking it. + cache: Where the OCR of each PDF is kept. The directory is shared + between worktrees, so Mathpix converts each PDF once. + work: Where the folders are copied to be run. + settings: The environment the runs have available. + + Returns: + One summary per folder. + """ + work = Path(work) + # A sweep writes a record of each run beside the spec it read, and the + # record of a gate run is nobody's: it would land in the repository, under + # the specs the gate exists to replay. So the specs are copied under `work` + # and read from there. + specs = work / "specs" + # Fresh each run, so that a spec taken out of the tree is gone from the + # copy the sweep reads rather than left over from the run before. + shutil.rmtree(specs, ignore_errors=True) + # Joining an absolute path to the baseline's directory returns the absolute + # path, so a private corpus named by its path on one machine is unchanged. + written = baseline.directory / baseline.specs + if written.is_dir(): + shutil.copytree(written, specs) + report = Report() + for name, folder in baseline.folders.items(): + rows = corpus.sweep( + baseline.directory / folder.root, + paths=[Path(name)], + suffixes=folder.suffixes, + results=work / name / "results.csv", + work=work / "work", + specs=specs, + replay=True, + cache=cache, + settings=settings, + ) + summary = Summary(recorded=folder.built) + outcomes = {row.source: row.outcome for row in rows} + reasons = {row.source: row.reason for row in rows} + for outcome in outcomes.values(): + if outcome == "built": + summary.built += 1 + else: + summary.counts[outcome] = summary.counts.get(outcome, 0) + 1 + if record: + folder.built = summary.built + folder.documents = outcomes + # A recording run reports what this sweep did, so it fails nothing. + summary.recorded = summary.built + else: + for document, was in folder.documents.items(): + now = outcomes.get(document, MISSING) + if worse(now, was): + summary.regressions.append( + Regression(document, was, now, reasons.get(document, "")) + ) + report.folders[name] = summary + return report + + +def folder_line(name: str, summary: Summary) -> str: + """The gate's line for a folder: what it did, beside what is recorded. + + A folder whose documents all did as well as recorded is one line. Each + document that did worse adds an indented line of its own beneath it, so + that a failure names the document rather than only the count. + + Args: + name: The folder, as the baseline names it. + summary: What it did. + + Returns: + The line, and a line per regression beneath it. + """ + counts = " ".join( + f"{outcome} {summary.counts.get(outcome, 0)}" + for outcome in ("faulted", "build refused", "skipped") + ) + other = sorted(set(summary.counts) - {"faulted", "build refused", "skipped"}) + recorded = ( + "not recorded" + if summary.recorded is None + else f"baseline built {summary.recorded}" + ) + return "\n".join( + [ + f"{name:<40} built {summary.built} {counts}" + + "".join(f" {outcome} {summary.counts[outcome]}" for outcome in other) + + f" ({recorded})", + *( + f" worse {one.document} {one.recorded} -> {one.current}" + + (f": {one.reason}" if one.reason else "") + for one in summary.regressions + ), + ] + ) diff --git a/in2lambda_agent/ocr.py b/in2lambda_agent/ocr.py index beee014..c4fb7dc 100644 --- a/in2lambda_agent/ocr.py +++ b/in2lambda_agent/ocr.py @@ -9,6 +9,7 @@ import shutil from dataclasses import dataclass from pathlib import Path +from typing import Optional from in2lambda_agent.mathpix import MathpixClient @@ -29,6 +30,27 @@ class OcrResult: fresh: bool +def cached(pdf: Path, cache_dir: Path) -> Optional[OcrResult]: + """The conversion already in the cache, or None where there is none. + + Asked before a client is built, so that a document whose OCR was fetched + once runs again with no Mathpix credentials at all: a worktree, or a CI job + on a fork, has the cache and not the account. + + Args: + pdf: The PDF whose conversion is wanted. + cache_dir: Holds one entry per document, named by the PDF's hash. + + Returns: + Where the markdown and its media folder are, or None. + """ + entry = Path(cache_dir) / _hash(pdf) + markdown = entry / SOURCE_NAME + if not markdown.exists(): + return None + return OcrResult(markdown, entry / MEDIA_NAME, fresh=False) + + def ocr_pdf( pdf: Path, *, cache_dir: Path, client: MathpixClient, fresh: bool = False ) -> OcrResult: @@ -46,11 +68,12 @@ def ocr_pdf( Raises: MathpixError: If the conversion fails; the entry is left absent. """ + if not fresh and (hit := cached(pdf, cache_dir)) is not None: + return hit + entry = Path(cache_dir) / _hash(pdf) markdown = entry / SOURCE_NAME media = entry / MEDIA_NAME - if markdown.exists() and not fresh: - return OcrResult(markdown, media, fresh=False) # A fresh pass restarts the pipeline for this document, so the whole entry # goes: anything a later stage comes to keep beside source.md — a draft, a diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index fe721d1..d57f05f 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -36,7 +36,7 @@ from in2lambda_agent.fix import RoundResult, fix_round, summary, unrepaired from in2lambda_agent.mathpix import MathpixClient from in2lambda_agent.model import Backend, ModelUnavailable, Usage, choose_backend -from in2lambda_agent.ocr import MEDIA_NAME, ocr_pdf +from in2lambda_agent.ocr import MEDIA_NAME, cached, ocr_pdf from in2lambda_agent.review import RECORD, Question, Review, choose from in2lambda_agent.settings import Settings from in2lambda_agent.spec import RECORD_NAME, record_run, spec_path, write_spec @@ -155,7 +155,8 @@ def run( Raises: MathpixError: If a PDF cannot be converted, MissingCredentials among - them when the run has no Mathpix credentials. + them when a conversion is needed and the run has no Mathpix + credentials. A PDF already in the cache needs none. ModelUnavailable: If a spec must be written and no backend can run. BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. @@ -521,12 +522,18 @@ def _markdown( the OCR stage says about it. Raises: - MathpixError: If the PDF cannot be converted. + MathpixError: If the PDF cannot be converted, MissingCredentials among + them. A PDF already in the cache is not converted and needs none. """ if document.suffix.lower() != ".pdf": return document, None, f"not needed for {document.name}" - client = mathpix or MathpixClient.from_settings(settings) - ocr = ocr_pdf(document, cache_dir=cache_dir, client=client, fresh=fresh) + # The cache is asked before the client is built: a document converted once + # runs again with no credentials, which is what lets a worktree or a fork's + # CI job replay a corpus of PDFs it cannot pay for. + ocr = None if fresh else cached(document, cache_dir) + if ocr is None: + client = mathpix or MathpixClient.from_settings(settings) + ocr = ocr_pdf(document, cache_dir=cache_dir, client=client, fresh=fresh) # A fresh pass is a restart: every stage below reads the new markdown. if ocr.fresh: return ocr.markdown, ocr.media, f"fresh pass, restarting from {ocr.markdown}" diff --git a/tests/test_cli.py b/tests/test_cli.py index e495e44..77805be 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,7 @@ import pytest from conftest import FakeMathpix -from in2lambda_agent import cli, compare, corpus, pipeline +from in2lambda_agent import cli, compare, corpus, gate, pipeline from in2lambda_agent.cli import build_parser, main, reviewer_name from in2lambda_agent.model import Usage from in2lambda_agent.settings import Settings @@ -91,6 +91,7 @@ def test_corpus_defaults(): assert args.results == Path("results.csv") assert args.work == Path(".in2lambda-agent/corpus") assert args.specs == Path("corpus-specs") + assert args.cache == Path(".in2lambda-agent") def test_corpus_every_option(): @@ -113,6 +114,8 @@ def test_corpus_every_option(): "working", "--specs", "saved", + "--cache", + "cached", ] ) @@ -123,6 +126,104 @@ def test_corpus_every_option(): assert args.results == Path("sweep.csv") assert args.work == Path("working") assert args.specs == Path("saved") + assert args.cache == Path("cached") + + +def test_the_corpus_cache_is_handed_to_the_sweep(monkeypatch): + given = {} + + def record(*args, **kwargs): + given.update(kwargs) + return [] + + monkeypatch.setattr(corpus, "sweep", record) + + main(["corpus", "ExampleContents", "--cache", "cached"]) + + assert given["cache"] == Path("cached") + + +def test_gate_defaults(): + args = build_parser().parse_args(["gate", "gate-baseline.json"]) + + assert args.command == "gate" + assert args.baseline == Path("gate-baseline.json") + assert args.record is False + assert args.cache == Path.home() / ".cache" / "in2lambda-agent" + # Chosen when the command runs, so that two runs do not share a directory. + assert args.work is None + + +def test_gate_every_option(): + args = build_parser().parse_args( + [ + "gate", + "saved.json", + "--record", + "--cache", + "cached", + "--work", + "working", + ] + ) + + assert args.record is True + assert args.cache == Path("cached") + assert args.work == Path("working") + + +def test_a_gate_that_passes_exits_zero(tmp_path, monkeypatch, capsys): + path = written_baseline(tmp_path) + report = gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + monkeypatch.setattr(gate, "run", lambda *args, **kwargs: report) + + code = main(["gate", str(path)]) + + assert code == 0 + assert "tex" in capsys.readouterr().out + + +def test_a_gate_that_fails_exits_one_and_says_what_the_folder_built( + tmp_path, monkeypatch, capsys +): + path = written_baseline(tmp_path) + summary = gate.Summary(built=1, counts={"faulted": 1}, recorded=2) + monkeypatch.setattr( + gate, "run", lambda *args, **kwargs: gate.Report(folders={"tex": summary}) + ) + + code = main(["gate", str(path)]) + + assert code == 1 + assert "tex" in capsys.readouterr().out + + +def test_recording_writes_the_baseline_and_exits_zero(tmp_path, monkeypatch): + path = written_baseline(tmp_path) + + def record(baseline, **kwargs): + baseline.folders["tex"].built = 2 + return gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + + monkeypatch.setattr(gate, "run", record) + + code = main(["gate", str(path), "--record"]) + + assert code == 0 + assert gate.read_baseline(path).folders["tex"].built == 2 + + +def written_baseline(tmp_path): + """A baseline file on disk, for the gate command to read.""" + path = tmp_path / "baseline.json" + gate.write_baseline( + gate.Baseline( + specs=Path("corpus-specs"), + folders={"tex": gate.Folder(root=tmp_path / "corpus", suffixes=["tex"])}, + ), + path, + ) + return path def test_compare_defaults(): diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 38d2f46..2124397 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -99,6 +99,22 @@ def test_the_row_says_what_the_spec_made_of_the_document(root, tmp_path): assert sheet.model_seconds == 0.0 and sheet.wall_seconds > 0 +def test_the_sweeps_cache_is_where_each_run_looks_for_its_ocr( + root, tmp_path, monkeypatch +): + given = [] + + def record(source, **kwargs): + given.append(kwargs["cache_dir"]) + raise RuntimeError("as far as this goes") + + monkeypatch.setattr(pipeline, "run", record) + + rows = sweep(root, tmp_path, cache=tmp_path / "shared") + + assert given == [tmp_path / "shared"] * len(rows) + + def test_a_saved_spec_and_its_source_replay_with_no_model_call(root, tmp_path): for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): saved = tmp_path / "specs" / folder / SPEC_NAME diff --git a/tests/test_docs.py b/tests/test_docs.py index 35b43e0..467318a 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -13,6 +13,7 @@ ROOT = Path(__file__).resolve().parent.parent README = (ROOT / "README.md").read_text(encoding="utf-8") HOW_IT_WORKS = (ROOT / "docs" / "how-it-works.md").read_text(encoding="utf-8") +WORKFLOW = (ROOT / ".github" / "workflows" / "gate.yml").read_text(encoding="utf-8") def _options(parser: argparse.ArgumentParser) -> set[str]: @@ -60,3 +61,47 @@ def test_how_it_works_names_every_stage(): assert len(names) == 9 missing = [one for one in sorted(names) if f"`{one}`" not in HOW_IT_WORKS] assert not missing + + +def test_the_readme_compiles_the_ci_corpus_pdf_as_the_workflow_does(): + # The repository does not hold ci-corpus/pdf, so the reader compiles the + # PDF before the gate reads it. The README and the workflow give the same + # command, because another command writes other bytes, and the PDF's bytes + # are the key the OCR cache reads under. + for line in ( + "SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1", + "xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex", + ): + assert line in WORKFLOW + assert line in README + + +def test_no_github_check_is_required_on_main(): + # The workbench merges with `gh pr merge` as soon as its own check passes, + # and `gh pr merge` cannot wait for a GitHub check, so a required check + # refuses every merge the workbench makes. The workflow is CI's report. + for text in (README, WORKFLOW): + assert "required_status_checks" not in text + assert "branch-protection" not in text + + +def test_the_readme_names_the_private_baseline_by_an_absolute_path(): + # The workbench check runs in a worktree, and the worktree holds neither + # the baseline for ExampleContents nor the specs it names. + check = next( + line + for line in README.splitlines() + if "in2lambda-agent gate " in line and "gate-baseline.json" in line + ) + path = check.split("in2lambda-agent gate ", 1)[1].split()[0] + assert path.startswith("/") + assert path.endswith("/corpus-specs/gate-baseline.json") + + +def test_the_workflow_installs_a_pandoc_of_its_own(): + # ubuntu-24.04 packages pandoc 3.1.3, under which every ci-corpus document + # faults on a block no selector reaches. The job installs a pinned + # release, the one the baselines were recorded under. + apt = WORKFLOW.split("apt-get install", 1)[1].split("\n\n", 1)[0] + assert "pandoc" not in apt + assert "https://github.com/jgm/pandoc/releases/download/" in WORKFLOW diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..7e2c2ca --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,382 @@ +"""The merge gate: a replay over a corpus, checked against a recorded baseline.""" + +import json +from pathlib import Path + +import pytest +from test_corpus import make_set +from test_pipeline import SPEC, TEX_SPEC + +from in2lambda_agent import gate, pair +from in2lambda_agent.gate import Baseline, Folder +from in2lambda_agent.settings import Settings +from in2lambda_agent.spec import SPEC_NAME + + +@pytest.fixture +def corpus_root(tmp_path): + """A corpus of two folders, with a saved spec each so a replay builds.""" + made = tmp_path / "corpus" + make_set(made, "sheets", ["sheet.md", "sheet-2.md"]) + make_set(made, "tex", ["tex-sheet.tex", "tex-sheet-2.tex"]) + for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): + saved = tmp_path / "specs" / folder / SPEC_NAME + saved.parent.mkdir(parents=True) + saved.write_text(text) + return made + + +@pytest.fixture +def baseline(corpus_root, tmp_path): + """The baseline as it is written by hand, before anything is recorded.""" + return Baseline( + specs=tmp_path / "specs", + folders={ + "sheets": Folder(root=corpus_root, suffixes=["md"]), + "tex": Folder(root=corpus_root, suffixes=["tex"]), + }, + ) + + +def run(baseline, tmp_path, **kwargs): + """A gate run with its cache and work directory under tmp_path.""" + return gate.run( + baseline, + cache=tmp_path / "cache", + work=tmp_path / "gate", + settings=Settings(), + **kwargs, + ) + + +def test_recording_fills_the_counts_and_leaves_what_was_written_by_hand( + baseline, corpus_root, tmp_path +): + report = run(baseline, tmp_path, record=True) + + assert not report.failed + assert baseline.folders["sheets"].built == 2 + assert baseline.folders["tex"].built == 2 + assert baseline.folders["tex"].documents == { + "tex/tex-sheet.tex": "built", + "tex/tex-sheet-2.tex": "built", + } + # The hand-written half is the run's to read, not to write over. + assert baseline.folders["tex"].root == corpus_root + assert baseline.folders["tex"].suffixes == ["tex"] + + +def test_a_recorded_baseline_passes_the_run_that_recorded_it(baseline, tmp_path): + run(baseline, tmp_path, record=True) + + report = run(baseline, tmp_path) + + assert not report.failed + assert [one.built for one in report.folders.values()] == [2, 2] + + +def test_a_folder_that_builds_fewer_than_recorded_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built += 1 + + report = run(baseline, tmp_path) + + assert report.failed + assert report.folders["tex"].failed and not report.folders["sheets"].failed + + +def test_a_folder_whose_documents_no_longer_build_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # The spec a replay has nothing to replay without. + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert report.failed + summary = report.folders["tex"] + assert (summary.built, summary.recorded) == (0, 2) + assert summary.counts == {"no spec": 2} + assert sorted(one.document for one in summary.regressions) == [ + "tex/tex-sheet-2.tex", + "tex/tex-sheet.tex", + ] + + +def test_a_folder_the_baseline_records_no_count_for_passes(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # A folder added to the gate before it replays to a build worth defending: + # `--record` writes both halves, so neither is there yet. + baseline.folders["tex"].built = None + baseline.folders["tex"].documents = {} + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["tex"].built == 0 + + +def test_a_document_the_baseline_does_not_know_is_not_a_failure( + baseline, corpus_root, tmp_path +): + run(baseline, tmp_path, record=True) + (corpus_root / "sheets" / "sheet-3.md").write_text( + (corpus_root / "sheets" / "sheet.md").read_text() + ) + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["sheets"].built == 3 + + +def test_a_document_the_corpus_no_longer_holds_is_a_regression( + baseline, corpus_root, tmp_path +): + run(baseline, tmp_path, record=True) + (corpus_root / "tex" / "tex-sheet-2.tex").unlink() + + report = run(baseline, tmp_path) + + assert report.failed + regression = report.folders["tex"].regressions[0] + assert (regression.document, regression.current) == ( + "tex/tex-sheet-2.tex", + gate.MISSING, + ) + + +def test_a_baseline_of_no_builds_still_notices_a_document_that_did_worse( + baseline, tmp_path +): + # The check that gives a corpus where nothing builds teeth: the built count + # is 0 on both sides, so only the document's own outcome says anything. + baseline.folders["tex"].built = 0 + baseline.folders["tex"].documents = {"tex/tex-sheet.tex": "faulted"} + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + summary = report.folders["tex"] + assert summary.built == summary.recorded == 0 + assert report.failed + assert [one.document for one in summary.regressions] == ["tex/tex-sheet.tex"] + + +@pytest.mark.parametrize( + ("current", "recorded", "expected"), + [ + ("built", "built", False), + ("built", "faulted", False), + ("skipped", "built", True), + # A document read and faulted, then not read at all. + ("skipped", "faulted", True), + ("build refused", "built", True), + ("faulted", "built", True), + ("faulted", "build refused", True), + ("no spec", "faulted", True), + (gate.MISSING, "built", True), + (gate.MISSING, "no spec", False), + ], +) +def test_what_counts_as_worse(current, recorded, expected): + assert gate.worse(current, recorded) is expected + + +def test_the_folder_line_says_what_each_outcome_came_to(baseline, tmp_path): + run(baseline, tmp_path, record=True) + summary = run(baseline, tmp_path).folders["sheets"] + + line = gate.folder_line("sheets", summary) + + assert line.split() == [ + "sheets", + "built", + "2", + "faulted", + "0", + "build", + "refused", + "0", + "skipped", + "0", + "(baseline", + "built", + "2)", + ] + + +def test_the_folder_line_names_an_outcome_of_its_own(baseline, tmp_path): + summary = gate.Summary(built=1, counts={"no spec": 2}, recorded=3) + + line = gate.folder_line("tex", summary) + + assert "no spec 2" in line and "(baseline built 3)" in line + + +def test_the_folder_line_names_each_document_that_did_worse(): + summary = gate.Summary( + built=0, + counts={"faulted": 1}, + recorded=0, + regressions=[ + gate.Regression("tex/sheet-3.tex", "built", "faulted", "KaTeX"), + gate.Regression("tex/sheet-4.tex", "built", gate.MISSING), + ], + ) + + first, second, third = gate.folder_line("tex", summary).splitlines() + + assert first.startswith("tex") + assert second == " worse tex/sheet-3.tex built -> faulted: KaTeX" + assert third == " worse tex/sheet-4.tex built -> missing" + + +def test_the_folder_line_says_where_no_count_is_recorded(baseline, tmp_path): + line = gate.folder_line("tex", gate.Summary(built=1)) + + assert line.endswith("(not recorded)") + + +def test_the_baseline_survives_being_written_and_read(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built = None + path = tmp_path / "baseline.json" + + gate.write_baseline(baseline, path) + read = gate.read_baseline(path) + + assert (read.specs, read.folders) == (baseline.specs, baseline.folders) + # The directory the paths are read from is the file's own and is not a + # field of the file. + assert read.directory == tmp_path + + +@pytest.fixture +def beside(tmp_path): + """A baseline naming its specs and its corpus relative to its own directory.""" + made = tmp_path / "beside" + (made / "sheets").mkdir(parents=True) + (made / "sheets" / SPEC_NAME).write_text(SPEC) + make_set(made / "corpus", "sheets", ["sheet.md"]) + (made / "baseline.json").write_text( + json.dumps( + { + "specs": ".", + "folders": {"sheets": {"root": "corpus", "suffixes": ["md"]}}, + } + ) + ) + return made / "baseline.json" + + +def test_the_specs_and_a_relative_root_are_read_beside_the_baseline( + beside, tmp_path, monkeypatch +): + # The private baseline sits beside the specs it names, outside the + # repository, and the workbench check runs the gate in a worktree. Reading + # the file's paths from its own directory gives the same run from any + # directory. + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + + report = run(gate.read_baseline(beside), tmp_path) + + assert report.folders["sheets"].built == 1 + + +def test_recording_writes_the_paths_as_they_were_written(beside, tmp_path): + read = gate.read_baseline(beside) + + run(read, tmp_path, record=True) + gate.write_baseline(read, beside) + + written = json.loads(beside.read_text()) + assert written["specs"] == "." + assert written["folders"]["sheets"]["root"] == "corpus" + assert written["folders"]["sheets"]["built"] == 1 + + +def test_a_run_writes_nothing_under_the_directory_it_was_run_from( + baseline, tmp_path, monkeypatch +): + ran_from = tmp_path / "empty" + ran_from.mkdir() + monkeypatch.chdir(ran_from) + + run(baseline, tmp_path) + + assert list(ran_from.iterdir()) == [] + + +def test_a_run_leaves_the_committed_spec_tree_as_it_found_it(baseline, tmp_path): + # A sweep writes a record of each run beside the spec it read. The specs + # are the repository's, so the gate reads a copy and the records land there. + specs = tmp_path / "specs" + before = sorted(path.relative_to(specs) for path in specs.rglob("*")) + + run(baseline, tmp_path) + + assert sorted(path.relative_to(specs) for path in specs.rglob("*")) == before + + +def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): + # Shared between worktrees on purpose: OCR already fetched for a PDF is not + # fetched again on the next branch. + assert gate.DEFAULT_CACHE_DIR == Path.home() / ".cache" / "in2lambda-agent" + + +REPOSITORY = Path(__file__).resolve().parent.parent + + +def test_the_committed_baseline_names_its_folders_and_their_specs(): + baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") + + assert set(baseline.folders) == {"tex", "docx", "pdf"} + for name, folder in baseline.folders.items(): + # The spec each folder replays, at the path the sweep reads it from. + assert (REPOSITORY / baseline.specs / name / SPEC_NAME).is_file() + # Recorded, however low: a count of none is what the gate defends. + assert folder.built is not None + assert folder.documents + + +def test_the_committed_baseline_names_no_path_outside_the_repository(): + # An absolute root is a path on one machine, and under ExampleContents it + # is also the name of a folder of private documents. + baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") + + assert not baseline.specs.is_absolute() + assert all(not folder.root.is_absolute() for folder in baseline.folders.values()) + + +def test_the_private_corpus_keeps_its_specs_and_its_baseline_out_of_the_repository(): + # The specs for ExampleContents quote the headings of private documents, + # and the baseline beside them records those documents' file names and the + # path of the corpus on one machine. The workbench check names that + # baseline by its absolute path instead of reading it from the worktree. + assert "corpus-specs/" in (REPOSITORY / ".gitignore").read_text() + assert not (REPOSITORY / "gate-baseline.json").exists() + + +def test_the_ci_corpus_pairs_a_solutions_document_with_its_questions(): + # The corpus exists to exercise the separate-solutions document, which it + # does only when `pair` matches the file's name. Name it so that it does + # not — solutions-2.tex rather than sheet-2-solutions.tex — and the sweep + # reads it as a sheet of its own and the path is never run. + assert pair.solutions_beside(REPOSITORY / "ci-corpus/tex/sheet-2.tex") is not None + + +def test_no_document_of_the_ci_corpus_is_a_solutions_file(): + # A solutions document is frozen as the second source of the questions + # document beside it, so a sweep gives it no row of its own. + baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") + + named = [ + document + for folder in baseline.folders.values() + for document in folder.documents + if pair.questions_stem(Path(document)) is not None + ] + assert named == [] diff --git a/tests/test_ocr.py b/tests/test_ocr.py index c4fedaa..144c7c5 100644 --- a/tests/test_ocr.py +++ b/tests/test_ocr.py @@ -9,7 +9,7 @@ import pytest from in2lambda_agent.mathpix import MathpixError -from in2lambda_agent.ocr import ocr_pdf +from in2lambda_agent.ocr import cached, ocr_pdf from conftest import PNG, FakeMathpix @@ -108,6 +108,19 @@ def test_the_markdown_is_written_as_utf8_under_any_locale(pdf, tmp_path): assert written.decode("utf-8") == "# Ångström ½\n" +def test_a_document_not_in_the_cache_is_no_hit(pdf, tmp_path): + assert cached(pdf, tmp_path / "cache") is None + + +def test_a_cached_document_is_a_hit_without_a_client(pdf, tmp_path): + written = ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix()) + + hit = cached(pdf, tmp_path / "cache") + + assert hit is not None and not hit.fresh + assert (hit.markdown, hit.media) == (written.markdown, written.media) + + def test_each_document_gets_its_own_entry(pdf, tmp_path): other = tmp_path / "other.pdf" other.write_bytes(b"%PDF-1.4 a different sheet") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index dc8fc32..d07639c 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -15,6 +15,7 @@ from in2lambda_agent import package, pair, pipeline from in2lambda_agent.cli import main from in2lambda_agent.model import ModelUnavailable +from in2lambda_agent.ocr import ocr_pdf from in2lambda_agent.package import SpecRejected from in2lambda_agent.review import ReviewError from in2lambda_agent.settings import Settings @@ -1401,6 +1402,26 @@ def test_a_second_run_over_the_same_pdf_uses_the_cache(pdf, tmp_path): assert len(backend.calls) == 1 +def test_a_cached_pdf_runs_with_no_mathpix_credentials(pdf, tmp_path, monkeypatch): + monkeypatch.delenv("MATHPIX_APP_ID", raising=False) + monkeypatch.delenv("MATHPIX_API_KEY", raising=False) + ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix(SOURCE.read_text())) + + # No client at all: the cache is what the run reads, and building one from + # empty settings would raise before it got there. + result = pipeline.run( + pdf, + out_dir=tmp_path / "out", + settings=Settings(), + cache_dir=tmp_path / "cache", + backend=FakeBackend(SPEC), + ) + ocr = next(stage for stage in result.stages if stage.name == "ocr") + + assert ocr.message.startswith("cached ") + assert result.zip_path.exists() + + def test_a_pdf_without_credentials_exits_one_naming_the_variables( pdf, tmp_path, monkeypatch, capsys ):