diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 6b23bf6..4d41253 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -1,9 +1,10 @@ # The merge gate. Every push to main and every pull request runs the tests and -# then replays the three folders of ci-corpus, the committed corpus, which -# ci-baseline.json names. The job is CI's report on a branch and nothing -# merges on it: the workbench merges with `gh pr merge` as soon as its own -# check passes, and that check replays the private corpus, whose folders a -# runner cannot read, on the machine that holds it. +# then replays the committed target, ci-corpus/targets/sheet, against the set +# in2lambda's own writer wrote from it: the sheet's saved filter and reply are +# read back, so the job makes no model call and needs no credentials. The job +# is CI's report on a branch and nothing merges on it: the workbench merges +# with `gh pr merge` as soon as its own check passes, and that check replays +# the private targets, which a runner cannot read, on the machine holding them. name: gate on: @@ -14,17 +15,16 @@ on: jobs: gate: name: gate - # Pinned rather than ubuntu-latest: the runner's TeX Live decides what - # xelatex writes, and the PDF's bytes are the OCR cache's key. + # Pinned rather than ubuntu-latest: the runner's TeX Live is what the set + # checks compile with, and a release of it decides what they refuse. runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - # Pandoc reads every source document, so its version decides the blocks - # a spec selects. ubuntu-24.04 packages pandoc 3.1.3, under which every - # ci-corpus document faults on a block no selector reaches, and the - # tests fail with it. This release is the one the baselines were - # recorded under. + # Pandoc reads every source document and runs route B's filter over it, + # so its version decides what the gate's target converts to. ubuntu-24.04 + # packages pandoc 3.1.3, under which the tests fail. This release is the + # one the committed reply was written under. - name: Install pandoc run: | curl -fsSL -o "${RUNNER_TEMP}/pandoc.deb" \ @@ -66,49 +66,8 @@ jobs: - name: Install the package run: poetry install --with dev - # SOURCE_DATE_EPOCH fixes the timestamp xelatex writes into the PDF, so - # the same tex source compiles to the same bytes and the OCR cache key - # does not change between runs. - - name: Compile the CI corpus PDF - run: | - cd ci-corpus/tex - SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ - xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex - rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log - - # Keyed by the PDF's bytes, which is also how ocr_pdf names its entry - # inside the directory. A restored entry for another PDF is unused. - - name: Restore the OCR cache - uses: actions/cache@v4 - with: - path: ~/.cache/in2lambda-agent - key: ocr-${{ hashFiles('ci-corpus/pdf/*.pdf') }} - restore-keys: ocr- - - name: Tests run: poetry run pytest -q - name: Gate - env: - MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} - MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} - run: poetry run in2lambda-agent gate ci-baseline.json - - # What this run would record, so that a baseline change is committed from - # what CI saw. Written whether or not the gate passed. The file is - # recorded where it is checked out, because a baseline names its specs - # relative to its own directory and a copy under RUNNER_TEMP would find - # none; the job throws the checkout away. - - name: Record what this run did - if: always() - env: - MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} - MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} - run: poetry run in2lambda-agent gate --record ci-baseline.json - - - name: Upload it - if: always() - uses: actions/upload-artifact@v4 - with: - name: recorded-baseline - path: ci-baseline.json + run: poetry run in2lambda-agent gate ci-corpus/targets --filters ci-corpus/filters diff --git a/.gitignore b/.gitignore index c3ff88d..6a1063e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,13 +12,11 @@ dist/ # absolute path of the corpus on one machine. corpus-specs/ # The default `--filters`: a target's filter is written from a private -# document's structure, and its reply.json holds that document's text. -targets/ +# document's structure, and its reply.json holds that document's text. At the +# root alone, so that the filters of the committed target are committed. +/targets/ # in2lambda's KaTeX converter writes this into the working directory on import. log # The default `--cache`, which a run and the tests write into the directory # they are run from. .in2lambda-agent/ -# The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex -# in the workflow, so the repository holds the tex source and not the output. -ci-corpus/pdf/*.pdf diff --git a/README.md b/README.md index 8663549..4dcfaa1 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,6 @@ a field that call cannot settle is flagged for a person to read. Every field of route must be a quote of the document. [docs/plan.md](docs/plan.md) describes each step and what it detects. -`run --route spec` is the earlier route. One model call per document set writes a spec -of selectors, which is layer 1, and in2lambda runs that spec over the frozen source. A -draft the checks fault returns to the model as a fixing round, which repairs the draft -one field at a time at layers 3 and 4, until the checks report no error or the round -limit is reached. [docs/how-it-works.md](docs/how-it-works.md) describes that route -stage by stage: the line each stage prints, the file each stage writes, and what each of -the three model calls is given and may write. - ## Install ```sh @@ -50,9 +42,9 @@ empty variable counts as unset. The keys choose the backend. `ANTHROPIC_API_KEY` selects the Anthropic API, `OPENROUTER_API_KEY` selects OpenRouter, and with neither variable set the calls run on the Claude Code login, which needs [Claude Code](https://claude.com/claude-code) -installed and `claude login` run. A run over a set whose spec is saved makes no model -call, and needs no key at all. A stage that needs a variable names that variable and -the run exits 1. +installed and `claude login` run. A `gate` run over a target whose filter and reply are +saved makes no model call unless the two routes word a field differently, so the run +needs no key. A stage that needs a variable names that variable and the run exits 1. ## Convert @@ -123,186 +115,6 @@ pandoc failed, and 0 otherwise. A flagged field does not change the exit code: the zip is written whatever the flags say, and a person reads the flags after it. -## Run: the spec route - -`run DOCUMENT` converts the document as `convert` does and takes the same options. -`--route spec` runs the earlier route instead, which the rest of this section describes. -The two routes' options do not mix: `--review` without `--route spec`, or -`--write-filter` with it, is refused naming the route the option belongs to, because a -run that read it and ignored it would be a spec written twice or a review never stopped -for. - -```sh -poetry run in2lambda-agent run sheet.pdf --route spec -``` - -In full: - -```sh -poetry run in2lambda-agent run SOURCE --route spec [--spec FILE] [--review none|sample|per-question] [--rounds N] [--tries N] [--sample N] [--cache DIR] [--fresh-ocr] [--out DIR] -``` - -`SOURCE` is a PDF, markdown, tex or docx file. Mathpix converts a PDF first, and the -agent keeps the markdown and the images under the PDF's hash in `--cache` (default -`./.in2lambda-agent`), so a second run over the same PDF makes no Mathpix call. The -cache also holds the copy of the set's other sheet each spec is run over. -`--fresh-ocr` converts the PDF again and restarts the run from the new markdown. -`--out` defaults to `./out`, where in2lambda writes the set's JSON folder and its zip. - -A sheet whose solutions are a document of their own is one run. The solutions file is -the one beside `SOURCE` whose name is the sheet's with `_solutions`, `-solutions` or -` Solutions` after it, in any case, and whose suffix is the same: `Worksheet_1.pdf` -and `Worksheet_1_solutions.pdf`. Both are frozen into the one draft, the questions -first and the solutions second, and the run is named after the questions file whether -you name that file or the solutions one. A solutions file with no questions file -beside it is converted on its own, and a `pair` line names the questions file the -agent looked for. The marker above each group of solutions is that run's question. - -The run writes a spec — the YAML selectors naming which blocks of the source are -questions, parts and solutions — and saves it as `in2lambda-spec.yaml` beside `SOURCE`. -A folder of sheets is one document set and shares one spec, so the second sheet in that -folder runs with no model call. `--spec` keeps the set's spec elsewhere; the agent reads -that file if it exists and writes it if it does not. - -Every sheet of the set reuses that spec, so the run writes it in up to `--tries` model -calls, three by default, and keeps the best of them. Each call after the first reads the -spec before it, what running that spec covered, the errors the checks found, and the -blocks that spec left in no field in another document of the folder. The run keeps the -spec that left the fewest blocks unassigned and the fewest errors, and makes no further -call once a spec leaves neither. - -Each spec is run over a copy of that other document, kept in `--cache`, so the run -writes nothing beside the set's own sheets: the draft an earlier run left beside a -sheet, and the fields and the commands in it, stay as that run wrote them. The sheet's -own solutions file is not another document, since it is in this run's draft already. A -PDF beside a PDF source is passed over, because converting it takes a Mathpix call. The -record's `second` names the document the specs were run over, or names the one the run -passed over and says why, and a `set` line says the same. - -A run appends a line to `in2lambda-agent-runs.jsonl` beside the spec. The line records -the layout, the blocks and the fields of the spec run, the tokens and the seconds of the -model calls, one `iterations` entry per spec the run wrote, and the commands of each -fixing round. A run that stops for a review appends its line at the last approval. - -Each stage prints a line: - -``` -ocr fresh pass, restarting from /home/me/sheets/.in2lambda-agent/9f2c…/source.md -freeze /home/me/sheets/.in2lambda-agent/9f2c…/source.draft.json -spec wrote /home/me/sheets/in2lambda-spec.yaml via anthropic, 1883 tokens, 6.4s (try 1 of 3) -coverage PartsSepSol: 14 blocks, 8 fields at layer 1, 4 ignored, b12, b13 unassigned -validate b12 (lines 19-19) is in no field and not marked ignore.; b13 (lines 21-21) is in no field and not marked ignore. -set sheet-2.md: PartsSepSol: 11 blocks, 7 fields at layer 1, 3 ignored, b9 unassigned -freeze /home/me/sheets/.in2lambda-agent/9f2c…/source.draft.json -spec wrote /home/me/sheets/in2lambda-spec.yaml via anthropic, 2410 tokens, 7.1s (try 2 of 3) -coverage PartsSepSol: 14 blocks, 10 fields at layer 1, 4 ignored, none unassigned -validate nothing to report -set sheet-2.md: PartsSepSol: 11 blocks, 8 fields at layer 1, 3 ignored, none unassigned -spec kept try 2 of 3 -review not asked for (mode none) -build /home/me/sheets/out/set.zip -``` - -| Stage | What the line reports | -| --- | --- | -| `ocr` | the markdown of a PDF source, and whether Mathpix converted it | -| `freeze` | the draft in2lambda wrote from the source | -| `spec` | the spec file, with the backend and the tokens where the model wrote it | -| `coverage` | the layout, the blocks, the fields per layer, and the blocks in no field | -| `set` | what the spec covered of the set's other document, or why none was run over | -| `validate` | what the checks found, or `nothing to report` | -| `fix` | the round's number, the commands the model ran, and the tokens | -| `render` | the question PDFs a reviewer reads, or why there are none | -| `review` | the questions waiting for a verdict, or the mode that asked for none | -| `build` | the zip, or in2lambda's refusal to write one | - -[docs/how-it-works.md](docs/how-it-works.md#the-stages) lists every message each stage -prints, and the in2lambda function each stage calls. - -What the checks find — a block of the source in no field, two fields from the same -lines, a gap in the numbering, a part nothing answers — returns to the model as a -fixing round, with in2lambda's draft commands as its tools: `mark ignore`, -`question add`, `part add`, `question solution`, `part solution`, `split block`, -`field set` for a field that is empty or took the wrong lines, and `field replace` for -wording that no range of the source gives. The model writes a field by naming where its -text is in the source rather than by typing the text out. in2lambda records each -command in the draft's log with the layer of the field it wrote, so a reader can read -off the draft what the model did to it, and replay it without the model. - -`--rounds` is how many such rounds the run may make, three by default. The run -validates again after each round, and stops with the report and no zip, exiting 1, when -the rounds run out. A round that leaves only findings it was already given ends the run -there with those findings in the report: the agent reports a part whose solution is not -on the sheet and never writes one. `literal` types at most 80 characters, the length of -a repair such as a dropped brace, and the round refuses a longer one. `field replace` -types at most the same 80 characters, and the round refuses one that would replace the -whole of a field: it repairs wording inside a field and does not write a field. A round -that tries to write one ends the run, and the last line names the field and what the -checks say about it, for a person or a later command to quote the source range into. -Where the run -reused a saved spec and the checks fault the draft, the run writes the spec again with -what that spec covered in the prompt, if `--rounds` is 1 or more: a spec that covers the -whole set repairs every sheet in it. That rewrite takes `--tries` calls like any other -spec, and is not one of the rounds. - -### Exit codes - -`run --route spec` and `review` exit 0 where they wrote a zip, and where a review is still waiting -for a verdict. They exit 1 where the checks still fault the draft, where in2lambda -refused the build, and where a review has no question left to answer and no zip was -written. A missing credential, an unavailable backend, a reply that is not a spec, a -spec in2lambda refuses, a review command naming a question that is not under review, -and a draft command in2lambda refuses each print `in2lambda-agent: MESSAGE` on stderr -and exit 1. - -`corpus` exits 0 where it ran at least one sheet and every sheet built a set. It exits 1 -where it found no sheet, and where any sheet built no set — a row whose reason begins -`no set:`. A sheet route B failed on built its set from route A and does not change the -exit code. - -`compare` exits 1 where Mathpix, the model or `pdftoppm` failed, and 0 otherwise. A -difference it reports does not change the exit code. - -### Review - -`--review` decides how much of a set someone reads before it is built. `none`, the -default, builds as soon as the checks report no error. `sample` shows a few questions — -`--sample N`, three by default, the ones a fixing round or an edit wrote first — and -`per-question` shows every question. In either mode the run writes no zip and prints -each question with the PDF it was rendered to and the lines of the frozen source its -fields were copied from: - -``` -render 2 questions to /home/me/out/render -review mode sample, 2 of 2 questions waiting: - q1 pending: /home/me/out/render/question_000_Question_1.pdf, /home/me/sheets/sheet.md lines 5-5, 7-7 - q2 pending: /home/me/out/render/question_001_Question_2.pdf, /home/me/sheets/sheet.md lines 13-13 - answer with `in2lambda-agent review approve Q --cache /home/me/.in2lambda-agent`, … -``` - -Rendering the questions needs pandoc and xelatex. Without them the `render` line -carries in2lambda's own message, each question reads `not rendered`, and the review -goes on with the lines of the frozen source. - -The reviewer answers from the command line, in as many commands as they like: - -```sh -poetry run in2lambda-agent review approve q1 [--cache DIR] -poetry run in2lambda-agent review reject q2 --note "the solution answers (a), not (b)" [--cache DIR] -poetry run in2lambda-agent review edit q1.text "m/s" "m/s^2" [--by NAME] [--cache DIR] -``` - -A rejection's note returns to the model as a fixing round of its own, the checks run -again, and the question returns to the reviewer. An edit runs `field replace` with the -reviewer as the log's author, and in2lambda marks the field as edited. Once the -reviewer has approved every question shown, the checks run once more — the review's own -rounds and edits have changed the draft since the checks last ran — and where they -report no error the agent writes the zip and appends the run's line, with the mode, the -verdicts and the notes in it. Where they report an error, the agent builds nothing: the -last line names what the checks found, and a rejection or an edit answers it. Until -then the review waits in `review.json` under `--cache`, which each of those commands -reads. - ### Trying it in the browser ```sh @@ -346,11 +158,13 @@ p1: P(Z<\frac{0-0.120}{0.583}) → .0583 in the denominator digit dropped 1 findings over 2 pages, 16092 tokens, 36.8s ``` -`compare` writes no draft, no spec and no set, and no stage of the pipeline reads its -findings. `compare` does write the OCR cache entry `run` writes: a PDF the cache holds -no entry for is converted and cached, and `--fresh-ocr` deletes the entry a previous -run cached and converts the PDF again. [docs/ocr-comparison.md](docs/ocr-comparison.md) -lists the differences `compare` found over three corpus documents and judges each one. +`compare` writes no set, and no other command reads its findings. `compare` does write +the OCR cache entry `convert` writes: a PDF the cache holds no entry for is converted +and cached, and `--fresh-ocr` deletes the entry a previous run cached and converts the +PDF again. `compare` exits 1 where Mathpix, the model or `pdftoppm` failed, and 0 +otherwise. A difference it reports does not change the exit code. +[docs/ocr-comparison.md](docs/ocr-comparison.md) lists the differences `compare` found +over three corpus documents and judges each one. ## Corpus @@ -391,19 +205,22 @@ set, sheet, questions, parts, fields, agreed, adjudicated, flagged, not_verbatim tokens, seconds, reason ``` -[docs/how-it-works.md](docs/how-it-works.md#the-corpus-table) names each column and -where its value comes from. `reason` is empty where the sheet ran through both routes +`reason` is empty where the sheet ran through both routes and built its set. One sheet whose conversion raises is one row, with `no set:` and the error as its reason, and the sheets after it still run. A set with no filter — its filter call did not finish, or every sheet of it is a PDF, which pandoc cannot read — converts every sheet through route A alone, and each of those rows reads `no filter:` and why. A PDF among sheets pandoc reads leaves the rest of the set on both routes: the -filter is written from one of those sheets, and the PDF alone fails route B. The command -exits 1 where a sheet built no set. +filter is written from one of those sheets, and the PDF alone fails route B. + +`corpus` exits 0 where it ran at least one sheet and every sheet built a set. It exits 1 +where it found no sheet, and where any sheet built no set — a row whose reason begins +`no set:`. A sheet route B failed on built its set from route A and does not change the +exit code. `--cache` is where the OCR of each PDF is kept. It defaults to `./.in2lambda-agent`, -the directory `run` caches into, so a sweep over PDFs that `run` has already converted -makes no Mathpix call. It still reads `MATHPIX_APP_ID` and `MATHPIX_API_KEY`: `convert` +the directory `convert` caches into, so a sweep over PDFs that `convert` has already +converted makes no Mathpix call. It still reads `MATHPIX_APP_ID` and `MATHPIX_API_KEY`: `convert` builds the Mathpix client before it asks the cache, and refuses a PDF where either variable is unset, whether or not the cache holds that PDF. @@ -483,117 +300,65 @@ each PDF is kept. ## Gate -Nothing merges without a replay over real documents. `gate` reruns the saved specs -over the folders a baseline file names, and compares what each folder did this run -with what the baseline records: +Nothing merges without a run over real documents. `gate` replays every target under +`ROOT` against the set Lambda Feedback exported from it: ```sh -poetry run in2lambda-agent gate BASELINE [--record] [--cache DIR] [--work DIR] +poetry run in2lambda-agent gate ROOT [PATH ...] --filters DIR [--cache DIR] [--work DIR] ``` -Every run replays the spec route's saved specs and command logs, so no model call is -made. The command prints one line -per folder, and exits 1 when a folder builds fewer documents than the baseline records -or when a single document does worse than the baseline records it doing. The second -check is what a baseline of no builds rests on: a corpus where every document faults -still reports the document that stops being read. +`gate` converts and compares each target as `targets` does, prints the same lines, and +exits 0 where every target ran and reported no new difference. `gate` differs from +`targets` in one thing: a target whose `reply.json`, or whose `filter.lua`, is not saved +under `--filters` is reported as an error and is not converted. The two model calls that +read a document are therefore never made, and what the gate reports is a change to the +agent and not a model wording a field differently today. ``` work /tmp/in2lambda-agent-gate-3f1a -tex built 2 faulted 0 build refused 0 skipped 0 no spec 2 (baseline built 4) - worse tex/sheet-3.tex built -> no spec: replay: no model call is allowed +sheet: 0 differ, 0 known, 0 new, 0 flagged +1 target, 0 new differences ``` -The gate reads `specs`, and a folder's `root` where `root` is relative, from the -directory `BASELINE` is in, so the command gives the same run from any directory. +The error names the file and the command that writes it. Run +`in2lambda-agent targets ROOT --filters DIR` over that target, read the reply and the +filter it saves, and commit them. -There are two corpora and a baseline for each: - -| Baseline | Corpus | Run by | -| --- | --- | --- | -| `corpus-specs/gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | -| `ci-baseline.json` | the three folders of `ci-corpus` | `.github/workflows/gate.yml` | - -The repository holds `ci-baseline.json` and the specs it names, under -`ci-corpus/specs/`, because `ci-corpus` is synthetic. The repository holds neither the -specs for `ExampleContents` nor the baseline that names them: the specs quote the -headings of private documents, and the baseline records those documents' file names and -the path of the corpus on one machine. `.gitignore` lists `corpus-specs/`, and -`gate-baseline.json` sits in that directory beside the specs it reads, with `"specs": -"."`. - -`ci-corpus` is synthetic, so the repository holds its documents — every one but the -PDF, which xelatex compiles from `ci-corpus/tex/sheet-1.tex`. Run the command the -workflow runs before `gate ci-baseline.json`, because the PDF's bytes are the key the -OCR cache reads under: +`--cache` defaults to `~/.cache/in2lambda-agent`, which is outside every worktree, +because the gate runs in a worktree of its own: a cache inside the branch's directory +converts each PDF again on the next branch. `--work` defaults to a new directory under +the system temp directory, which the gate does not delete: the set of a target that +differs is written there for a person to read. Neither directory is inside the +repository, so `git status` after a gate run reports no new file. + +The repository holds one target, `ci-corpus/targets/sheet`: two synthetic markdown +documents, and the export `set_Sheet` that in2lambda's writer wrote from the saved +reply. `ci-corpus/filters/sheet` holds that target's filter and reply. The two routes +agree on every field of the sheet, so the run adjudicates nothing and reads no +credential. `.github/workflows/gate.yml` runs pytest and then ```sh -cd ci-corpus/tex -SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ - xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex -rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log +poetry run in2lambda-agent gate ci-corpus/targets --filters ci-corpus/filters ``` -Without the PDF the `pdf` folder holds no document, builds 0 against a recorded 1, -and the gate exits 1. +on every push to `main` and every pull request. -`ExampleContents` is a set of private documents and is never in the repository: the -gate reads it at the absolute `root` that `gate-baseline.json` gives, which is a path -on the machine the check runs on. - -Each folder's `root` and `suffixes` are written by hand. `built`, the count of documents -that built, and `documents`, the outcome of each single document, are what `--record` -writes. A folder the file records no `built` for passes on any count, and its line reads -`(not recorded)`. A change to a recorded count or outcome belongs in a pull request that -says why the count or the outcome changed. - -Over `ExampleContents` today, every document replays to `faulted` and the baseline -records 0 built for all three folders: pandoc's line wrapping is reported as a math -delimiter error, and each document needs a fixing round that a replay does not run. The -recorded outcomes are what the gate defends until a later ticket raises the count. - -`--cache` defaults to `~/.cache/in2lambda-agent`, which is outside every worktree, -because the gate runs in a worktree of its own: a PDF converted on one branch is -converted again on the next if the cache sits in the branch's directory. `--work` -defaults to a new directory under the system temp -directory, which the gate does not delete: read the -drafts of a folder that failed there. The gate also copies the spec tree into the work -directory and replays the copy, because a sweep appends a record of each run beside the -spec it reads. Neither directory is inside the repository, so `git status` after a gate -run reports no new file. - -`.github/workflows/gate.yml` runs pytest and then `gate ci-baseline.json` on every push -to `main` and every pull request. Mathpix reads `ci-corpus/pdf/sheet-1.pdf` once and the -workflow stores the markdown in the Actions cache under the PDF's hash. The job needs -two repository secrets, `MATHPIX_APP_ID` and `MATHPIX_API_KEY`. - -A pull request from a fork is given neither secret. `actions/cache` restores the cache -of the base branch for a fork, and the cached markdown is what the `pdf` folder then -replays, so the job passes without the secrets. If the cache is empty — the PDF's bytes -changed, or GitHub evicted the entry — Mathpix cannot be called and the `pdf` folder -builds 0 against a recorded 1, so the job fails. Push that branch to a branch of this -repository, where the secrets are read, and the job runs Mathpix once. +The private targets under `ExampleContents/targets` are replayed by the same command, +over the filter tree on the machine that holds them. The repository holds neither those +documents nor their filters: a filter is written from a private document's structure, +and a `reply.json` holds that document's text. `.gitignore` lists `/targets/`, which is +the default `--filters` directory. The job is CI's report on a branch and no merge waits for it. The workbench merges with `gh pr merge` as soon as its own check passes, and `gh pr merge` cannot wait for a GitHub check, so requiring the job on `main` would refuse every merge the workbench makes. -The workbench check runs the gate over `ExampleContents`, which is the larger corpus of -the two: - -```sh -poetry install -q --with dev && poetry run pytest -q && poetry run in2lambda-agent gate /Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/corpus-specs/gate-baseline.json -``` - -The path is absolute because the check runs in a worktree and the worktree holds -neither the baseline nor the specs it names. - ## Docker The image carries pandoc, a TeX Live whose xelatex runs the PDF generator's `template.latex`, and Node: ```sh -docker build -t in2lambda-agent . && docker run --rm -v "$PWD:/work" -w /work in2lambda-agent run sheet.md +docker build -t in2lambda-agent . && docker run --rm -v "$PWD:/work" -w /work in2lambda-agent convert sheet.md ``` diff --git a/ci-baseline.json b/ci-baseline.json deleted file mode 100644 index 3311388..0000000 --- a/ci-baseline.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "specs": "ci-corpus/specs", - "folders": { - "tex": { - "root": "ci-corpus", - "suffixes": [ - "tex" - ], - "built": 3, - "documents": { - "tex/sheet-1.tex": "built", - "tex/sheet-2.tex": "built", - "tex/sheet-3.tex": "built" - } - }, - "docx": { - "root": "ci-corpus", - "suffixes": [ - "docx" - ], - "built": 1, - "documents": { - "docx/sheet.docx": "built" - } - }, - "pdf": { - "root": "ci-corpus", - "suffixes": [ - "pdf" - ], - "built": 1, - "documents": { - "pdf/sheet-1.pdf": "built" - } - } - } -} diff --git a/ci-corpus/docx/sheet.docx b/ci-corpus/docx/sheet.docx deleted file mode 100644 index 137c8d0..0000000 Binary files a/ci-corpus/docx/sheet.docx and /dev/null differ diff --git a/ci-corpus/filters/sheet/filter.lua b/ci-corpus/filters/sheet/filter.lua new file mode 100644 index 0000000..1983428 --- /dev/null +++ b/ci-corpus/filters/sheet/filter.lua @@ -0,0 +1,67 @@ +-- The CI target's route B filter, saved as a model wrote it once: it reads the +-- sheet by its Question headings, and the solutions document, under the role +-- pandoc passes it, by the question number each answer paragraph opens with. +-- The gate replays it rather than asking for it again, so this file is what +-- route B does over ci-corpus/targets/sheet and no call is made for it. +local function esc(s) + return (s:gsub('[%z\1-\31"\\]', function(c) + if c == '"' then return '\\"' + elseif c == '\\' then return '\\\\' + elseif c == '\n' then return '\\n' + else return string.format('\\u%04x', string.byte(c)) + end + end)) +end + +local function render(blocks) + local md = pandoc.write(pandoc.Pandoc(blocks), 'commonmark_x', { wrap_text = 'wrap-none' }) + return (md:gsub('^%s+', ''):gsub('%s+$', '')) +end + +local function questions(doc) + local out, current = {}, nil + for _, b in ipairs(doc.blocks) do + if b.t == 'Header' then + current = nil + if pandoc.utils.stringify(b.content):match('^Question') then + current = {} + out[#out + 1] = current + end + elseif current then + current[#current + 1] = b + end + end + local pieces = {} + for _, blocks in ipairs(out) do + pieces[#pieces + 1] = '{"title": "", "main_text": "' .. esc(render(blocks)) .. '", "parts": []}' + end + return pieces +end + +local function solutions(doc) + local answers, inside = {}, false + for _, b in ipairs(doc.blocks) do + if b.t == 'Header' then + inside = pandoc.utils.stringify(b.content):match('^Solutions') ~= nil + elseif inside and (b.t == 'Para' or b.t == 'Plain') then + local text = render({ b }) + local n = tonumber(text:match('^(%d+)')) + if n then + answers[n] = answers[n] or {} + table.insert(answers[n], text) + end + end + end + local pieces = {} + for i = 1, #answers do + pieces[#pieces + 1] = '{"title": "", "main_text": "", "parts": [{"content": "", "options": [], "answer": "", "worked_solution": "' + .. esc(table.concat(answers[i], '\n\n')) .. '"}]}' + end + return pieces +end + +function Pandoc(doc) + local role = pandoc.utils.stringify(doc.meta.in2lambda_role or 'questions') + local pieces = role == 'solutions' and solutions(doc) or questions(doc) + return pandoc.Pandoc({ pandoc.CodeBlock('[' .. table.concat(pieces, ',\n') .. ']') }) +end diff --git a/ci-corpus/filters/sheet/reply.json b/ci-corpus/filters/sheet/reply.json new file mode 100644 index 0000000..ab5815c --- /dev/null +++ b/ci-corpus/filters/sheet/reply.json @@ -0,0 +1,26 @@ +[ + { + "title": "", + "main_text": "A ball is thrown straight up at $20\\,\\mathrm{m/s}$. Find the greatest height it reaches.", + "parts": [ + { + "content": "", + "options": [], + "answer": "", + "worked_solution": "1 The height is $h = v^2 / 2g = 20.4\\,\\mathrm{m}$." + } + ] + }, + { + "title": "", + "main_text": "A wire of resistance $R$ carries a current $I$. Find the power it dissipates.", + "parts": [ + { + "content": "", + "options": [], + "answer": "", + "worked_solution": "2 The power is $P = I^2 R$." + } + ] + } +] diff --git a/ci-corpus/pdf/.gitkeep b/ci-corpus/pdf/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/ci-corpus/specs/docx/in2lambda-spec.yaml b/ci-corpus/specs/docx/in2lambda-spec.yaml deleted file mode 100644 index 97c98ce..0000000 --- a/ci-corpus/specs/docx/in2lambda-spec.yaml +++ /dev/null @@ -1,6 +0,0 @@ -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/ci-corpus/specs/pdf/in2lambda-spec.yaml b/ci-corpus/specs/pdf/in2lambda-spec.yaml deleted file mode 100644 index 3e4997f..0000000 --- a/ci-corpus/specs/pdf/in2lambda-spec.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# The spec for the pdf set, which is sheet-1.tex compiled by xelatex and read -# back by Mathpix. The OCR writes the sheet's shape back: the question as a -# paragraph, its parts as a numbered list, and the solutions under the -# `Solutions` heading. The selectors are the tex set's for that reason. -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/ci-corpus/specs/tex/in2lambda-spec.yaml b/ci-corpus/specs/tex/in2lambda-spec.yaml deleted file mode 100644 index c95f041..0000000 --- a/ci-corpus/specs/tex/in2lambda-spec.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# The spec for the tex set: each question a paragraph, its parts the items of -# the list under it, and its solutions under a `Solutions` heading — in the -# same file for sheet-1 and sheet-3, and in sheet-2-solutions.tex for sheet-2. -# A solution begins with its label, `1(a)`, so no solution matches `question`. -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/ci-corpus/targets/sheet/set_Sheet/question_000_Question_1.json b/ci-corpus/targets/sheet/set_Sheet/question_000_Question_1.json new file mode 100644 index 0000000..c911177 --- /dev/null +++ b/ci-corpus/targets/sheet/set_Sheet/question_000_Question_1.json @@ -0,0 +1 @@ +{"orderNumber": 0, "title": "Question 1", "masterContent": "A ball is thrown straight up at $20\\,\\mathrm{m/s}$. Find the greatest height it reaches.", "publish": true, "displayFinalAnswer": true, "displayStructuredTutorial": true, "displayWorkedSolution": true, "displayChatbot": true, "parts": [{"orderNumber": 0, "content": "", "answerContent": "", "responseAreas": [], "workedSolution": {"content": "1 The height is $h = v^2 / 2g = 20.4\\,\\mathrm{m}$.", "children": []}}]} \ No newline at end of file diff --git a/ci-corpus/targets/sheet/set_Sheet/question_001_Question_2.json b/ci-corpus/targets/sheet/set_Sheet/question_001_Question_2.json new file mode 100644 index 0000000..9f24cf2 --- /dev/null +++ b/ci-corpus/targets/sheet/set_Sheet/question_001_Question_2.json @@ -0,0 +1 @@ +{"orderNumber": 1, "title": "Question 2", "masterContent": "A wire of resistance $R$ carries a current $I$. Find the power it dissipates.", "publish": true, "displayFinalAnswer": true, "displayStructuredTutorial": true, "displayWorkedSolution": true, "displayChatbot": true, "parts": [{"orderNumber": 0, "content": "", "answerContent": "", "responseAreas": [], "workedSolution": {"content": "2 The power is $P = I^2 R$.", "children": []}}]} \ No newline at end of file diff --git a/ci-corpus/targets/sheet/set_Sheet/set_Sheet.json b/ci-corpus/targets/sheet/set_Sheet/set_Sheet.json new file mode 100644 index 0000000..a41bcbf --- /dev/null +++ b/ci-corpus/targets/sheet/set_Sheet/set_Sheet.json @@ -0,0 +1 @@ +{"name": "Sheet", "description": "", "manuallyHidden": true, "finalAnswerVisibility": "OPEN_WITH_WARNINGS", "workedSolutionVisibility": "OPEN_WITH_WARNINGS", "structuredTutorialVisibility": "OPEN", "chatbotVisibility": "HIDE"} \ No newline at end of file diff --git a/ci-corpus/targets/sheet/sheet.md b/ci-corpus/targets/sheet/sheet.md new file mode 100644 index 0000000..b9e3ea8 --- /dev/null +++ b/ci-corpus/targets/sheet/sheet.md @@ -0,0 +1,9 @@ +# Tutorial Sheet 1 + +## Question 1 + +A ball is thrown straight up at $20\,\mathrm{m/s}$. Find the greatest height it reaches. + +## Question 2 + +A wire of resistance $R$ carries a current $I$. Find the power it dissipates. diff --git a/ci-corpus/targets/sheet/sheet_solutions.md b/ci-corpus/targets/sheet/sheet_solutions.md new file mode 100644 index 0000000..e4db28c --- /dev/null +++ b/ci-corpus/targets/sheet/sheet_solutions.md @@ -0,0 +1,7 @@ +# Tutorial Sheet 1 + +## Solutions + +1 The height is $h = v^2 / 2g = 20.4\,\mathrm{m}$. + +2 The power is $P = I^2 R$. diff --git a/ci-corpus/tex/figures/ball.png b/ci-corpus/tex/figures/ball.png deleted file mode 100644 index f37764b..0000000 Binary files a/ci-corpus/tex/figures/ball.png and /dev/null differ diff --git a/ci-corpus/tex/sheet-1.tex b/ci-corpus/tex/sheet-1.tex deleted file mode 100644 index d62b66d..0000000 --- a/ci-corpus/tex/sheet-1.tex +++ /dev/null @@ -1,36 +0,0 @@ -% Synthetic, in the shape of the corpus's problem sheets: a starred section -% heading, each question a paragraph with its parts as a lettered list, and a -% solutions section at the end. Nothing here is copied from ExampleContents. -\documentclass[12pt]{article} -\usepackage{amsmath} -\usepackage{graphicx} - -\begin{document} - -\section*{Problem Sheet 1: Kinematics} - -A ball is thrown straight up at $20\,\mathrm{m/s}$. - -\begin{enumerate} -\item Find the greatest height it reaches. -\item Find its time of flight. -\end{enumerate} - -A block of mass $m$ rests on a slope of angle $\theta$. - -\begin{enumerate} -\item Name the three forces acting on the block. -\item Find the least coefficient of friction that holds it still. -\end{enumerate} - -\section*{Solutions} - -1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ - -1(b) $t = 2v/g = 4.08\,\mathrm{s}$ - -2(a) Weight, the normal reaction, and friction along the slope. - -2(b) $\mu = \tan\theta$ - -\end{document} diff --git a/ci-corpus/tex/sheet-2-solutions.tex b/ci-corpus/tex/sheet-2-solutions.tex deleted file mode 100644 index 9ba70bf..0000000 --- a/ci-corpus/tex/sheet-2-solutions.tex +++ /dev/null @@ -1,24 +0,0 @@ -% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the -% shape that first failed a sweep. The name is what pairs the two: `pair` reads -% the stem before the `-solutions` ending, finds sheet-2.tex beside it, and the -% run freezes the questions first and these second. The spec reads every block -% under the `Solutions` heading as a solution, so this file adds no question. -\documentclass[12pt]{article} -\usepackage{amsmath} -\usepackage{graphicx} - -\begin{document} - -\section*{Problem Sheet 2: Answers} - -\subsection*{Solutions} - -1(a) Take the divergence term by term; each pair cancels. - -1(b) $B = \mu_0 m / 2\pi z^3$ - -2(a) $r = mv / qB$ - -2(b) $T = 2\pi m / qB$ - -\end{document} diff --git a/ci-corpus/tex/sheet-2.tex b/ci-corpus/tex/sheet-2.tex deleted file mode 100644 index aee9b53..0000000 --- a/ci-corpus/tex/sheet-2.tex +++ /dev/null @@ -1,26 +0,0 @@ -% Synthetic: questions with lettered parts and no solutions on the sheet. The -% solutions are in sheet-2-solutions.tex, which `pair` freezes as the draft's -% second source. Nothing here is copied from ExampleContents. -\documentclass[12pt]{article} -\usepackage{amsmath} -\usepackage{graphicx} - -\begin{document} - -\section*{Problem Sheet 2: Fields} - -A dipole sits at the origin. - -\begin{enumerate} -\item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. -\item Find the field on the axis at a distance $z$. -\end{enumerate} - -A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. - -\begin{enumerate} -\item Find the radius of its circular path. -\item Find the period of the motion. -\end{enumerate} - -\end{document} diff --git a/ci-corpus/tex/sheet-3.tex b/ci-corpus/tex/sheet-3.tex deleted file mode 100644 index e19e16d..0000000 --- a/ci-corpus/tex/sheet-3.tex +++ /dev/null @@ -1,38 +0,0 @@ -% Synthetic: a sheet whose first question includes a figure, which is the shape -% that first failed a sweep on the path of an image. Nothing here is copied -% from ExampleContents. -\documentclass[12pt]{article} -\usepackage{amsmath} -\usepackage{graphicx} - -\begin{document} - -\section*{Problem Sheet 3: Statics} - -The beam below carries a load $W$ at its midpoint. -\includegraphics[width=0.2\textwidth]{figures/ball.png} - -\begin{enumerate} -\item Find the reaction at each support. -\item Find the bending moment at the midpoint. -\end{enumerate} - -A ladder of mass $m$ leans against a smooth wall. - -\begin{enumerate} -\item Draw the forces acting on the ladder. -\item Find the least angle at which it does not slip. -\end{enumerate} - -\section*{Solutions} - -1(a) $W/2$ at each support. - -1(b) $M = WL/4$ - -2(a) The weight at the centre, the normal reaction at the wall, and the normal -reaction and friction at the floor. - -2(b) $\tan\alpha = 1/2\mu$ - -\end{document} diff --git a/docs/how-it-works.md b/docs/how-it-works.md deleted file mode 100644 index 1383e0b..0000000 --- a/docs/how-it-works.md +++ /dev/null @@ -1,471 +0,0 @@ -# How a run works - -`in2lambda-agent run` converts one source file into a Lambda Feedback set. This page -names each stage of a run, the in2lambda function the stage calls, the file the stage -writes and every message the stage prints. [README.md](../README.md) gives the -commands and their options. - -The agent makes three kinds of model call and no others: one writes the set's spec, -one rewrites a saved spec the checks fault, and one answers a validation report. A run -makes the spec call up to `--tries` times, three by default, and saves one of the specs -it wrote. in2lambda performs every other step and every write. - -## The stages - -A run prints one line per stage. The line is the stage name padded to nine characters, -then the message: - -``` -ocr fresh pass, restarting from /home/me/sheets/.in2lambda-agent/9f2c…/source.md -freeze /home/me/sheets/.in2lambda-agent/9f2c…/source.draft.json -spec wrote /home/me/sheets/in2lambda-spec.yaml via anthropic, 1883 tokens, 6.4s (try 1 of 3) -coverage PartsSepSol: 14 blocks, 8 fields at layer 1, 4 ignored, b12, b13 unassigned -validate b12 (lines 19-19) is in no field and not marked ignore.; b13 (lines 21-21) is in no field and not marked ignore. -set sheet-2.md: PartsSepSol: 11 blocks, 7 fields at layer 1, 3 ignored, b9 unassigned -freeze /home/me/sheets/.in2lambda-agent/9f2c…/source.draft.json -spec wrote /home/me/sheets/in2lambda-spec.yaml via anthropic, 2410 tokens, 7.1s (try 2 of 3) -coverage PartsSepSol: 14 blocks, 10 fields at layer 1, 4 ignored, none unassigned -validate nothing to report -set sheet-2.md: PartsSepSol: 11 blocks, 8 fields at layer 1, 3 ignored, none unassigned -spec kept try 2 of 3 -review not asked for (mode none) -build /home/me/sheets/out/set.zip -``` - -There are twelve stage names: `pair`, `ocr`, `freeze`, `spec`, `coverage`, `replay`, -`validate`, `set`, `fix`, `render`, `review` and `build`. A run prints `pair` only for -a solutions file with no questions file beside it, and `replay` only when it was given -a saved log to run, which is what a corpus replay is given. The spec loop prints -`freeze`, `spec`, `coverage`, `validate` and `set` once per try, and a run prints -`validate` once per check and `fix` once per fixing round, so those names repeat. Each -line is printed as the run makes it, so a `--tries 3` run prints seven lines before its -second model call: `ocr`, the five lines of the first try, and the `freeze` of the -second. The run above made two of its three tries, because the second spec left no -block unassigned and no error behind. - -| Stage | in2lambda function | What the stage writes | -| --- | --- | --- | -| `pair` | none: the agent reads the file names | nothing | -| `ocr` | none: Mathpix converts the PDF | `CACHE/HASH/source.md` and `CACHE/HASH/media/` | -| `freeze` | `in2lambda.source.add` | `SOURCE.draft.json`, beside the frozen source | -| `spec` | `in2lambda.source.show`, for the model's prompt | `in2lambda-spec.yaml`, beside `SOURCE` or at `--spec` | -| `coverage` | `in2lambda.draft.execute` with `in2lambda.draft.spec_command` | the layer 1 fields of the draft | -| `replay` | `in2lambda.draft.execute` once per saved command | the fields and the log of the draft | -| `set` | `in2lambda.source.add`, then `in2lambda.draft.execute` with `in2lambda.draft.spec_command` | the draft of the copy in `CACHE/second/`, and nothing beside the set's own sheets | -| `validate` | `in2lambda.draft.report.validate` | the report inside the draft | -| `fix` | `in2lambda.source.show`, then `in2lambda.draft.execute` once per command | the fields and the log of the draft | -| `render` | `in2lambda.draft.export.render` | `OUT/render/question_000_Question_1.pdf`, one PDF per question | -| `review` | none | `CACHE/review.json` | -| `build` | `in2lambda.draft.export.build` | `OUT/set.zip` and the set's JSON folder | - -`CACHE` is `--cache`, `./.in2lambda-agent` by default. `OUT` is `--out`, `./out` by -default. `HASH` is the sha256 of the PDF's bytes. - -A run also appends one line to `in2lambda-agent-runs.jsonl`, beside the spec. The run -appends that line last, whether it wrote a zip or ended with a report. A run that stops -for a review appends no line: the approval that completes the review appends it. - -### `pair` - -A sheet and the solutions file beside it are one run, named after the questions file. -The stage prints one message, and only where the source is a solutions file that the -folder holds no questions file for: - -* `no questions file named Tutorial_2.pdf beside Tutorial_2_Solutions.pdf; converting - the solutions alone` — the run converts the solutions file as a document of its own. - The spec prompt then says that the document holds solutions and no questions, so the - model writes a `question` selector for the marker above each group of solutions. - -### `ocr` - -The stage converts a PDF to markdown through Mathpix. A source of any other suffix is -already markdown, tex or docx, and the stage converts nothing. The stage prints one of -three messages: - -* `not needed for sheet.tex` — the source is not a PDF, and the stage names it. -* `cached /home/me/.in2lambda-agent/9f2c…/source.md` — the cache already holds the - markdown of this PDF, and Mathpix is not called. -* `fresh pass, restarting from /home/me/.in2lambda-agent/9f2c…/source.md` — Mathpix - converted the PDF, because the cache held no entry for it or because the run was - given `--fresh-ocr`. - -`--fresh-ocr` deletes the whole cache entry first, so every stage below reads the new -markdown. A conversion that fails writes no entry. - -### `freeze` - -`in2lambda.source.add` reads the source through pandoc, cuts it into numbered blocks -and writes `SOURCE.draft.json` beside the source. The message is that path and nothing -else. The draft of a PDF is written beside `source.md` in the cache entry. Each run -freezes the source again, so a second run over one file replaces the draft the first -run wrote. - -### `spec` - -The stage prints one of three messages: - -* `reused /home/me/sheets/in2lambda-spec.yaml` — the spec file exists, and the stage - makes no model call. A run that reuses a spec the checks then fault prints this stage - a second time in its `wrote` form. -* `wrote /home/me/sheets/in2lambda-spec.yaml via anthropic, 1883 tokens, 6.4s (try 1 of - 3)` — the model wrote the spec. The backend is `anthropic`, `openrouter` or - `agent-sdk`. The token count is the call's input and output tokens added together, - and the time is the wall time of the call to one decimal place. The try number counts - from 1 to `--tries`. -* `kept try 2 of 3` — the loop wrote more than one spec, and this names the try saved - for the set: the one that scored lowest. A try's score adds up the blocks it left - unassigned, the errors the checks then found, the images it dropped and the blocks - it left unassigned in the set's other document. The loop writes one spec where the - first scores zero, and prints no `kept` line. - -in2lambda refuses a spec it cannot run, and the run raises `SpecRejected`. A spec this -run wrote is deleted before that refusal reaches the user, and a spec this run wrote -over an older one is replaced by the older one, so the next run over the set reads a -spec in2lambda accepts. - -### `coverage` - -`in2lambda.draft.execute` runs the spec over the frozen source and fills the draft's -layer 1 fields. The message has one form: - -``` -PartsSepSol: 14 blocks, 9 fields at layer 1, 4 ignored, b13 unassigned -``` - -The layout is the spec's own `layout` key. `14 blocks` is the number of blocks in the -frozen source. The field counts are one phrase per layer, in layer order, and the -stage prints `no fields` where the spec wrote none. `4 ignored` is the number of -blocks the spec's `ignore` selector matched. The unassigned blocks are listed by id, -and the stage prints `none unassigned` where every block reached a field. - -A spec that dropped an image adds a second half to the line, after the unassigned -blocks: `; 2 images dropped: b10 (lines 29-30), b14 (lines 41-42)`, or `; 1 image -dropped: b4 (lines 7-8)` for one. A dropped image is a block the spec marked ignore -whose lines hold a `![`. Mathpix writes a figure as a paragraph of its own — the image -line and then its caption — so an `ignore` selector matching the caption drops the -figure, and the set is built without it. in2lambda's checks report nothing about an -ignored block, so `package.ignored_images` reads the ignored blocks back and the half -names each dropped image by its block and its lines. - -Each dropped image counts toward the try's score, so a spec that drops one is written -again. Where the next spec drops the image too, the coverage line of the try the run -kept names it, and the run goes on to `validate` and builds the set without it. - -### `replay` - -The stage runs the commands an earlier run's fixing rounds ran, read from the file the -run was given, in the order they were saved. It calls no model: the commands name the -blocks, field keys and line ranges each one wrote, so the draft the checks then see is -the draft the earlier run's rounds left. The message has one form: - -``` -5 commands from /home/me/corpus-specs/sheets/sheet.md.commands.json -``` - -in2lambda refusing one of the commands ends the run, and the refusal names which -command it was: `command 3 of 5, question add: b7b is in a field already`. The spec -route's sweep, `corpus.sweep`, writes these files, and the gate's replay reads them; a -`run` is given one through `pipeline.run`. - -### `set` - -The stage runs the try's spec over another document of the set, so that a spec is -judged on the set it is saved for rather than on this one sheet. The document is the -first other document of the source's folder, by name, whose suffix is the source's. The -run copies it under `CACHE/second/` and freezes that copy, so the draft an earlier run -left beside that sheet stays as that run wrote it. The stage prints one of four -messages: - -* `sheet-2.md: PartsSepSol: 11 blocks, 7 fields at layer 1, 3 ignored, b9 unassigned` — - the spec ran over the other document, in the `coverage` line's own form. The blocks - it left in no field there count toward the try's score, beside the blocks and the - errors this source left. -* `sheet-2.md: in2lambda refused the spec: ERROR` — in2lambda ran the spec over this - source and refused it over the other document. The try scores as leaving every block - of that document in no field. The next try is run over the document again. -* `sheet-2.md cannot be read: ERROR` — in2lambda refuses the document itself, a Word - lock file beside a docx among them. The run continues, judges its tries on this - source alone, and runs no later try over the document. -* `sheet-2.pdf passed over: converting it takes an OCR call, and the spec loop makes no - call but the model's` — the run passed the document over before running a spec over - it. A PDF beside a PDF source is passed over for the reason the message gives, and a - document the run cannot copy is passed over saying so. - -A folder holding one sheet prints no `set` line. The sheet's own solutions file is no -other document of the set: it is a second source of this run's own draft already. - -### `validate` - -`in2lambda.draft.report.validate` checks the draft and writes its report into the -draft file. A finding is an error or a warning. `build` refuses an error and proceeds -past a warning, so a report holding warnings alone is one the run builds. The stage -prints one of six messages: - -* `nothing to report` — the checks found nothing. -* `a part of q2 has no solution. — warnings, building` — the checks found warnings - alone. The warnings are joined with `; `. -* `b13 (lines 21-21) is in no field and not marked ignore.` — the errors, joined with - `; `. A fixing round follows where `--rounds` is 1 or more. Under `--rounds 0` the run - ends on this line, with no `fix` line after it. -* `ERRORS — writing the set's spec again` — the run reused a saved spec, `--rounds` is - 1 or more, and the checks fault the draft or the spec run dropped an image. The - errors are joined with `; `, and the message of each dropped image follows them, so a - reused spec whose one fault is `b4 (lines 7-8) holds an image and is marked ignore.` - says that alone. The run writes the spec again and prints `freeze`, `spec`, - `coverage` and `validate` a second time. -* `ERRORS — left by round 2, no zip` — round 2 answered no error it was given, so the - run ends with those errors and writes no zip. -* `ERRORS — round limit 3 reached, no zip` — the last round of `--rounds` ran and the - checks still fault the draft. - -### `fix` - -One round is one model call with in2lambda's draft commands as its tools. The stage -prints one message per round: - -``` -round 1: 1 command (question solution q2), 2604 tokens, 4.1s -``` - -The count and the commands come from the calls the model made: `no commands` where it -made none, `1 command` or `2 commands` and then each command with the block, question, -field or range it names. The tokens and the time are the round's own. - -`in2lambda-agent review reject` prints a second form, where the run that stopped for -the review was given a round limit below 1. The message quotes that limit: - -``` -no rounds left to answer the note with: the run was --rounds 0 -``` - -### `render` - -The stage runs only where the run is in review. It calls -`in2lambda.draft.export.render(draft, output_dir)`, which compiles one PDF per -question under `OUT/render` as Lambda Feedback's own PDF generator compiles it. Each -file is named after the question's place in the set and its title: -`question_000_Question_1.pdf`. The stage prints one of two messages: - -* `2 questions to /home/me/out/render` — the PDFs were written, and each question's - line in the listing names its file. -* in2lambda's own message, such as `Rendering questions needs xelatex.` — pandoc or - xelatex is missing, or no question compiled. The review goes on, and each question - reads `not rendered`. - -### `review` - -The stage prints one of six messages: - -* `not asked for (mode none)` — `--review` is `none`, and the run builds. -* the listing of the questions still waiting, printed by every command that leaves a - review unfinished: - - ``` - mode sample, 2 of 2 questions waiting: - q1 pending: /home/me/out/render/question_000_Question_1.pdf, /home/me/sheets/sheet.md lines 5-5, 7-7 - q2 rejected: /home/me/out/render/question_001_Question_2.pdf, /home/me/sheets/sheet.md lines 13-13 — the solution answers (a), not (b) - answer with `in2lambda-agent review approve Q --cache /home/me/.in2lambda-agent`, `review reject Q --note "..."` or `review edit FIELD OLD NEW` - ``` - - A question's line names the question, its status, its PDF or `not rendered`, the - frozen source with the lines its fields were copied from, and the note of a - rejection. A question with no lines reads `lines none`. Where the checks fault the - draft, one more line follows the questions: `the checks fault the draft, so it - cannot be built yet: ERRORS`. -* `q1 approved` — one approval, with the listing after it. -* `q1 approved, and that is all of them` — the last approval, which runs the checks - again and builds. -* `q2 rejected: the solution answers (a), not (b)` — one rejection, before its fixing - round. -* `me edited q1.text` — one `review edit`, naming the reviewer. - -### `build` - -`in2lambda.draft.export.build` writes the set's JSON folder and its zip under `--out`. -The stage prints one of two messages: - -* `/home/me/sheets/out/set.zip` — the zip that was written. -* `refused: q1 refers to ball.png, which is not beside the draft` — in2lambda would - not write the set out. The run ends with no zip. - -## The three model calls - -Each call is a system prompt, a user prompt, an optional list of tools and one reply. -Each backend limits a call differently: - -* Tool rounds: every backend stops a call that asks for tools 8 times without - answering. `anthropic` and `openrouter` raise `the anthropic backend asked for tools - for 8 rounds without answering`, naming themselves. `agent-sdk` passes the same 8 to - the SDK as `max_turns`, and the SDK's own stop raises `the agent-sdk backend stopped - on SUBTYPE: RESULT`. -* Output tokens: `anthropic` asks the API for at most 8192 output tokens per reply. - `openrouter` and `agent-sdk` send no limit. -* Timeout: `anthropic` and `openrouter` fail a request that takes longer than 300 - seconds. `agent-sdk` sets no timeout. - -| Call | What it is given | What it may write | -| --- | --- | --- | -| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds, the images that spec dropped and the blocks that spec left in no field in the set's other document | `in2lambda-spec.yaml`, and nothing else | -| Spec rewrite | the same, with the saved spec and what running it covered as the first call's try 0 | `in2lambda-spec.yaml`, and nothing else | -| Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the eight draft commands, and nothing else | - -The spec call has no tools. Its reply is the YAML of a spec, past a code fence where -the model wrote one. The agent refuses a reply that is not YAML, a reply that is not a -mapping, and a reply naming a `layout` outside `PartsSepSol`, `PartsOneSol`, -`PartSolPartSol` and `PartPartSolSol`. - -A run makes the spec call up to `--tries` times, three by default. Each call after the -first is asked for a spec that leaves fewer blocks unassigned, fewer images ignored and -fewer errors behind than the one before it, over this source and over the set's other -document. The run makes no further call once a spec scores zero, and saves the try that -scored lowest. - -The spec rewrite is the same loop, with the saved spec and what running it covered as -try 0. It runs where the run reused a saved spec and the checks fault the draft or the -spec run dropped an image, before any fixing round, and takes `--tries` calls like any -other spec. It writes layer 1 fields, and it is not one of the `--rounds`. - -The fixing round's tools are the eight in2lambda draft commands: `mark ignore`, -`question add`, `part add`, `question solution`, `part solution`, `field replace`, -`field set` and `split block`. -in2lambda writes the field, records the command in the draft's log as `in2lambda-agent` -and decides the layer. A command in2lambda refuses returns its refusal to the model as -the tool's result, and the round continues. A field's text is named by block id or line -range; `literal` types at most 80 characters, and the round refuses a longer one. A -round that leaves only errors it was already given ends the run, because the next round -would read the same prompt and the same report. - -## The four layers and the `edited` flag - -Every field of a draft carries the layer that wrote it. - -| Layer | Written by | -| --- | --- | -| 1 | the spec, run by `in2lambda spec run` | -| 2 | a predicate, which no spec the agent writes names | -| 3 | a draft command naming a block id or a line range | -| 4 | a draft command's `literal` | - -in2lambda runs predicates: a spec's `predicates` key names a Python file, and a -selector calls a function from that file. The agent's spec prompt lists the keys a -spec may hold and leaves `predicates` out, so layer 2 is 0 in every run the agent -makes. - -`field replace` writes no layer. in2lambda leaves the field at the layer that wrote it, -leaves it quoting the lines it was copied from, and sets the field's `edited` flag. So a -round that replaces a layer 1 field records `layer1=1, layer4=0, edited=1`. -`package.questions` is the one reader that counts an edited field as layer 4, -and it does so to sort the questions a `sample` review shows. - -## Review modes - -`--review` decides how much of a set a reviewer reads before it is built. - -| Mode | What the reviewer reads | What follows | -| --- | --- | --- | -| `none` | nothing | the run builds as soon as the checks report no error | -| `sample` | `--sample N` questions, 3 by default | the run builds after the last approval | -| `per-question` | every question | the run builds after the last approval | - -A sample lists the questions of layer 3 or 4 first, in question order, and fills the -rest of the count by drawing from the remaining questions at random. - -A run in `sample` or `per-question` mode stops once the checks report no error. It runs -the `render` stage over the chosen questions, writes `review.json` under `--cache`, -prints the listing and writes no zip. Every path in that file is absolute, because the reviewer's commands -run from another directory. - -`in2lambda-agent review` answers the record, one command per verdict: - -* `approve` marks the question approved. Where questions remain, the command saves the - record and prints the listing. Where none remains, the command runs the checks again, - builds the set, appends the run record with a `review` key and deletes `review.json`. - Where those checks fault the draft, the command builds nothing and the review stays - open. -* `reject` marks the question rejected, stores the note and runs a fixing round with - the note as its instruction. The round limit is the `--rounds` of the run that wrote - the record. The checks run again, the question returns to `pending`, and the listing - is printed. -* `edit` runs `field replace` with the reviewer's name as the log's author, which sets - the field's `edited` flag. The checks run again, and the question holding that field - returns to `pending`. - -## The run record - -`in2lambda-agent-runs.jsonl` sits beside the spec and holds one JSON object per line: - -| Key | Value | -| --- | --- | -| `source` | the file the run converted | -| `reused` | whether the run ran the saved spec without a model call | -| `layout` | the layout the spec named | -| `blocks` | how many blocks the frozen source holds | -| `fields` | how many fields each layer wrote, keyed by layer number | -| `ignored` | how many blocks the spec's `ignore` selector matched | -| `unassigned` | the ids of the blocks the spec run left in no field | -| `input_tokens` | what the run's model calls read | -| `output_tokens` | what they wrote | -| `seconds` | how long they took, to three decimal places | -| `rounds` | one object per fixing round: `round`, `input_tokens`, `output_tokens`, `seconds`, `commands` and `left`, how many errors the checks still found after the round | -| `review` | the review's `mode`, its `questions` with a `status` and a `note` each, its `rejections` and its `edits`. The key is absent from an unreviewed run. | - -## The corpus table - -`in2lambda-agent corpus` writes one row per sheet to `--results`. The columns are these -12, in this order: - -| Column | Where it comes from | -| --- | --- | -| `set` | the sheet's folder, relative to the corpus root | -| `sheet` | the questions document, relative to the corpus root; the solutions document beside it is read into the same row | -| `questions` | how many questions the conversion returned | -| `parts` | how many parts those questions hold | -| `fields` | `routes.fields`: how many text fields the questions and parts hold, the options of a multiple-choice part among them | -| `agreed` | how many fields the two routes returned the same text for, after `routes.fold` folds the whitespace and the notation that renders the same | -| `adjudicated` | how many fields the adjudication call decided | -| `flagged` | how many fields a person is asked to read | -| `not_verbatim` | how many of the flagged fields are not quotes of the source | -| `tokens` | what the sheet's model calls read and wrote, the direct call and the adjudication call; the set's filter call is counted on the set's first sheet | -| `seconds` | how long the sheet took, the filter call included on the set's first sheet | -| `reason` | empty where both routes ran and the sheet built its set; `no set: ` where the conversion raised, `route B failed: ` where the set is route A's alone, and `no filter: ` where no sheet of the set ran route B, because its filter call did not finish or because pandoc reads no sheet of it | - -A field only one route filled is neither agreed nor adjudicated: the count of those is -`fields - agreed - adjudicated`. `fields` counts what the conversion returned, so a -sheet route B did not run on still reports its fields. - -## Backends and settings - -`load_settings` reads four variables. Where the agent reads the process's own -environment, it first loads a `.env` found from the working directory upwards; a -variable already set in the environment wins over the `.env`. An unset or empty -variable becomes `None`. - -| Variable | Read by | -| --- | --- | -| `MATHPIX_APP_ID` | converting a PDF | -| `MATHPIX_API_KEY` | converting a PDF | -| `ANTHROPIC_API_KEY` | the model calls | -| `OPENROUTER_API_KEY` | the model calls | - -`choose_backend` reads the settings and returns the first backend the keys allow: - -1. `ANTHROPIC_API_KEY` is set: the `anthropic` backend, calling the Anthropic Messages - API with the model `claude-sonnet-5`. -2. `OPENROUTER_API_KEY` is set: the `openrouter` backend, calling OpenRouter's - OpenAI-compatible endpoint with the model `anthropic/claude-sonnet-5`. -3. Neither is set: the `agent-sdk` backend, calling the Claude Code login through the - Claude Agent SDK. It needs `claude` on the path. - -The three backends limit a call differently, as [the three model -calls](#the-three-model-calls) records. - -A run asks for a backend only where it must write a spec, and `review reject` asks for -one only where a round is left to run. So a set whose spec is saved runs with no key, -and a `--rounds 0` rejection is recorded on a machine with no key. A backend that -cannot run raises `ModelUnavailable` with one of three messages: - -``` -set ANTHROPIC_API_KEY to use the anthropic backend -set OPENROUTER_API_KEY, a key from https://openrouter.ai/keys, to use the openrouter backend -install Claude Code and run `claude login` to use the agent-sdk backend, or set ANTHROPIC_API_KEY -``` diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index 8ea341f..f98f9cc 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -1,105 +1,47 @@ """The `in2lambda-agent` command.""" import argparse -import getpass import subprocess import sys import tempfile from pathlib import Path from typing import Optional, Sequence -from in2lambda_agent import compare, gate, pair, pipeline, routes, sweep, targets +from in2lambda_agent import compare, gate, ocr, pair, routes, sweep, targets from in2lambda_agent.mathpix import MathpixClient, MathpixError from in2lambda_agent.model import ModelError, ModelUnavailable, choose_backend -from in2lambda_agent.ocr import ocr_pdf -from in2lambda_agent.package import CommandRefused, SpecRejected -from in2lambda_agent.review import ReviewError from in2lambda_agent.settings import load_settings -from in2lambda_agent.spec import BadSpec -def sample_count(given: str) -> int: - """How many questions a sample shows, which is at least one. - - Args: - given: What was typed after `--sample`. - - Returns: - The count. - - Raises: - ArgumentTypeError: it is below one. A review of no questions is not a - review: it would stop the run, write a record nothing can answer, - and never build. - """ - count = int(given) - if count < 1: - raise argparse.ArgumentTypeError( - f"a sample shows at least one question, not {count} — " - "--review none is how a set is built without a review" - ) - return count - - -def try_count(given: str) -> int: - """How many specs the agent may write, which is at least one. - - Args: - given: What was typed after `--tries`. - - Returns: - The count. - - Raises: - ArgumentTypeError: it is below one. A run that may write no spec has - none to run, and a set with no saved spec has nothing to reuse. - """ - count = int(given) - if count < 1: - raise argparse.ArgumentTypeError( - f"a run writes at least one spec, not {count}" - ) - return count - - -def reviewer_name(given: Optional[str]) -> str: - """Who the draft's log records an edit as being by. - - Asked only on the `review` branch, and never while the arguments are being - parsed: a container with no passwd entry for its user — `--user 1001` with - no LOGNAME set, which this repo's own image is run as — has no login name - to give, and a run that does not touch `--by` should not care. - - Args: - given: What was typed after `--by`, or None where nothing was. +def build_parser() -> argparse.ArgumentParser: + """The command line as the design spec describes it. Returns: - That name, or the login name, or `reviewer` where there is none. + A parser with the `convert`, `corpus`, `targets`, `gate`, `compare` and + `ui` subcommands. """ - if given is not None: - return given - try: - return getpass.getuser() - except (OSError, KeyError): - # 3.13 and after raise OSError where there is no name to be had; - # earlier versions raise KeyError. - return "reviewer" - - -def _conversion_options(parser: argparse.ArgumentParser) -> None: - """Adds the options of a conversion, which `convert` and `run` both take. + parser = argparse.ArgumentParser( + prog="in2lambda-agent", + description="Turns a source file into a Lambda Feedback set.", + ) + subcommands = parser.add_subparsers(dest="command", required=True) - Args: - parser: The subcommand's parser. - """ - parser.add_argument( + convert = subcommands.add_parser( + "convert", help="Convert DOCUMENT into a set through both routes." + ) + convert.add_argument( + "document", + type=Path, + help="The question file to convert: a PDF, markdown, tex or docx file.", + ) + convert.add_argument( "--solutions", type=Path, default=None, help="The solutions document. Default: the file beside the document " "whose name is the document's with `_solutions` after it.", ) - filter_ = parser.add_mutually_exclusive_group() + filter_ = convert.add_mutually_exclusive_group() filter_.add_argument( "--filter", type=Path, @@ -113,30 +55,6 @@ def _conversion_options(parser: argparse.ArgumentParser) -> None: help="Write route B's filter for this document with a model call, and " "keep it at `OUT/filter.lua`.", ) - - -def build_parser() -> argparse.ArgumentParser: - """The command line as the design spec describes it. - - Returns: - A parser with the `convert`, `run`, `review`, `corpus`, `targets`, - `gate`, `compare` and `ui` subcommands. - """ - parser = argparse.ArgumentParser( - prog="in2lambda-agent", - description="Turns a source file into a Lambda Feedback set.", - ) - subcommands = parser.add_subparsers(dest="command", required=True) - - convert = subcommands.add_parser( - "convert", help="Convert DOCUMENT into a set through both routes." - ) - convert.add_argument( - "document", - type=Path, - help="The question file to convert: a PDF, markdown, tex or docx file.", - ) - _conversion_options(convert) convert.add_argument( "--out", type=Path, @@ -146,107 +64,10 @@ def build_parser() -> argparse.ArgumentParser: convert.add_argument( "--cache", type=Path, - default=pipeline.DEFAULT_CACHE_DIR, + default=ocr.DEFAULT_CACHE_DIR, help="Where the OCR of each PDF is kept.", ) - run = subcommands.add_parser( - "run", help="Convert SOURCE into a set; `convert` under the default route." - ) - run.add_argument( - "source", - type=Path, - help="The question file to convert. A solutions file beside it, named " - "after it, is frozen with it.", - ) - run.add_argument( - "--route", - choices=("direct", "spec"), - default="direct", - help="Which route converts the document: `direct` is the `convert` " - "command, and `spec` writes a spec of selectors and runs it.", - ) - _conversion_options(run) - run.add_argument( - "--spec", - type=Path, - help="The set's spec file: read if present, written if not.", - ) - run.add_argument( - "--review", - choices=pipeline.REVIEW_MODES, - default="none", - help="How much of the set a reviewer sees before it is built.", - ) - run.add_argument( - "--rounds", - type=int, - default=3, - help="How many times the agent may try to fix validation errors.", - ) - run.add_argument( - "--tries", - type=try_count, - default=3, - help="How many specs the agent may write before keeping the best.", - ) - run.add_argument( - "--sample", - type=sample_count, - default=3, - help="How many questions a review in sample mode shows.", - ) - run.add_argument( - "--cache", - type=Path, - default=pipeline.DEFAULT_CACHE_DIR, - help="Where the OCR of each PDF, and a waiting review, are kept.", - ) - run.add_argument( - "--fresh-ocr", - action="store_true", - help="Convert a PDF again even if it is already cached.", - ) - run.add_argument( - "--out", - type=Path, - default=Path("out"), - help="Where to write the set's JSON folder and zip.", - ) - - review = subcommands.add_parser( - "review", help="Answer the review a run stopped for." - ) - verdicts = review.add_subparsers(dest="verdict", required=True) - approve = verdicts.add_parser("approve", help="Accept one question as it is.") - approve.add_argument("question", help="The question, by its key: q2.") - reject = verdicts.add_parser( - "reject", help="Send one question back with a note to fix it by." - ) - reject.add_argument("question", help="The question, by its key: q2.") - reject.add_argument( - "--note", required=True, help="What is wrong with it, for the agent to fix." - ) - edit = verdicts.add_parser("edit", help="Change the wording of one field.") - edit.add_argument("field", help="The field to change, by its key: q1.text.") - edit.add_argument("old", help="The wording to replace, which is in it once.") - edit.add_argument("new", help="What to put there instead.") - edit.add_argument( - "--by", - default=None, - help=( - "Who the reviewer is, as the draft's log records the edit. " - "The login name by default, or `reviewer` where there is none." - ), - ) - for verdict in (approve, reject, edit): - verdict.add_argument( - "--cache", - type=Path, - default=pipeline.DEFAULT_CACHE_DIR, - help="Where the run left the review.", - ) - corpus_command = subcommands.add_parser( "corpus", help="Convert every set of a corpus and record what each sheet did." ) @@ -281,7 +102,7 @@ def build_parser() -> argparse.ArgumentParser: corpus_command.add_argument( "--cache", type=Path, - default=pipeline.DEFAULT_CACHE_DIR, + default=ocr.DEFAULT_CACHE_DIR, help="Where the OCR of each PDF is kept, so a sweep pointed at a cache " "another run filled converts nothing.", ) @@ -322,18 +143,28 @@ def build_parser() -> argparse.ArgumentParser: against_export.add_argument( "--cache", type=Path, - default=pipeline.DEFAULT_CACHE_DIR, + default=ocr.DEFAULT_CACHE_DIR, help="Where the OCR of each PDF is kept.", ) check = subcommands.add_parser( - "gate", help="Replay the corpus the baseline names and check it against it." + "gate", + help="Replay every target under ROOT against its export, with no " + "call that reads a document.", ) - check.add_argument("baseline", type=Path, help="The committed baseline file.") + check.add_argument("root", type=Path, help="The directory the targets are under.") check.add_argument( - "--record", - action="store_true", - help="Write this run's counts to the baseline instead of checking them.", + "paths", + nargs="*", + type=Path, + help="Folders under ROOT to run, defaulting to all of them.", + ) + check.add_argument( + "--filters", + type=Path, + required=True, + help="The tree each target's saved filter and reply are read from. A " + "target with neither saved is an error rather than a model call.", ) check.add_argument( "--cache", @@ -346,7 +177,7 @@ def build_parser() -> argparse.ArgumentParser: "--work", type=Path, default=None, - help="Where the folders are copied to be run, under the system temp " + help="Where each target's set is written, under the system temp " "directory by default so the check writes nothing where it was run.", ) @@ -357,7 +188,7 @@ def build_parser() -> argparse.ArgumentParser: against.add_argument( "--cache", type=Path, - default=pipeline.DEFAULT_CACHE_DIR, + default=ocr.DEFAULT_CACHE_DIR, help="Where the OCR of each PDF is kept.", ) against.add_argument( @@ -385,61 +216,41 @@ def build_parser() -> argparse.ArgumentParser: return parser -# Each route's own options. `run` takes both sets, because argparse cannot know -# the route until it has parsed the line, so the run refuses an option of the -# route it is not taking rather than reading it and throwing it away. -_SPEC_ROUTE_OPTIONS = { - "spec": "--spec", - "review": "--review", - "rounds": "--rounds", - "tries": "--tries", - "sample": "--sample", - "fresh_ocr": "--fresh-ocr", -} -_DIRECT_ROUTE_OPTIONS = { - "solutions": "--solutions", - "filter": "--filter", - "write_filter": "--write-filter", -} - - -def misplaced_option(args: argparse.Namespace) -> Optional[str]: - """What is wrong where `run` was given an option of the other route. +def target_summary(results: Sequence[targets.Result]) -> int: + """Prints the last line of a run over targets and returns its exit code. Args: - args: The parsed arguments of `run`. + results: One result per target, as `targets.run` and `gate.run` both + return them. Returns: - What to print, naming the option and the route it belongs to, or None - where every option given belongs to the route the run is taking. + 0 where at least one target ran, none failed and none differs from its + export in a field the maintainer has not accepted. A root with no + target under it is a mistyped path rather than a clean run, so an empty + run fails like a new difference does. """ - if args.route == "direct": - options, route, fix = _SPEC_ROUTE_OPTIONS, "spec", "add --route spec" - else: - options, route, fix = _DIRECT_ROUTE_OPTIONS, "direct", "drop --route spec" - # An option counts as given where it is not the parser's default, which is - # read back from the parser rather than repeated here. - defaults = build_parser().parse_args(["run", str(args.source)]) - for dest, name in options.items(): - if getattr(args, dest) != getattr(defaults, dest): - return f"{name} is an option of the {route} route; {fix}" - return None + new = sum(len(one.new) for one in results) + failed = [one for one in results if one.error] + print( + f"{len(results)} target{'' if len(results) == 1 else 's'}, " + f"{new} new difference{'' if new == 1 else 's'}" + + (f", {len(failed)} did not run" if failed else "") + ) + return 0 if results and not new and not failed else 1 def convert_command(args: argparse.Namespace) -> int: """Converts one document through both routes and prints the report. Args: - args: The parsed arguments of `convert`, or of `run` under the direct - route, which takes the same options. + args: The parsed arguments of `convert`. Returns: 0 where the zip was written, and 1 where a conversion step failed. A flagged field does not change the code: the flags are what a person reads after the build, and no check blocks the write. """ - # `run SOURCE` converts the same document, under the other name. - document = Path(getattr(args, "document", None) or args.source) + document = Path(args.document) if args.solutions is not None: # The user named the two documents, so the folder is not asked. solutions = args.solutions @@ -479,10 +290,9 @@ def convert_command(args: argparse.Namespace) -> int: name=document.stem, ) except (MathpixError, ModelUnavailable, ModelError, OSError, routes.BadReply) as error: - # A document that is not there raises an OSError here, because this route reads - # the file itself and in2lambda never sees the name. A reply that is not a JSON - # list of questions raises BadReply, as a reply that is not a spec raises - # BadSpec on the other route. + # A document that is not there raises an OSError here, because the conversion + # reads the file itself and in2lambda never sees the name. A reply that is not + # a JSON list of questions raises BadReply. print(f"in2lambda-agent: {error}", file=sys.stderr) return 1 except subprocess.CalledProcessError as error: @@ -508,13 +318,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: """ args = build_parser().parse_args(argv) - if args.command == "run": - wrong = misplaced_option(args) - if wrong: - print(f"in2lambda-agent: {wrong}", file=sys.stderr) - return 1 - - if args.command == "convert" or (args.command == "run" and args.route == "direct"): + if args.command == "convert": return convert_command(args) if args.command == "corpus": @@ -535,51 +339,38 @@ def main(argv: Optional[Sequence[str]] = None) -> int: return 0 if rows and all(row.built for row in rows) else 1 if args.command == "targets": - results = targets.run( - args.root, - paths=args.paths, - filters=args.filters, - out_dir=args.out, - cache_dir=args.cache, - settings=load_settings(), - fresh=args.fresh, - ) - new = sum(len(one.new) for one in results) - failed = [one for one in results if one.error] - print( - f"{len(results)} target{'' if len(results) == 1 else 's'}, " - f"{new} new difference{'' if new == 1 else 's'}" - + (f", {len(failed)} did not run" if failed else "") + return target_summary( + targets.run( + args.root, + paths=args.paths, + filters=args.filters, + out_dir=args.out, + cache_dir=args.cache, + settings=load_settings(), + fresh=args.fresh, + ) ) - # A root with no target under it is a mistyped path rather than a clean - # run, so an empty run fails like a new difference does. - return 0 if results and not new and not failed else 1 if args.command == "gate": - baseline = gate.read_baseline(args.baseline) - # The directory is printed and is not deleted, so that the drafts of a - # folder that failed can be read after the run. + # The directory is printed and is not deleted, so that the sets of a + # target that differs can be read after the run. work = args.work or Path(tempfile.mkdtemp(prefix="in2lambda-agent-gate-")) print(f"work {work}") - report = gate.run( - baseline, - record=args.record, - cache=args.cache, - work=work, - settings=load_settings(), + return target_summary( + gate.run( + args.root, + paths=args.paths, + filters=args.filters, + work=work, + cache=args.cache, + settings=load_settings(), + ) ) - for name, summary in report.folders.items(): - print(gate.folder_line(name, summary)) - if args.record: - gate.write_baseline(baseline, args.baseline) - print(f"recorded {args.baseline}") - return 0 - return 1 if report.failed else 0 if args.command == "compare": settings = load_settings() try: - ocr = ocr_pdf( + converted = ocr.ocr_pdf( args.pdf, cache_dir=Path(args.cache).resolve(), client=MathpixClient.from_settings(settings), @@ -587,14 +378,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) result = compare.compare( args.pdf, - ocr.markdown.read_text(encoding="utf-8"), + converted.markdown.read_text(encoding="utf-8"), choose_backend(settings), ) except (MathpixError, ModelUnavailable, compare.RenderFailed) as error: print(f"in2lambda-agent: {error}", file=sys.stderr) return 1 - print(f"ocr {'fresh pass' if ocr.fresh else 'cached'} {ocr.markdown}") + print( + f"ocr {'fresh pass' if converted.fresh else 'cached'} " + f"{converted.markdown}" + ) if result.raw and not result.findings: # No findings is either a page the markdown matches or a reply the # parser could not read, and from here the two look the same. So @@ -630,55 +424,3 @@ def main(argv: Optional[Sequence[str]] = None) -> int: server.serve(args.corpus, args.port, open_browser=not args.no_open) return 0 - try: - if args.command == "run": - result = pipeline.run( - args.source, - out_dir=args.out, - settings=load_settings(), - spec=args.spec, - review=args.review, - rounds=args.rounds, - tries=args.tries, - sample=args.sample, - cache_dir=args.cache, - fresh_ocr=args.fresh_ocr, - ) - else: - result = pipeline.resume( - args.cache, - verdict=args.verdict, - settings=load_settings(), - key=getattr(args, "question", None), - note=getattr(args, "note", None), - field=getattr(args, "field", None), - old=getattr(args, "old", None), - new=getattr(args, "new", None), - by=reviewer_name(getattr(args, "by", None)), - ) - except ( - MathpixError, - ModelUnavailable, - ModelError, - BadSpec, - SpecRejected, - ReviewError, - CommandRefused, - ) as error: - # Missing credentials among them: the message names the variables, or - # the login to run, or what the provider said stopped a call, or what a - # spec says that a spec cannot say, or the question a review command - # names that is not under review. - print(f"in2lambda-agent: {error}", file=sys.stderr) - return 1 - - for stage in result.stages: - print(f"{stage.name:<9} {stage.message}") - - # A run that the checks found something in stops before the zip, and its - # stage lines say what they found. A run with a question still to answer - # has not failed: it is halfway through, and the review commands finish it. - # A review nothing is left to answer and no zip came out of is a failure - # like any other build that did not happen. - waiting = result.review is not None and not result.review.done - return 0 if result.zip_path or waiting else 1 diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py deleted file mode 100644 index 225a02a..0000000 --- a/in2lambda_agent/corpus.py +++ /dev/null @@ -1,532 +0,0 @@ -"""The test plan's sweep: the agent over a corpus, one row per document. - -This is the spec route's sweep, and `gate.run` is its one caller: the gate -replays the saved specs, which the two-route conversion has nothing to replay. -The `corpus` command runs the two-route sweep of `sweep.py`. - -The design spec asks, per document, what share of the fields each layer wrote, -how many rounds the checks took to come clean or that they never did, what the -model calls cost in tokens and time, and whether the set's spec was reused. That -is one run of the pipeline per document and one row of a table per run, written -where two runs can be diffed against each other. - -A questions document and the solutions document beside it are one run and one -row, which is the questions document's. - -Nothing here writes into the corpus. A run leaves a draft beside its source, -and a spec and a record beside that, so each set's folder is copied into -a work directory and run there, and the specs are kept in a tree of their own -mirroring the corpus. The work directory is deleted and written again every -sweep. A sweep keeps in the spec tree the set's spec, each document's log of -the commands its fixing rounds ran, and the record every run appends a line to. -A replay runs the spec, then the document's log, and makes no model call, so a -document a round repaired replays to the set the sweep built. -""" - -import csv -import json -import shutil -import time -from dataclasses import asdict, dataclass, fields -from pathlib import Path -from typing import Optional, Sequence - -from in2lambda_agent import package, pair, pipeline -from in2lambda_agent.model import Backend, ModelError, ModelUnavailable -from in2lambda_agent.package import SpecRejected, is_document -from in2lambda_agent.settings import Settings -from in2lambda_agent.spec import RECORD_NAME, SPEC_NAME, BadSpec - -DEFAULT_SUFFIXES = ("tex", "md", "docx") -"""What a sweep runs over unless told otherwise: everything but the PDFs, which -need Mathpix credentials and a call each.""" - -DEFAULT_RESULTS = Path("results.csv") -"""The table, under the directory the user ran from.""" - -DEFAULT_WORK_DIR = Path(".in2lambda-agent/corpus") -"""Where each set's folder is copied to be run, and wiped before it is.""" - -DEFAULT_SPEC_DIR = Path("corpus-specs") -"""The mirror tree the sets' specs are kept in, which the work directory being -wiped does not touch.""" - -COMMANDS_SUFFIX = ".commands.json" -"""What a document's saved log is named with, beside its set's spec: the -commands the fixing rounds ran, each with the block ids, field keys and line -ranges it named, and the wording a `field replace` or a typed field spells out. -The draft's `fields`, which hold every field's captured text, stay in the work -directory.""" - -ROOT_SET = "_root" -"""What the corpus root's own documents are staged under. They are a set like -any other, but the set's folder under the work directory would be the work -directory itself, and that is not one set's to empty.""" - - -@dataclass -class Row: - """One document's line of the table, in the order the columns are written. - - Attributes: - source: The document, relative to the corpus root. - set: The folder it is in, which is its document set. - outcome: `built`, `build refused` where the checks came clean and - in2lambda still would not write the set out, `faulted` for a draft - the checks still fault and no zip, `skipped` for a file that is not - a document and for a solutions document with no questions document - beside it, `no spec` for a replay with nothing saved to replay, - `replay refused` where in2lambda would not run one of the saved - commands, `no model`, `spec failed` and `fix failed` where a model - call did not finish, `spec rejected`, `bad spec`, or - `error: `. - reason: What the run had to say for itself, in the words of whatever - said it: the refusal, the first error the checks were still finding, - or what the exception said. On a `built` row it holds the warnings - the build proceeded past — a part whose solution is not on the sheet - — and is empty where there were none. One line always, so that the - table can be read on its own and two sweeps diffed. - spec: `wrote`, `reused`, or `rewritten` where a saved spec the checks - faulted was written again. - layout: The layout the spec chose. - blocks: How many blocks the spec run saw in the frozen source. A round - that splits a block adds one this does not count: it is the spec's - own reach, which is what decides whether the spec is worth reusing. - fields: How many fields the draft ended with, over every layer. - layer1: How many of them the spec wrote. - layer2: How many a predicate wrote, which is not built yet. - layer3: How many a fixing round quoted out of the source. - layer4: How many a fixing round typed out. - edited: How many fields no longer say what the lines they quote say. - unassigned: How many blocks the spec left in no field and not ignored, - which is spec-time like `blocks`: a round may since have assigned - them, so a `built` row can still report some. - rounds: How many fixing rounds ran. - input_tokens: What the run's model calls read. - output_tokens: What they wrote. - model_seconds: How long they took. - wall_seconds: How long the whole document took. - review: The review mode the run was given. - rejections: How many questions a reviewer turned down, which is always - none in mode `none` and is the column a later mode fills. - """ - - source: str - set: str - outcome: str = "" - reason: str = "" - spec: str = "" - layout: str = "" - blocks: int = 0 - fields: int = 0 - layer1: int = 0 - layer2: int = 0 - layer3: int = 0 - layer4: int = 0 - edited: int = 0 - unassigned: int = 0 - rounds: int = 0 - input_tokens: int = 0 - output_tokens: int = 0 - model_seconds: float = 0.0 - wall_seconds: float = 0.0 - review: str = "none" - rejections: int = 0 - - -COLUMNS = tuple(one.name for one in fields(Row)) -"""The table's header, which is the row's own order.""" - - -class NoModel: - """A backend that refuses, so that a replay cannot make a call. - - `pipeline.run` asks a backend whether it can run before it calls one, and - raises `ModelUnavailable` when it cannot. So a replay over a set with no - saved spec stops there and says so in its row, rather than spending the call - that would have written one. - """ - - name = "none" - - def unavailable(self) -> str: - """Why no call may be made.""" - return "replay: no model call is allowed" - - def call(self, system: str, prompt: str, tools: Sequence = ()) -> None: - """Never reached: nothing calls a backend it has been told is unavailable.""" - raise ModelUnavailable(self.unavailable()) - - -def documents( - root: Path, paths: Sequence[Path] = (), suffixes: Sequence[str] = DEFAULT_SUFFIXES -) -> list[Path]: - """Every document of the corpus a sweep is to run. - - Every file of a wanted suffix, `is_document` or not: the ones that are not - are the sweep's to say so in a row of the table, rather than to leave out - of it silently. - - Args: - root: The corpus directory. - paths: Folders under it to run, relative to it; all of it if empty. - suffixes: The file suffixes that are documents, with or without the dot. - - Returns: - The documents, sorted by their path relative to the root, so that a set's - sheets run together and in the same order every time. Paths that overlap - — a folder and the root above it, or one folder named twice — name a - document once: the table is one row per document. - """ - root = Path(root) - wanted = {"." + one.lower().lstrip(".") for one in suffixes} - found = { - path - for where in ([root / one for one in paths] or [root]) - for path in where.rglob("*") - if path.is_file() and path.suffix.lower() in wanted - } - return sorted(found, key=lambda path: path.relative_to(root).as_posix()) - - -def stage( - root: Path, folder: Path, work: Path, suffixes: Sequence[str] = DEFAULT_SUFFIXES -) -> Path: - """Copies one set's folder into the work directory, and empties it first. - - The set is the folder and everything under it: the sheets, and the figures, - styles and class files they name, wherever in the tree those sit. A - subfolder comes along whatever is in it, sheets of its own included, since - deciding that per folder is what dropped the figures PHYS40002's sheets - name. A subfolder that is a set in its own right is staged again under its - own name when the sweep reaches it, so the corpus root brings a second copy - of every set beneath it. - - What no run could read is left behind: the archives, and the PDFs unless - they are what is being run. They are most of what a corpus weighs, and the - copy is made again every sweep. So is what an earlier run of the agent left - in the corpus, at whatever depth: a spec, a record, a draft, and the cache - directory. And so is the work directory itself when it sits under the set - being staged, as the default `--work` does under the corpus root: a copy of - it would hold a copy of itself, and so on until the path is too long for - the filesystem. - - A set staged twice is emptied first, so a sweep starts from nothing every - time. That is the set's own folder and never the work directory, which holds - the other sets of the sweep and whatever else the user pointed `--work` at. - - Args: - root: The corpus directory. - folder: The set's folder under it. - work: Where the copies are kept. - suffixes: What counts as a document, as `documents` reads it. - - Returns: - The folder's copy, which is what the runs are given. - """ - relative = folder.relative_to(root) - into = Path(work) / (ROOT_SET if relative == Path(".") else relative) - if into.exists(): - shutil.rmtree(into) - into.parent.mkdir(parents=True, exist_ok=True) - pdfs = "pdf" in {one.lower().lstrip(".") for one in suffixes} - # Asked at every level of the tree, so what an earlier run left is left - # behind wherever in the set it sits. - by_name = shutil.ignore_patterns( - "*.zip", - *(() if pdfs else ("*.pdf",)), - SPEC_NAME, - RECORD_NAME, - # A draft is named after the source it was frozen from, so there is one - # per sheet rather than one per folder: the name is not known in advance - # and the suffix is what says a file is one. - "*" + package.DRAFT_SUFFIX, - pipeline.DEFAULT_CACHE_DIR.name, - ) - target = Path(work).resolve() - - def ignore(where: str, named: list[str]) -> set[str]: - """What is left behind in one folder: by name, and the work directory.""" - return by_name(where, named) | { - one for one in named if (Path(where) / one).resolve() == target - } - - shutil.copytree(folder, into, ignore=ignore) - return into - - -def _one_line(text: str) -> str: - """A reason as one line of the table: a message that wraps stays one cell.""" - return " ".join(text.split()) - - -def run_one( - source: Path, - *, - name: str, - set_name: str, - spec: Path, - settings: Settings, - rounds: int = 3, - tries: int = 3, - replay: bool = False, - cache: Path = pipeline.DEFAULT_CACHE_DIR, - backend: Optional[Backend] = None, -) -> Row: - """Runs the pipeline over one document and reads the row off what it did. - - Nothing a document does ends the sweep: what the pipeline raises, and what - anything else raises, becomes this document's outcome and the next one runs. - - Args: - source: The document, in the work directory rather than the corpus. - name: What to call it in the table, relative to the corpus root. - set_name: Its set, relative to the corpus root. - spec: The set's spec, in the mirror tree: read if it is there, written - if it is not. The document's log is written beside it, named after - the document. - settings: The environment the run has available. - rounds: The round limit, ignored in a replay, which can run none. - tries: How many specs the run may write before keeping the best. - replay: Run the saved spec and the document's saved log, making no - model call. - cache: Where the OCR of each PDF is kept. - backend: The backend to write a spec with, chosen from the settings if - absent. - - Returns: - The document's row. - """ - existed = Path(spec).is_file() - # The log is one document's and the spec is the set's, so the log is named - # after the document, beside the spec. - commands = Path(spec).parent / f"{source.name}{COMMANDS_SUFFIX}" - row = Row(source=name, set=set_name) - started = time.monotonic() - try: - result = pipeline.run( - source, - out_dir=source.parent / "out", - settings=settings, - spec=spec, - # The log of an earlier sweep's fixing rounds, which is what a - # replay has in place of the rounds. A sweep writes it below. - commands=commands if replay else None, - review=row.review, - # A replay has no model to run a round with, so it stops at the - # report: the row then says what the saved spec left rather than - # that a call could not be made. - rounds=0 if replay else rounds, - tries=tries, - cache_dir=cache, - backend=NoModel() if replay else backend, - ) - except ModelUnavailable as error: - row.outcome = "no model" if existed else "no spec" - row.reason = _one_line(str(error)) - except package.CommandRefused as error: - row.outcome = "replay refused" - row.reason = _one_line(str(error)) - except ModelError as error: - # Which call did not finish, and what the provider said it stopped on. - # A row reading `error: ResultError` says neither. - row.outcome = f"{error.stage or 'model'} failed" - row.reason = _one_line(str(error)) - except SpecRejected as error: - row.outcome = "spec rejected" - row.reason = _one_line(str(error)) - except BadSpec as error: - row.outcome = "bad spec" - row.reason = _one_line(str(error)) - except Exception as error: - # Whatever else a document manages to raise is that document's row: a - # sweep of a corpus is not worth ending over one file in it. - row.outcome = f"error: {type(error).__name__}" - row.reason = _one_line(str(error)) - else: - # A build in2lambda refused is not a draft the checks faulted: the - # report came clean and the export is what stopped, which the rounds - # column would otherwise misreport as the spec never covering the sheet. - if result.zip_path: - row.outcome = "built" - else: - row.outcome = "build refused" if result.clean else "faulted" - row.reason = _one_line(result.reason) - row.spec = "reused" if result.reused else "rewritten" if existed else "wrote" - if result.coverage is not None: - row.layout = result.coverage.layout - # Coverage is what the spec run alone made of the source, so these - # two stay spec-time on purpose: they say how far the spec got - # before any round, which is what says whether it is worth reusing. - row.blocks = result.coverage.blocks - row.unassigned = len(result.coverage.unassigned) - if result.draft is not None: - if not replay: - # What a later replay runs in place of the fixing rounds. A run - # that took no round writes an empty list, so that a replay - # reading no file there knows the document was never swept. - commands.write_text( - json.dumps(package.fix_log(result.draft), indent=1), - encoding="utf-8", - ) - # The counts are keyed by the column names they fill, and `fields` - # is their total, since a round writes fields the spec run's own - # count knows nothing about. - counted = package.layers(result.draft) - for column, count in counted.items(): - setattr(row, column, count) - row.fields = sum( - count for column, count in counted.items() if column != "edited" - ) - row.rounds = len(result.rounds) - row.input_tokens = result.usage.input_tokens - row.output_tokens = result.usage.output_tokens - row.model_seconds = round(result.usage.seconds, 3) - row.wall_seconds = round(time.monotonic() - started, 3) - return row - - -def write_results(rows: Sequence[Row], path: Path) -> None: - """Writes the table, header and all, over whatever was there before. - - Args: - rows: The rows, in the order they are to be written. - path: The file to write. - """ - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", newline="", encoding="utf-8") as out: - writer = csv.DictWriter(out, fieldnames=COLUMNS) - writer.writeheader() - for row in rows: - writer.writerow(asdict(row)) - - -def sweep( - root: Path, - *, - paths: Sequence[Path] = (), - suffixes: Sequence[str] = DEFAULT_SUFFIXES, - results: Path = DEFAULT_RESULTS, - work: Path = DEFAULT_WORK_DIR, - specs: Path = DEFAULT_SPEC_DIR, - replay: bool = False, - rounds: int = 3, - tries: int = 3, - cache: Path = pipeline.DEFAULT_CACHE_DIR, - settings: Optional[Settings] = None, - backend: Optional[Backend] = None, -) -> list[Row]: - """Runs every document of a corpus, printing a line each and writing the table. - - Args: - root: The corpus directory. - paths: Folders under it to run, relative to it; all of it if empty. - suffixes: The file suffixes that are documents. - results: Where to write the table. - work: Where each set's folder is copied to be run. - specs: The tree the sets' specs are kept in, mirroring the corpus, with - each document's log of its fixing rounds beside its set's spec. - replay: Run the saved specs and logs and nothing else, making no model - call. - rounds: The round limit each run is given. - tries: How many specs each run may write before keeping the best. - cache: Where the OCR of each PDF is kept, so that a sweep pointed at a - cache another run filled converts nothing. - settings: The environment the runs have available. - backend: The backend to write the specs with, chosen from the settings - if absent. - - Returns: - One row per document, in the order they ran. - """ - root = Path(root).resolve() - work = Path(work).resolve() - specs = Path(specs).resolve() - settings = settings if settings is not None else Settings() - - staged: dict[Path, Optional[Path]] = {} - unstageable: dict[Path, tuple[str, str]] = {} - rows = [] - for document in documents(root, paths, suffixes): - folder = document.parent - relative = document.relative_to(root) - # Reading a file to see whether it is a document can fail the way any - # other read can, and that is this document's row as it was when the - # read happened inside `run_one`, rather than the end of the sweep. - try: - of_its_own = is_document(document) - except OSError as error: - row = Row( - source=relative.as_posix(), - set=relative.parent.as_posix(), - outcome=f"error: {type(error).__name__}", - reason=_one_line(str(error)), - ) - print(f"{row.outcome:<20} {row.source}") - rows.append(row) - continue - # Before the staging, so that a folder of drawings is never staged on - # one of their account: they are its parent set's, and came along with it. - if not of_its_own: - row = Row( - source=relative.as_posix(), - set=relative.parent.as_posix(), - outcome="skipped", - reason="no \\begin{document}", - ) - print(f"{row.outcome:<20} {row.source}") - rows.append(row) - continue - # A solutions document is frozen into the run of the questions document - # it answers, so the pair is one row, which is the questions document's. - # One with no questions document beside it is converted on its own, and - # is a row like any other document. - if ( - pair.questions_stem(document) is not None - and pair.questions_beside(document) is not None - ): - continue - # Staging is per set and the row is per document, so the guard is here - # rather than in `run_one`: what a copy raises — an unreadable folder, a - # file that goes while it is being read — is a row for every sheet of - # the set, none of which ran, and the sets after it still do. - if folder not in staged: - try: - staged[folder] = stage(root, folder, work, suffixes) - except Exception as error: - staged[folder] = None - unstageable[folder] = ( - f"error: {type(error).__name__}", - _one_line(str(error)), - ) - if staged[folder] is None: - outcome, reason = unstageable[folder] - row = Row( - source=relative.as_posix(), - set=relative.parent.as_posix(), - outcome=outcome, - reason=reason, - ) - print(f"{row.outcome:<20} {row.source}") - rows.append(row) - continue - spec = specs / relative.parent / SPEC_NAME - spec.parent.mkdir(parents=True, exist_ok=True) - row = run_one( - staged[folder] / document.name, - name=relative.as_posix(), - set_name=relative.parent.as_posix(), - spec=spec, - settings=settings, - rounds=rounds, - tries=tries, - replay=replay, - cache=cache, - backend=backend, - ) - print(f"{row.outcome:<20} {row.source}") - rows.append(row) - - write_results(rows, results) - return rows diff --git a/in2lambda_agent/fix.py b/in2lambda_agent/fix.py deleted file mode 100644 index 1e2bbe1..0000000 --- a/in2lambda_agent/fix.py +++ /dev/null @@ -1,467 +0,0 @@ -"""Layers 3 and 4: the model answers a validation report, one round at a time. - -`validate` hands back findings — a block in no field, a part nothing answers — -and each is answered by one of in2lambda's draft commands, which are the tools of -this call. The model chooses the command and the lines; in2lambda writes the -field, records the command in the draft's log and decides the layer, so a fix -leaves the same trail whoever asked for it. - -Nothing here reads the `check` or the `level` a finding names. A finding is a -field, some lines and a sentence, and that is all a round is given: in2lambda's -own checks of the set — the maths delimiters, KaTeX, the images, the PDF compile -— arrive as findings of this same shape under the check `problem`, and a round -that switched on the check's name would have to be taught each one as it arrived. -""" - -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable, Optional, Sequence - -from in2lambda_agent import package -from in2lambda_agent.model import Backend, Reply, Tool, ToolCall, Usage - -LITERAL_MAX = 80 -"""The most characters `literal` may type into a field. - -A literal is a repair — a brace the OCR dropped, a marker the field cannot quote -as it stands — and a repair is short. Anything longer is the model writing what -the document does not say, which is the one thing a field must never hold, so -the cap is on the tool's schema and again in the runner: a backend that does not -enforce a schema still gets the refusal. - -`field replace` types `new` into a field, so `new` is capped the same way. -""" - -WRITE_REFUSED = ( - "would be written, not repaired — the loop replaces wording inside a field " - "and does not write a field; what the source holds for the field is quoted " - "in by block or line range" -) -"""The refusal for a `field replace` that replaces the whole of a field. - -`old` matching nothing, `^$`, or everything, `.*`, makes `new` the field's whole -text, which is the model writing what the document does not say. `pipeline` -reads this clause back off the round's tool calls to tell such a refusal from an -ordinary one. -""" - -SYSTEM = f"""\ -You are fixing a draft of a question set. `in2lambda validate` has reported what \ -is wrong with it, and you have the commands that change one. Answer everything \ -the report names, then say in one line what you did. - -A draft is fields — `q1.text`, `q1.p2.text`, `q1.solution` — filled in from a \ -frozen source, which you are shown with every line numbered and every block's id \ -in the margin. Each finding names the field or the block it is about. - -Copy, do not type. A field is written by naming where its text is in the source: \ -a block id, `b7`, or lines, `s13` or `s13:14`. That is what freezing the source \ -was for, and it is the only way a field ends up saying what the document says. \ -`literal` types a few characters out instead — a repair, for where the source \ -spells the text wrongly and no range of it can be quoted as it stands. It takes \ -at most {LITERAL_MAX} characters and a longer one is refused. It is never for \ -writing a solution, a part or a question the document does not contain. - -A finding no range of the source can answer is left as it is. A part whose \ -solution is not on the sheet has no solution, and there is nothing in the source \ -to give it: do not write one. It is reported as a warning, and the set is built \ -with it. Say in your one-line reply which findings you left and why, and the run \ -reports them. - -The commands: - - mark_ignore a block that belongs in no question: a heading, a page - number, a rubric. This is the answer to a block that is in - no field because nothing should take one from it. - question_add a question, from the lines holding its stem. - part_add a lettered part of a question that is written already. - question_solution a question's worked solution, wherever it is written. It - answers every part of that question that has none of its own. - part_solution one part's own worked solution, from the lines under it on - the sheet. Use it where each part is answered where it - stands, and question_solution where one solution answers - the whole question. - split_block cuts a block in two at a line, so that each half can be - named: `b7` split at 14 becomes `b7a` and `b7b`. Use it when - one block holds two things — a question and its first part - run together, two solutions with no blank line between them. - field_replace changes wording inside a field that is written already, for - what no range of the source says correctly. The wording you - replace has to be in the field exactly once, and it may not - be empty or the whole of the field: this command repairs - wording and does not write a field. What you put in its - place is at most {LITERAL_MAX} characters, as a literal is. - A field whose text the source does not hold is left as a - finding. - field_set quotes other lines into a field that is written already, - for a field that is empty or that took the wrong lines. It - writes the field again from the lines you name and drops - the ones it held, which are then in no field until you - answer for them. It has no literal: what it writes is in - the source. - -Lines that are already in a field cannot be put in another one. A command naming \ -them is refused, and so is one naming a block that is not there; either way you \ -are told why, so read it and try something else rather than running it again. - -A reviewer who has read a question against the document may send a note as well. \ -Answer it with the same commands, and take what it says about the draft over what \ -the checks say.\ -""" - -_BLOCK = { - "type": "string", - "description": "A block id, as the margin of the source shows it: b7, b7a.", -} - -_QUESTION = { - "type": "string", - "description": "The question to add to, by the key of its text: q2.", -} - -_PART = { - "type": "string", - "description": "The part to answer, by the key of its text: q1.p2.", -} - -# Every command that fills a field takes one or the other of these, and in2lambda -# refuses both at once, so neither is required and the model is told which to use. -_WHERE = { - "text": { - "type": "string", - "description": ( - "Where the text is in the frozen source: a block id, b7, or lines, " - "s13 or s13:14. Give this or literal, not both." - ), - }, - "literal": { - "type": "string", - "maxLength": LITERAL_MAX, - "description": ( - "A few characters, typed out, where the source spells the text " - f"wrongly and no range of it can be quoted. At most {LITERAL_MAX} " - "characters, and never content the document does not hold; what it " - "writes is recorded as edited." - ), - }, -} - -_DESCRIPTIONS = { - "mark ignore": "Mark one block of the source as belonging in no question.", - "question add": "Add a question, from the lines holding its stem.", - "part add": "Add a lettered part to a question that is written already.", - "question solution": "Give a question the worked solution written for it.", - "field replace": ( - "Change one piece of wording inside a field already written. The wording " - "replaced may not be empty or the whole of the field, and what replaces " - f"it is at most {LITERAL_MAX} characters: this repairs a field and does " - "not write one." - ), - "split block": "Cut one block in two at a line, so each half can be named.", - "field set": ( - "Quote other lines of the source into a field already written, for a " - "field that is empty or that took the wrong lines. The lines it held " - "are dropped, and are then in no field." - ), - "part solution": ( - "Give one part the worked solution written under it. Where one " - "solution answers the whole question, question solution writes it." - ), -} - -_PARAMETERS: dict[str, dict[str, Any]] = { - "mark ignore": { - "type": "object", - "properties": {"block": _BLOCK}, - "required": ["block"], - }, - "question add": {"type": "object", "properties": _WHERE, "required": []}, - "part add": { - "type": "object", - "properties": {"question": _QUESTION, **_WHERE}, - "required": ["question"], - }, - "question solution": { - "type": "object", - "properties": {"question": _QUESTION, **_WHERE}, - "required": ["question"], - }, - "field replace": { - "type": "object", - "properties": { - "field": { - "type": "string", - "description": "The field to change, by its key: q1.text.", - }, - "old": { - "type": "string", - "description": ( - "The wording to replace, which is in the field once. It may " - "not be empty or the whole of the field's text." - ), - }, - "new": { - "type": "string", - "maxLength": LITERAL_MAX, - "description": ( - "What to put there instead: a repair of at most " - f"{LITERAL_MAX} characters, and never content the document " - "does not hold." - ), - }, - "regex": { - "type": "boolean", - "description": "Read `old` as a regular expression rather than text.", - }, - }, - "required": ["field", "old", "new"], - }, - "split block": { - "type": "object", - "properties": { - "block": _BLOCK, - "at": { - "type": "integer", - "description": "The first line of the second half.", - }, - }, - "required": ["block", "at"], - }, - # No `literal`: this command quotes lines into a field and in2lambda takes - # nothing else, so the wording no range of the source holds is `field - # replace`'s to repair once the field has been set. - "field set": { - "type": "object", - "properties": { - "field": { - "type": "string", - "description": "The field to write again, by its key: q1.text.", - }, - "text": { - "type": "string", - "description": ( - "Where the field's text is in the frozen source: a block " - "id, b7, or lines, s13 or s13:14." - ), - }, - }, - "required": ["field", "text"], - }, - "part solution": { - "type": "object", - "properties": {"part": _PART, **_WHERE}, - "required": ["part"], - }, -} - - -@dataclass -class RoundResult: - """What one round of fixing did, for the run's record. - - Attributes: - number: Which round it was, from 1. - commands: The commands the model ran, and what each was told back. - usage: What the round's one model call cost. - left: How many findings the checks still had once the commands had run. - """ - - number: int - commands: list[ToolCall] - usage: Usage - left: int - - -def tools(draft: Path) -> list[Tool]: - """The draft commands, as the tools of one model call. - - Args: - draft: The draft file the commands change. - - Returns: - One tool per command in `package.COMMANDS`, each running it against that - draft. Tools are named with underscores, which is the only shape a name - may have on the wire; the command keeps the name the log records. - """ - return [ - Tool( - name=name.replace(" ", "_"), - description=_DESCRIPTIONS[name], - parameters=_PARAMETERS[name], - run=_runner(draft, name), - ) - for name in package.COMMANDS - ] - - -def fix_round( - draft: Path, - shown: str, - report: package.Report, - backend: Backend, - instruction: Optional[str] = None, -) -> Reply: - """One round: the report and the source to the model, its commands to the draft. - - Args: - draft: The draft file. - shown: The numbered source with block ids, as `source show` prints it, - read again each round so that a split made last round is in it. - report: What the checks found, which is what this round is to answer. - backend: The backend to call, already known to be available. - instruction: A reviewer's note about the draft, which this round is to - answer as well as the report — and by itself where a reviewer - rejected a question the checks had nothing to say about. - - Returns: - The reply, whose tool calls are the commands the draft now records. The - draft is changed by the tools as the model runs them, so a round that - answers nothing leaves it exactly as it was. - """ - findings = "\n".join( - # The field and the lines are named before the sentence as well as in it. - # Every message in2lambda writes today quotes both, but a check it grows - # later need not, and a round has to be able to act on a finding by - # itself: what to name in the command, and which lines it is about. - f"- {finding.level} {finding.check} {finding.field}" - f"{_lines(finding.ranges)}: {finding.message}" - for finding in report.findings - ) - prompt = f"Here is the source, one line each with its block id:\n\n{shown}\n\n" - if findings: - prompt += ( - "in2lambda validate reports this about the draft written from it:" - f"\n\n{findings}\n" - ) - else: - prompt += "in2lambda validate has nothing to report about the draft.\n" - if instruction is not None: - prompt += f"\nA reviewer has read the draft and says this:\n\n{instruction}\n" - return backend.call(SYSTEM, prompt, tools(draft)) - - -def summary(calls: Sequence[ToolCall]) -> str: - """The commands of one round, as the stage line names them. - - Args: - calls: What the model ran, in order. - - Returns: - How many there were and which, each with what it was about. - """ - if not calls: - return "no commands" - named = ", ".join( - f"{call.name.replace('_', ' ')} {_subject(call)}" for call in calls - ) - return f"{len(calls)} command{'' if len(calls) == 1 else 's'} ({named})" - - -def unrepaired(calls: Sequence[ToolCall]) -> list[str]: - """The fields a round's `field replace` was refused over as a write. - - Args: - calls: What the model ran, in order. - - Returns: - The field named by each `field replace` refused as writing a field, in - the order the round ran them. A refusal leaves nothing in the draft, so - the tool's result is the only record of one, and this reads WRITE_REFUSED - back off that result. - """ - return [ - str(call.arguments.get("field")) - for call in calls - if call.name == "field_replace" and WRITE_REFUSED in call.result - ] - - -def _lines(ranges: list[list[int]]) -> str: - """The lines a finding is about, or "" where it is about no line at all.""" - if not ranges: - return "" - return " lines " + ", ".join(f"{start}-{end}" for start, end in ranges) - - -def _runner(draft: Path, name: str) -> Callable[[dict[str, Any]], str]: - """What one tool does when the model calls it. - - A refusal is something for the model to read and work round — the block it - named is not there, the lines it wants are in a field already — so it comes - back as the tool's result. Raising would end the call, and with it the round - and every fix the model had left to make. - - A `literal` over LITERAL_MAX is refused here as well as by the schema, and - for the same reason as any other refusal: nothing is written, nothing is - logged, and the model is told why. `field replace` types `new` into a field, - so `new` is measured the same way, and a `field replace` that would replace - the whole of a field is refused too. - """ - - def run(arguments: dict[str, Any]) -> str: - refusal = _too_long(name, "literal", arguments.get("literal")) - if name == "field replace": - refusal = refusal or _too_long(name, "new", arguments.get("new")) - refusal = refusal or _writes_field(draft, name, arguments) - if refusal is not None: - return refusal - try: - return f"{name} wrote {package.command(draft, name, arguments)}" - except package.CommandRefused as refused: - return f"{name} was refused: {refused}" - - return run - - -def _too_long(name: str, argument: str, typed: Any) -> Optional[str]: - """The refusal for text over LITERAL_MAX, or None where it is short enough. - - Only a string is measured: a backend that sends `null`, or a number, is not - typing anything too long, and in2lambda already has something to say about - an argument of the wrong shape. - """ - if not isinstance(typed, str) or len(typed) <= LITERAL_MAX: - return None - return ( - f"{name} was refused: {argument} is {len(typed)} characters, and at " - f"most {LITERAL_MAX} may be typed — a field's text is copied from " - f"the source by block or line range, and what the source does not " - f"hold is left as a finding" - ) - - -def _writes_field(draft: Path, name: str, args: dict[str, Any]) -> Optional[str]: - """The refusal for a `field replace` that replaces the whole of a field. - - The field is read from the draft, since what `old` matches is only a whole - field against the text the field holds. A field the draft does not hold, and - a regular expression Python cannot compile, are left to in2lambda, which - refuses each of them and says so. - """ - key = args.get("field") - old = args.get("old") - if not isinstance(key, str) or not isinstance(old, str): - return None - value = package.field_value(draft, key) - if value is None: - return None - if args.get("regex"): - try: - found = re.search(old, value) - except re.error: - return None - matched = None if found is None else found.group(0) - else: - matched = old - if matched not in ("", value): - return None - return f"{name} was refused: {key} {WRITE_REFUSED}" - - -def _subject(call: ToolCall) -> str: - """What one command was about, for the stage line: a block, question or field.""" - for name in ("block", "question", "part", "field", "text", "literal"): - if name in call.arguments: - return str(call.arguments[name]) - return "" diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py index e55e30e..ffc2d72 100644 --- a/in2lambda_agent/gate.py +++ b/in2lambda_agent/gate.py @@ -1,326 +1,72 @@ -"""The merge gate: a replay over real documents, compared with a baseline. +"""The merge gate: every target against its export, and no model call. Every ticket before this one was tested on synthetic fixtures. The first sweep -over a folder of real PDFs failed on four faults no fixture had: a list-valued -selector, a file of solutions with no questions, an image path, and a wrapped -line. A merge now requires an end-to-end run over real documents. - -The run is `corpus.sweep(replay=True)` over the specs the repository keeps, so -it makes no model call. The baseline file names the folders to run and records -both how many documents each folder built and what each single document did. -The gate fails when a folder builds fewer documents than the baseline records, -or when any one document does worse than it is recorded as doing. The second -check is what gives a baseline of no builds at all teeth: a folder where every -document faults still notices the day one of them stops being read. A folder -the baseline records no count for passes on any count, which is how a folder is -added to the gate before it replays to a build worth defending. A change to a -recorded count belongs in a pull request that states why the count changed. - -A baseline names its specs, and a corpus of its own, relative to the directory -the file is in. `ci-baseline.json` is at the root of the repository and names -`ci-corpus/specs`; the baseline for the private corpus is at -`corpus-specs/gate-baseline.json`, beside the specs it names, and the -repository holds neither file. +over real documents failed on four faults no fixture had. A merge now requires +an end-to-end run over real documents, and a target is the one place the agent +can be marked right or wrong rather than merely flagged: what it converts is +compared with the set Lambda Feedback exported from the same two documents. + +So the gate is a `targets` run with `replay` set. The filter and route A's +reply are read back from the tree the repository keeps rather than asked for, so +the run makes neither of the two calls that read the document: what fails is a +change to the agent and not a model wording a field differently today. A target +whose filter or reply is not saved is an error naming the file, rather than a +call nobody asked for. + +`in2lambda-agent gate ci-corpus/targets --filters ci-corpus/filters` is what CI +runs, over the one target the repository holds. The private targets are run the +same way, by the same command, over the tree on the machine that holds them. """ -import json -import shutil -from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Optional, Sequence -from in2lambda_agent import corpus +from in2lambda_agent import targets +from in2lambda_agent.model import Backend from in2lambda_agent.settings import Settings DEFAULT_CACHE_DIR = Path.home() / ".cache" / "in2lambda-agent" """Where the gate reads the OCR of each PDF. The directory is outside every worktree, so a PDF converted on one branch is not converted again on the next. -`corpus` on its own keeps its cache under the directory the user ran from.""" - -RANK = {"built": 0, "build refused": 1, "faulted": 2, "skipped": 3} -"""How bad each outcome is. `skipped` is the worst of them: a document that -faults was at least read, and one that is skipped was not. Anything else is a -document that did not run either — a spec gone, a file unreadable, a document -the sweep no longer finds — and ranks with `skipped`.""" - -MISSING = "missing" -"""What a recorded document the sweep no longer finds is compared as.""" - - -def worse(current: str, recorded: str) -> bool: - """Whether a document did worse this run than the baseline records. - - A document that stops being read is a regression whatever it did before, - which is the case a baseline of nothing but faults rests on: widen what - counts as a solutions file and a document leaves the sweep as `skipped` - without a single count changing. - - Args: - current: What the document did this run. - recorded: What the baseline records it doing. - - Returns: - Whether it is a regression. - """ - return RANK.get(current, 3) > RANK.get(recorded, 3) - - -@dataclass -class Folder: - """One folder of a corpus, as the baseline holds it. - - Attributes: - root: The corpus directory the folder is under, read relative to the - directory the baseline file is in. `ci-corpus` for the committed - corpus; an absolute path for a corpus outside the repository. - suffixes: The file suffixes that are documents in the folder: `pdf` for - a folder of scans, `tex` or `docx` for sources. - built: How many documents built when the baseline was recorded, or None - where no count is recorded. - documents: What each document did when the baseline was recorded, by - its path under the corpus root. `root` and `suffixes` are written - by hand; `built` and `documents` are what `--record` writes. - """ - - root: Path - suffixes: list[str] - built: Optional[int] = None - documents: dict[str, str] = field(default_factory=dict) - - -@dataclass -class Regression: - """One document that did worse this run than the baseline records. - - Attributes: - document: Its path under the corpus root, as the baseline names it. - recorded: What the baseline records it doing. - current: What it did this run, or `missing` where the sweep no longer - finds it. - reason: What the sweep said about it, where it said anything. - """ - - document: str - recorded: str - current: str - reason: str = "" - - -@dataclass -class Baseline: - """The file the gate compares a sweep against. - - Attributes: - specs: The tree the folders' specs are kept in, read relative to the - directory the baseline file is in. Folder `A/B` reads - `/A/B/in2lambda-spec.yaml`. - folders: The folders to run, by their path under their own root. - directory: The directory the baseline file was read from. `specs` and - a relative `root` are read from there, so the command gives the - same run whichever directory it is run in. The paths themselves are - held as they are written, so `--record` writes them back unchanged. - """ - - specs: Path - folders: dict[str, Folder] - directory: Path = Path(".") - - -@dataclass -class Summary: - """What one folder did this run, beside what the baseline records. - - Attributes: - built: How many documents built. - counts: How many did each other thing, by outcome. - recorded: How many built when the baseline was recorded, or None where - no count is recorded. - regressions: The documents that did worse than the baseline records. - """ - - built: int = 0 - counts: dict[str, int] = field(default_factory=dict) - recorded: Optional[int] = None - regressions: list[Regression] = field(default_factory=list) - - @property - def failed(self) -> bool: - """Whether this folder fails the gate.""" - if self.recorded is not None and self.built < self.recorded: - return True - return bool(self.regressions) - - -@dataclass -class Report: - """The whole run: one summary per folder, in the baseline's order.""" - - folders: dict[str, Summary] = field(default_factory=dict) - - @property - def failed(self) -> bool: - """Whether any folder fails the gate.""" - return any(one.failed for one in self.folders.values()) - - -def read_baseline(path: Path) -> Baseline: - """Reads a baseline file. - - Args: - path: The JSON file. The paths it names are read from the directory it - is in. - - Returns: - The baseline. - """ - written = json.loads(Path(path).read_text(encoding="utf-8")) - return Baseline( - directory=Path(path).parent, - specs=Path(written["specs"]), - folders={ - name: Folder( - root=Path(one["root"]), - suffixes=list(one["suffixes"]), - built=one.get("built"), - documents=dict(one.get("documents", {})), - ) - for name, one in written["folders"].items() - }, - ) - - -def write_baseline(baseline: Baseline, path: Path) -> None: - """Writes the baseline back, counts and all. - - Args: - baseline: What to write. - path: The JSON file, which is overwritten. - """ - written = { - "specs": baseline.specs.as_posix(), - "folders": { - name: { - "root": one.root.as_posix(), - "suffixes": one.suffixes, - "built": one.built, - "documents": one.documents, - } - for name, one in baseline.folders.items() - }, - } - Path(path).write_text(json.dumps(written, indent=2) + "\n", encoding="utf-8") +`targets` on its own keeps its cache under the directory the user ran from.""" def run( - baseline: Baseline, + root: Path, *, - record: bool = False, - cache: Path, + paths: Sequence[Path] = (), + filters: Path, work: Path, + cache: Path = DEFAULT_CACHE_DIR, settings: Optional[Settings] = None, -) -> Report: - """Replays every folder the baseline names and checks what each one did. + backend: Optional[Backend] = None, +) -> list[targets.Result]: + """Replays every target under a root and compares each with its export. - Nothing is written into the repository or into a corpus: each folder is - swept into its own directory under `work`, and the table is written there. + Nothing is written into the repository or into the targets: each target's + set is written under `work`. Args: - baseline: The folders to run and what to compare against. Its `specs` - and each relative `root` are read from the directory it was read - from. In record mode this run's counts and outcomes replace them. - record: Take this run as the new baseline rather than checking it. - cache: Where the OCR of each PDF is kept. The directory is shared - between worktrees, so Mathpix converts each PDF once. - work: Where the folders are copied to be run. + root: The directory the targets are under. + paths: Folders under it to run, relative to it; all of it if empty. + filters: The tree the targets' filters and saved replies are kept in. + work: Where each target's set is written, under the target's own name. + cache: Where the OCR of each PDF is kept, shared between worktrees. settings: The environment the runs have available. + backend: The backend the adjudication calls are made to, chosen from + the settings if absent. Returns: - One summary per folder. + One result per target, in the order they ran. """ - work = Path(work) - # A sweep writes a record of each run beside the spec it read, and the - # record of a gate run is nobody's: it would land in the repository, under - # the specs the gate exists to replay. So the specs are copied under `work` - # and read from there. - specs = work / "specs" - # Fresh each run, so that a spec taken out of the tree is gone from the - # copy the sweep reads rather than left over from the run before. - shutil.rmtree(specs, ignore_errors=True) - # Joining an absolute path to the baseline's directory returns the absolute - # path, so a private corpus named by its path on one machine is unchanged. - written = baseline.directory / baseline.specs - if written.is_dir(): - shutil.copytree(written, specs) - report = Report() - for name, folder in baseline.folders.items(): - rows = corpus.sweep( - baseline.directory / folder.root, - paths=[Path(name)], - suffixes=folder.suffixes, - results=work / name / "results.csv", - work=work / "work", - specs=specs, - replay=True, - cache=cache, - settings=settings, - ) - summary = Summary(recorded=folder.built) - outcomes = {row.source: row.outcome for row in rows} - reasons = {row.source: row.reason for row in rows} - for outcome in outcomes.values(): - if outcome == "built": - summary.built += 1 - else: - summary.counts[outcome] = summary.counts.get(outcome, 0) + 1 - if record: - folder.built = summary.built - folder.documents = outcomes - # A recording run reports what this sweep did, so it fails nothing. - summary.recorded = summary.built - else: - for document, was in folder.documents.items(): - now = outcomes.get(document, MISSING) - if worse(now, was): - summary.regressions.append( - Regression(document, was, now, reasons.get(document, "")) - ) - report.folders[name] = summary - return report - - -def folder_line(name: str, summary: Summary) -> str: - """The gate's line for a folder: what it did, beside what is recorded. - - A folder whose documents all did as well as recorded is one line. Each - document that did worse adds an indented line of its own beneath it, so - that a failure names the document rather than only the count. - - Args: - name: The folder, as the baseline names it. - summary: What it did. - - Returns: - The line, and a line per regression beneath it. - """ - counts = " ".join( - f"{outcome} {summary.counts.get(outcome, 0)}" - for outcome in ("faulted", "build refused", "skipped") - ) - other = sorted(set(summary.counts) - {"faulted", "build refused", "skipped"}) - recorded = ( - "not recorded" - if summary.recorded is None - else f"baseline built {summary.recorded}" - ) - return "\n".join( - [ - f"{name:<40} built {summary.built} {counts}" - + "".join(f" {outcome} {summary.counts[outcome]}" for outcome in other) - + f" ({recorded})", - *( - f" worse {one.document} {one.recorded} -> {one.current}" - + (f": {one.reason}" if one.reason else "") - for one in summary.regressions - ), - ] + return targets.run( + root, + paths=paths, + filters=filters, + out_dir=work, + cache_dir=cache, + settings=settings, + backend=backend, + fresh=False, + replay=True, ) diff --git a/in2lambda_agent/model.py b/in2lambda_agent/model.py index 2503ac5..b609e3f 100644 --- a/in2lambda_agent/model.py +++ b/in2lambda_agent/model.py @@ -1,9 +1,9 @@ """One model call, with tools, behind one interface. -The agent makes three kinds of call — write a spec from the numbered source, fix -a validation report using the package's draft commands as tools, and ask what a -page shows that its OCR does not — and all three are the same shape: a system -prompt, a user prompt, some tools, some page images, one final text back. That +The agent makes four kinds of call — read a document as a set of questions, write +route B's Lua filter, adjudicate the fields the two routes word differently, and +ask what a page shows that its OCR does not — and all four are the same shape: a +system prompt, a user prompt, some tools, some page images, one final text back. That shape is `Backend.call`, and this is the only module that imports a provider SDK. Callers ask `choose_backend` for a backend and never learn which one they got. @@ -50,7 +50,7 @@ REQUEST_TIMEOUT = 300.0 # Claude Code's own tools, named for `disallowed_tools`. The agent gives the -# model the tools each call needs and no others: a spec call has none, and a +# model the tools each call needs and no others: a conversion call has none, and a # call that could run Bash or Read on the paths its prompt names spends its # turns reading the corpus. `tools=[]` alone does not switch them off — the SDK # sends it as `--tools ""`, and the run that recorded `error_max_turns` on @@ -133,15 +133,7 @@ class ModelUnavailable(RuntimeError): class ModelError(RuntimeError): """A call was made and did not finish: the provider stopped it, or the - model asked for tools until the round limit and never answered. - - Attributes: - stage: Which of the agent's calls this was — `spec` or `fix` — set by - the pipeline and read by the corpus sweep, which names it in the - row's outcome. Empty where nothing set it. - """ - - stage: str = "" + model asked for tools until the round limit and never answered.""" def _encoded(image: bytes) -> str: diff --git a/in2lambda_agent/ocr.py b/in2lambda_agent/ocr.py index c4fb7dc..db99956 100644 --- a/in2lambda_agent/ocr.py +++ b/in2lambda_agent/ocr.py @@ -16,6 +16,9 @@ SOURCE_NAME = "source.md" MEDIA_NAME = "media" +DEFAULT_CACHE_DIR = Path(".in2lambda-agent") +"""Where the OCR of each PDF is kept, under the directory the user ran from.""" + @dataclass class OcrResult: @@ -75,9 +78,9 @@ def ocr_pdf( markdown = entry / SOURCE_NAME media = entry / MEDIA_NAME - # A fresh pass restarts the pipeline for this document, so the whole entry - # goes: anything a later stage comes to keep beside source.md — a draft, a - # spec run, a report — belongs to the pass that made it, not to this one. + # A fresh pass restarts the conversion of this document, so the whole entry + # is deleted: a file a later step keeps beside source.md belongs to the pass + # that made it. shutil.rmtree(entry, ignore_errors=True) # Built beside the entry and renamed into place, so a pass that fails part diff --git a/in2lambda_agent/package.py b/in2lambda_agent/package.py deleted file mode 100644 index a9d680a..0000000 --- a/in2lambda_agent/package.py +++ /dev/null @@ -1,683 +0,0 @@ -"""Every call the agent makes into in2lambda, in the agent's own shapes. - -in2lambda does every deterministic step and every write, and this is the only -module that knows how those steps are called, so that the pipeline reads as the -design spec's diagram rather than as someone else's API. - -The commands are the Python functions the `in2lambda` CLI itself calls, each -taking the path of the draft they act on rather than the working directory: - - in2lambda.source.add(files, start_over) -> Path # the draft written - in2lambda.source.show(draft) -> str # numbered, with block ids - in2lambda.draft.execute( - in2lambda.draft.spec_command(spec, by, draft), draft) -> str - in2lambda.draft.execute( - {"command": name, "args": {...}, "by": by}, draft) -> str - in2lambda.draft.report.validate(draft) -> list[Finding] - in2lambda.draft.export.render(draft, output_dir) -> list[Path] - in2lambda.draft.export.build(draft, output_dir) -> Path - -A draft lives beside the file it was frozen from and is named after it: -`source add sheet.tex` writes `sheet.draft.json` next to it, and every command -after it is given that file. A spec is named relative to the directory the draft -is in, since that is what the draft's log records having run. -""" - -import json -import re -import warnings -from dataclasses import dataclass, field -from os.path import relpath -from pathlib import Path -from typing import Any, Optional - -import in2lambda.draft -import in2lambda.draft.export -import in2lambda.draft.report -import in2lambda.source -import in2lambda.spec -from in2lambda.source import SourceError - -# Who ran the command, as the draft's log records it. The agent is one author -# whichever model wrote the spec it is running. -BY = "in2lambda-agent" - -DRAFT_SUFFIX = in2lambda.source.DRAFT_SUFFIX -"""What a frozen source's draft is named with, beside the source itself.""" - -draft_of = in2lambda.source.draft_of -"""Where the draft of a document goes, for naming one without freezing it.""" - -ERROR = in2lambda.draft.report.ERROR -"""The level of a finding a build refuses over. Anything else it says and goes -on past, which is what makes a warnings-only report one to build.""" - -COMMANDS = ( - "mark ignore", - "question add", - "part add", - "question solution", - "field replace", - "split block", - "field set", - "part solution", -) -"""The draft commands a report is fixed with, named as the log names them.""" - -RENDERED = re.compile(r"question_(\d+)_") -"""How in2lambda names the PDF it writes for a question: the question's place in -the set, counting from zero, so that a stack of them reads in order.""" - - -class SpecRejected(ValueError): - """in2lambda would not run a spec, and says why.""" - - -class CommandRefused(ValueError): - """in2lambda would not run a draft command, and says why.""" - - -class BuildRefused(ValueError): - """in2lambda would not write a validated draft out, and says why.""" - - -@dataclass -class Coverage: - """What a spec run made of the source, which is what a spec is judged on. - - Attributes: - layout: The layout the spec chose, which pairs solutions to parts. - blocks: How many blocks the frozen source has. - fields: How many fields each layer wrote, keyed by layer number. - ignored: How many blocks the spec's `ignore` selector matched. - unassigned: The ids of blocks in no field and not ignored. - dropped: One finding per ignored block whose lines hold an image, as - `ignored_images` reports them. The set is built without those - images, and nothing in2lambda checks says so. - """ - - layout: str - blocks: int - fields: dict[int, int] = field(default_factory=dict) - ignored: int = 0 - unassigned: list[str] = field(default_factory=list) - dropped: list["Finding"] = field(default_factory=list) - - def __str__(self) -> str: - """The one line the coverage stage prints.""" - written = ", ".join( - f"{count} fields at layer {layer}" - for layer, count in sorted(self.fields.items()) - ) - left = ", ".join(self.unassigned) if self.unassigned else "none" - line = ( - f"{self.layout}: {self.blocks} blocks, {written or 'no fields'}, " - f"{self.ignored} ignored, {left} unassigned" - ) - if self.dropped: - images = ", ".join(_where(one.field, one.ranges) for one in self.dropped) - plural = "image" if len(self.dropped) == 1 else "images" - line += f"; {len(self.dropped)} {plural} dropped: {images}" - return line - - -@dataclass -class QuestionInfo: - """One question of a draft, as a reviewer is given it. - - Attributes: - key: The question, by the key of its fields: `q2`. - layer: The highest layer any of its fields came from, counting an - edited field as layer 4. 3 or 4 means something other than the - spec wrote part of it, which is what sample review looks at first. - ranges: Every range of the frozen source its fields were copied from, - in order, so that the reviewer can read the question against it. - """ - - key: str - layer: int - ranges: list[list[int]] - - -@dataclass -class Finding: - """One thing the checks found, as the report writes it. - - Attributes: - check: Which check found it. `problem` is in2lambda's own validation of - the set the draft describes — the maths delimiters, KaTeX, the - images, the compile — reported against the field it is written in. - level: `error` or `warning`. A build refuses over an error and says a - warning and goes on past it. - field: The block id or field key it is about, which is what a command - fixing it names. - ranges: The lines in question, as `[[start, end], ...]`. - message: A sentence naming all of that, which is what the model is given. - """ - - check: str - level: str - field: str - ranges: list[list[int]] - message: str - - -@dataclass -class Report: - """What the checks found in a draft. - - `clean` is whether the draft can be built, not whether the checks found - nothing: a report holding only warnings — a part whose solution is not on - the sheet — is one in2lambda builds, saying each warning as it goes. So a - warnings-only report is clean, `errors` is empty and `warnings` is not. - """ - - clean: bool - errors: list[str] - warnings: list[str] = field(default_factory=list) - findings: list[Finding] = field(default_factory=list) - - -def is_document(path: Path) -> bool: - """Whether a file is a document of its own rather than input to one. - - A tex file with no `\\begin{document}` is a fragment: a TikZ source under a - `figures/` folder, or a preamble a sheet inputs. Frozen and built it is a set - of one question made of a drawing, which is not what the corpus holds it for. - The other suffixes have no such marker, and every file of them is a document. - """ - if path.suffix.lower() != ".tex": - return True - return r"\begin{document}" in path.read_text(encoding="utf-8", errors="replace") - - -def said(report: Report) -> str: - """The validate line of a report nothing stops: the warnings, or nothing.""" - if not report.warnings: - return "nothing to report" - return "; ".join(report.warnings) + " — warnings, building" - - -def froze(draft: Path, solutions: Optional[str] = None) -> str: - """The freeze line of a draft: the file, and what else was frozen into it. - - Args: - draft: The draft that was written. - solutions: The name of the solutions document, where the draft holds - one as its second source. - """ - if not solutions: - return str(draft) - return f"{draft}, with {solutions} as source 2" - - -def source_add(source: Path, *more: Path) -> Path: - """Freezes one or more source documents into the draft beside the first. - - Always from the beginning: the agent's run owns the draft it writes, so a - second run over the same file is a second run and not a continuation of - the first one's fields. - - Args: - source: The markdown, tex or docx file to freeze. The draft is named - after it, and its blocks are `b1` onwards. - more: Further documents to freeze into the same draft, in the order - they are to be numbered: a sheet's solutions written as a file of - their own. The blocks of the second source are `2/b1` onwards. - - Returns: - The `FILE.draft.json` that was written, which every command below is - given. - - Raises: - SourceError: pandoc or panflute is missing, the files are not all in - one directory, or a file cannot be read. - """ - return in2lambda.source.add([str(source), *(str(one) for one in more)], True) - - -def source_show(draft: Path) -> str: - """The frozen markdown, numbered, with block ids in the margin. - - Args: - draft: The draft file. - - Returns: - What the model is shown to write a spec from. A draft of two sources - heads each with `Source N: NAME`, and the ids of the second source's - blocks carry its number: `2/b1`. - - Raises: - SourceError: there is no draft there, or its source has moved on. - """ - return in2lambda.source.show(str(draft)) - - -def spec_run(draft: Path, spec: Path) -> Coverage: - """Runs a spec over a frozen source, filling the draft's layer 1 fields. - - Args: - draft: The draft file. - spec: The spec file, wherever it is kept. - - Returns: - What the run made of the source. - - Raises: - SpecRejected: the spec cannot be read, or one of its selectors claims - lines another has already claimed. - """ - # Relative to the draft, which is how the log names a file: an absolute - # path would record this machine rather than the run. - name = relpath(spec, Path(draft).parent) - try: - layout = in2lambda.spec.load(Path(spec).read_bytes()).layout - in2lambda.draft.execute( - in2lambda.draft.spec_command(name, BY, str(draft)), str(draft) - ) - except SourceError as error: - raise SpecRejected(str(error)) from None - - found = _frozen(draft) - coverage = Coverage(layout=layout, blocks=blocks(draft)) - for key, written in found["fields"].items(): - if key.endswith(".ignore"): - coverage.ignored += 1 - else: - layer = written["layer"] - coverage.fields[layer] = coverage.fields.get(layer, 0) + 1 - coverage.unassigned = [ - finding["field"] for finding in in2lambda.draft.report.uncovered(found) - ] - coverage.dropped = ignored_images(draft) - return coverage - - -def ignored_images(draft: Path) -> list[Finding]: - """The blocks a spec marked ignore whose lines hold an image. - - A figure belongs to the question or part it illustrates. Mathpix writes a - figure as one paragraph, the image line and then its caption, so an `ignore` - selector matching the caption's `Figure n:` marks the image ignored and the - set is built without it. in2lambda's checks report nothing about an ignored - block, so the agent reads the ignored blocks back and reports each image. - - Args: - draft: The draft file, after a spec has been run over it. - - Returns: - One finding per ignored block whose lines hold a markdown image, in the - order the blocks appear in the sources. The message follows the wording - of in2lambda's own coverage findings, so that a spec-writing prompt - reads the same for either finding. A source whose bytes are not text — - a docx, frozen as itself — has no such block to report. - """ - found = _frozen(draft) - read: dict[int, Optional[list[str]]] = {} - - def source_lines(number: int) -> Optional[list[str]]: - """The lines of one source, read the first time a block of it is ignored.""" - if number not in read: - path = Path(draft).parent / found["sources"][number]["source"] - read[number] = _source_lines(path) - return read[number] - - dropped = [] - for key, written in found["fields"].items(): - if not key.endswith(".ignore"): - continue - block = key[: -len(".ignore")] - # `2/b3` is the second source's block; `b3` is the first source's. - number, _, _ = block.rpartition("/") - lines = source_lines(int(number) - 1 if number else 0) - if lines is None: - continue - ranges = written["ranges"] - held = "\n".join("\n".join(lines[start - 1 : end]) for start, end in ranges) - if "![" in held: - dropped.append( - Finding( - check="coverage", - level=ERROR, - field=block, - ranges=ranges, - message=f"{_where(block, ranges)} holds an image and is " - "marked ignore.", - ) - ) - return sorted(dropped, key=lambda one: (one.field.rpartition("/")[0], one.ranges)) - - -def _where(field: str, ranges: list[list[int]]) -> str: - """A block and the lines it covers, as a finding's message names one. - - Returns `b10 (lines 29-30)`, which is how in2lambda's own findings name one. - """ - covered = ", ".join(f"{start}-{end}" for start, end in ranges) - return f"{field} (lines {covered})" - - -def _source_lines(path: Path) -> Optional[list[str]]: - """A frozen source read as text, or None where its bytes are not text. - - A docx is frozen as itself and is a zip, and a tex sheet of a real set need - not be UTF-8, so a source is decoded the way `is_document` decodes one and - a source holding a NUL byte is left alone: it has no line of markdown to - find an image in. - """ - raw = path.read_bytes() - if b"\x00" in raw: - return None - return raw.decode("utf-8", errors="replace").splitlines() - - -def blocks(draft: Path) -> int: - """How many blocks a draft's frozen source has. - - A spec run reports this count in its coverage. in2lambda runs no spec it - refuses, so a caller reporting how many blocks a refused spec left in no - field counts every block of the source. - - Args: - draft: The draft file. - - Returns: - The count. - """ - return sum(len(one["blocks"]) for one in _frozen(draft)["sources"]) - - -def _frozen(draft: Path) -> dict[str, Any]: - """A draft read off disk, as in2lambda writes one.""" - return json.loads(Path(draft).read_text()) - - -def command(draft: Path, name: str, args: dict[str, Any], by: str = BY) -> str: - """Runs one draft command, which is how every fix reaches a draft. - - in2lambda writes the field, records the command in the draft's log as it - applies it, and decides which layer the field came from — 3 for text copied - out of the frozen source, 4 for text typed out. So a fix leaves its whole - record without the agent keeping one of its own, and this is the only place - that knows what a log entry looks like. - - Args: - draft: The draft file. - name: One of COMMANDS. - args: The command's arguments, as the log records them. - by: Who asked for it, as the draft's log records it: the agent, or the - reviewer whose own edit this is. - - Returns: - What the command wrote: the key of the field, or the ids a split made. - - Raises: - CommandRefused: in2lambda would not run it — a block that is not there, - lines another field has taken, wording that is not in the field once. - """ - try: - return in2lambda.draft.execute( - {"command": name, "args": args, "by": by}, str(draft) - ) - except SourceError as error: - raise CommandRefused(str(error)) from None - - -def field_value(draft: Path, key: str) -> Optional[str]: - """The text a draft holds for one field. - - Args: - draft: The draft file. - key: The field's key: `q1.text`. - - Returns: - The field's text, or None where the draft holds no field of that key. - A block marked `ignore` is a field whose value is `true` rather than - text, and it has no text to return either. - """ - written = _frozen(draft)["fields"].get(key) - value = None if written is None else written["value"] - return value if isinstance(value, str) else None - - -def command_log(draft: Path) -> list[dict[str, Any]]: - """Every command the draft records having been built by, in the order they ran. - - Args: - draft: The draft file. - - Returns: - One entry per command, each `{"command", "args", "by"}`. - """ - return _frozen(draft)["log"] - - -def fix_log(draft: Path) -> list[dict[str, Any]]: - """The commands the fixing rounds ran, without the spec run before them. - - A replay runs the set's spec itself and then these, so the `spec run` entry - is left out: it names the spec file relative to the draft, and the replay's - draft is in the work directory rather than where the sweep's draft was. - - Args: - draft: The draft file. - - Returns: - One entry per command, each `{"command", "args", "by"}`, in the order - they ran. An entry's arguments are the block ids, field keys and line - ranges the command named, together with the wording a `field replace` - or a typed field spells out. The draft's `fields`, which hold every - field's captured text, are not here. - """ - return [one for one in command_log(draft) if one["command"] != "spec run"] - - -def replay(draft: Path, commands: list[dict[str, Any]]) -> int: - """Runs a saved list of commands over a draft, in the order they were saved. - - Args: - draft: The draft file, with the set's spec already run over it. - commands: The entries `fix_log` wrote. - - Returns: - How many commands ran. - - Raises: - CommandRefused: in2lambda would not run one of them. The message names - which command it was, and the commands before it have been applied. - """ - for index, entry in enumerate(commands, start=1): - try: - command(draft, entry["command"], entry["args"], entry.get("by", BY)) - except CommandRefused as error: - raise CommandRefused( - f"command {index} of {len(commands)}, {entry['command']}: {error}" - ) from None - return len(commands) - - -def questions(draft: Path) -> dict[str, QuestionInfo]: - """The questions a draft holds, in the order they are numbered. - - A question is its fields — `q2.text`, `q2.p1.solution` — so this is what - reading the draft's fields by their keys says about each one: where in the - source it was copied from, and whether anything past the spec wrote it. - - Args: - draft: The draft file. - - Returns: - One entry per question, keyed by `q1`, `q2`. - """ - found: dict[str, QuestionInfo] = {} - for key, written in _frozen(draft)["fields"].items(): - name = key.split(".")[0] - if not name.startswith("q"): - # A block marked ignore: `b3.ignore`, or `2/b3.ignore` for a block - # of a second frozen source, which is in no question either. What a - # block id looks like is in2lambda's, so this asks what the key is - # not rather than what it is. - continue - info = found.setdefault(name, QuestionInfo(name, 0, [])) - # An edit is layer 4 work whatever layer wrote the field first: - # in2lambda leaves a replaced field quoting the lines it came from and - # marks it as no longer saying what they say, which is the design - # spec's layer 1 field carrying a layer 4 edit. - info.layer = max(info.layer, 4 if written["edited"] else written["layer"]) - # A field a `literal` typed out is layer 4 with no source behind it, so - # it has no ranges to add: the listing's `none` is what it comes to. - info.ranges.extend(written.get("ranges") or []) - for info in found.values(): - info.ranges.sort() - return {key: found[key] for key in sorted(found, key=lambda key: int(key[1:]))} - - -def frozen_source(draft: Path) -> Path: - """The file the draft was frozen from, which the reviewer reads it against. - - Args: - draft: The draft file. - - Returns: - The first source, named from the directory the draft is in as the draft - itself names it. A draft the agent wrote has only the one. - """ - return Path(draft).parent / _frozen(draft)["sources"][0]["source"] - - -def render(draft: Path, out_dir: Path) -> dict[str, Path]: - """Writes each question of a draft as a PDF, for a reviewer to read. - - Args: - draft: The draft file. - out_dir: Where to write the PDFs. - - Returns: - The PDF written for each question, keyed as `questions` keys it. A - question the compiler gave up on while the rest rendered has no PDF and - is not in it, which is why the key comes from the file's name rather - than from its place in the list. - - Raises: - CommandRefused: the pages cannot be compiled — pandoc or xelatex is - missing, or in2lambda says why. - """ - try: - written = in2lambda.draft.export.render(str(draft), str(out_dir)) - except SourceError as error: - raise CommandRefused(str(error)) from None - keyed = {} - for path in written: - numbered = RENDERED.match(Path(path).name) - if numbered: - keyed[f"q{int(numbered.group(1)) + 1}"] = Path(path) - return keyed - - -def layers(draft: Path) -> dict[str, int]: - """How many fields each layer wrote, and how many of them were edited. - - The design spec's test plan asks per document for the share of fields from - each layer. Counts rather than shares: they diff cleanly between two runs, - and a share is one division away from them. - - Args: - draft: The draft file. - - Returns: - `{"layer1": n, ..., "layer4": n, "edited": n}`, always all five keys. - A block marked `ignore` is not a field and is in none of them, which is - what `Coverage.fields` counts too. - """ - fields = _frozen(draft)["fields"] - counted = {f"layer{number}": 0 for number in (1, 2, 3, 4)} - counted["edited"] = 0 - for key, written in fields.items(): - if key.endswith(".ignore"): - continue - counted[f"layer{written['layer']}"] += 1 - counted["edited"] += bool(written["edited"]) - return counted - - -def validate(draft: Path) -> Report: - """Checks a draft over and writes the report into it, as `build` requires. - - Args: - draft: The draft file. - - Returns: - Whether the draft can be built, and what the checks found. Only an error - stops a build: a report holding warnings alone is clean, and the build - says each of them and writes the set anyway. - - Raises: - SourceError: there is no draft there, or its source has moved on. - """ - # Named field by field rather than passed through, so that a check in2lambda - # grows later arrives here as a finding of the shape the fixer already reads. - findings = [ - Finding( - found["check"], - found["level"], - found["field"], - found["ranges"], - found["message"], - ) - for found in in2lambda.draft.report.validate(str(draft)) - ] - # An error is the only thing a build refuses over, so it is the only thing - # `clean` asks about: the warnings are said and gone past. - errors = [one.message for one in findings if one.level == ERROR] - return Report( - clean=not errors, - errors=errors, - warnings=[one.message for one in findings if one.level != ERROR], - findings=findings, - ) - - -@dataclass -class Built: - """A set in2lambda wrote, and what it warned as it wrote one. - - Attributes: - zip_path: The zip that was written. - warnings: What in2lambda warned while building, in the order it warned - each one, as the message alone. - """ - - zip_path: Path - warnings: list[str] - - -def build(draft: Path, out_dir: Path) -> Built: - """Writes a validated draft out as a Lambda Feedback set. - - in2lambda reports a warning-level finding through `warnings.warn`, which - Python prints to stderr with the line of in2lambda that raised it. This - function records each warning and returns it with the zip, so that the - caller decides what a reader sees. - - Args: - draft: The draft file. - out_dir: Where to write the set's folder and its zip. - - Returns: - The zip that was written and the warnings in2lambda said as it wrote - one. A report holding only warnings is one the build proceeds past, so - a draft with a part nothing answers still builds. - - Raises: - BuildRefused: the checks found an error in the draft, or it refers to an - image that is not beside it. - """ - try: - with warnings.catch_warnings(record=True) as said: - # A warning Python has shown once is not shown again by default, - # and a long-running harness builds more than one draft. - warnings.simplefilter("always") - zip_path = in2lambda.draft.export.build(str(draft), str(out_dir)) - except SourceError as error: - raise BuildRefused(str(error)) from None - return Built(zip_path, [str(one.message) for one in said]) diff --git a/in2lambda_agent/pair.py b/in2lambda_agent/pair.py index de054db..583013d 100644 --- a/in2lambda_agent/pair.py +++ b/in2lambda_agent/pair.py @@ -17,6 +17,9 @@ `target_documents` reads the other kind of folder, the one holding a single set. There is nothing to pair there, so the roles alone decide: the document whose name ends in `solutions` answers the other one, whatever either is called. + +`is_document` is the other half of reading a folder: which of its files is a +sheet at all, rather than a drawing a sheet inputs. """ import re @@ -40,6 +43,19 @@ """ +def is_document(path: Path) -> bool: + """Whether a file is a document of its own rather than input to one. + + A tex file with no `\\begin{document}` is a fragment: a TikZ source under a + `figures/` folder, or a preamble a sheet inputs. Converted it is a set of one + question made of a drawing, which is not what the corpus holds it for. The + other suffixes have no such marker, and every file of them is a document. + """ + if path.suffix.lower() != ".tex": + return True + return r"\begin{document}" in path.read_text(encoding="utf-8", errors="replace") + + def questions_stem(document: Path) -> Optional[str]: """The stem of the questions document a solutions document answers. diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py deleted file mode 100644 index 546c992..0000000 --- a/in2lambda_agent/pipeline.py +++ /dev/null @@ -1,877 +0,0 @@ -"""The run: source in, Lambda Feedback zip out. - -The stages are the design spec's pipeline. The agent acts at three of them — the -OCR pass, the model calls that write the set's spec, and the rounds that answer -what the checks found — and in2lambda does the rest: freezing the source, -running the spec over it, checking the draft and writing the zip. - -A spec that covers its source is layer 1 and builds with nothing more asked of -it. A draft the checks have something to say about gets the rounds: up to N -model calls, each with in2lambda's draft commands as its tools, writing fields -at layers 3 and 4 until the checks are quiet or the limit runs out, and then the -report and no zip. A round that leaves only what it was given ends the run there -rather than using the limit up: a finding no range of the source answers — a part -whose solution is not on the sheet — is reported, not invented, and the next -round would be the same prompt over the same report. A spec saved from an -earlier sheet is written again before any of that where the checks fault the -draft it filled, since a spec that covers the set is worth more than a field -repaired in one sheet of it; that rewrite is layer 1, and is not one of the -rounds. `spec.iterate_spec` is what writes a spec, in up to `tries` calls, and -what the run goes on with is the one of them that covered the set best. - -A run asked for a review stops once the checks are quiet: it renders the -questions the reviewer is to see, leaves a record of them in the cache, and -builds nothing. `resume` is the other half of that run — one call per verdict, -out of another process — and it is what finally builds, once every question the -reviewer was shown has been approved. -""" - -import json -import random -import re -import shutil -from dataclasses import dataclass, field -from pathlib import Path -from typing import Callable, Optional - -from in2lambda_agent import package, pair -from in2lambda_agent.fix import RoundResult, fix_round, summary, unrepaired -from in2lambda_agent.mathpix import MathpixClient -from in2lambda_agent.model import ( - Backend, - ModelError, - ModelUnavailable, - Usage, - choose_backend, -) -from in2lambda_agent.ocr import MEDIA_NAME, cached, ocr_pdf -from in2lambda_agent.review import RECORD, Question, Review, choose -from in2lambda_agent.settings import Settings -from in2lambda_agent.spec import ( - RECORD_NAME, - Previous, - Second, - SpecTry, - iterate_spec, - record_run, - spec_path, -) - -# Where the OCR of each PDF is kept, under the directory the user ran from. -DEFAULT_CACHE_DIR = Path(".in2lambda-agent") - -# Where the copy of the set's other document, which each candidate spec is run -# over, is kept under the cache directory. -SECOND_NAME = "second" - -# A markdown image whose file is in the OCR's media folder, as far as the folder -# name: `![a plot](media/plot.png)`. What a second source's images are renamed -# by when they are copied beside the first source's. -MEDIA_REFERENCE = re.compile(rf"(!\[[^\]]*\]\(){MEDIA_NAME}/") - -REVIEW_MODES = ("none", "sample", "per-question") - - -@dataclass -class StageResult: - """What one stage did, as one line of output.""" - - name: str - message: str - - -@dataclass -class RunResult: - """What a run did, in order, what it covered, and the zip it wrote. - - `draft` and `reused` are what the record already says, on the result as - well, so that a harness running many documents can read a run's provenance - and its spec reuse without reading the record back off disk; and `clean` is - whether the draft can be built, which `zip_path` does not answer, since a - build in2lambda refuses leaves a clean report and no zip. - - `reason` is what the run has to say for itself: why it ended without a zip, - in the words of the stage that stopped it — the refusal, or the first error - the checks were still finding — or, where it did build, the warnings it - built past. The stage lines say as much, but they are printed and gone; this - is what a harness has to write down. - - `on_stage` is called with each stage as the run adds it. A caller that - reads `stages` reads them once the run has returned; the local web page - sends each line to the browser while the run is still going. - """ - - stages: list[StageResult] = field(default_factory=list) - zip_path: Optional[Path] = None - coverage: Optional[package.Coverage] = None - usage: Usage = field(default_factory=Usage) - tries: list[SpecTry] = field(default_factory=list) - second: Optional[Second] = None - rounds: list[RoundResult] = field(default_factory=list) - review: Optional[Review] = None - draft: Optional[Path] = None - reused: bool = False - clean: bool = False - reason: str = "" - on_stage: Optional[Callable[[StageResult], None]] = field( - default=None, compare=False, repr=False - ) - - def add_stage(self, name: str, message: str) -> None: - """Records one stage's line and calls `on_stage` with it. - - Args: - name: The stage, as the printed line names it. - message: What the stage did, as the printed line says it. - """ - stage = StageResult(name, message) - self.stages.append(stage) - if self.on_stage is not None: - self.on_stage(stage) - - -def run( - source: Path, - *, - out_dir: Path, - settings: Settings, - spec: Optional[Path] = None, - commands: Optional[Path] = None, - review: str = "none", - rounds: int = 3, - tries: int = 3, - sample: int = 3, - cache_dir: Path = DEFAULT_CACHE_DIR, - fresh_ocr: bool = False, - mathpix: Optional[MathpixClient] = None, - backend: Optional[Backend] = None, - rng: Optional[random.Random] = None, - on_stage: Optional[Callable[[StageResult], None]] = None, -) -> RunResult: - """Drives in2lambda over one source file. - - Args: - source: The question file to convert, or the solutions file beside it, - which runs the questions file it answers. - out_dir: Where in2lambda writes the set's JSON folder and zip. - settings: The environment the run has available. - spec: The set's spec file, when it is not the one beside the source. - commands: A file holding the commands an earlier run's fixing rounds - ran, as `package.fix_log` wrote them. They are run after the spec, - which is how a replay rebuilds a draft the rounds repaired without - calling a model. A path that is not a file runs the spec alone. - review: One of REVIEW_MODES. - rounds: The round limit, N in the design spec: how many model calls may - answer what the checks found before the run stops without a zip. - tries: How many specs may be written before the best of them is saved - for the set. - sample: How many questions a review in sample mode shows. - cache_dir: Where the OCR of each PDF is kept, and where a review that - is waiting to be answered is left. - fresh_ocr: Convert a PDF again even if it is already cached. - mathpix: The client to convert with, built from the settings if absent. - backend: The model backend to write the spec with, chosen from the - settings if absent and asked for only when a spec must be written. - rng: What fills a sample out, so that a test can fix which questions it - picks. - on_stage: Called with each stage as the run adds it, for a caller that - shows the lines while the run is still going. - - Returns: - Each stage's line, what the spec covered, what the model calls cost, - what each fixing round did, the zip where one was written — or the - review waiting to be answered, where the run stopped for one — and why - where no zip was written. - - Raises: - MathpixError: If a PDF cannot be converted, MissingCredentials among - them when a conversion is needed and the run has no Mathpix - credentials. A PDF already in the cache needs none. - ModelUnavailable: If a spec must be written and no backend can run. - ModelError: If a call did not finish, with `stage` naming which — the - spec call or a fixing round. - BadSpec: If what the model answers with is not a spec. - SpecRejected: If in2lambda will not run the spec. - CommandRefused: If in2lambda will not run one of the saved commands. - SourceError: If in2lambda cannot freeze or check the source. - """ - # A relative --out means the directory the user ran from, whatever in2lambda - # does with the working directory along the way. - source = Path(source) - out_dir = Path(out_dir).resolve() - cache_dir = Path(cache_dir).resolve() - - # A sheet whose solutions are written as a file of their own is one run and - # one draft, named after the questions file. So the run is the questions - # file's from here on, whichever of the two the user named. - source, solutions = pair.of(source) - - # A solutions file with no questions file beside it is converted on its - # own. Its questions are the markers written above its solutions, which the - # spec prompt says so that the model writes `question` selectors for them. - alone = pair.questions_stem(source) if solutions is None else None - - # The set is the folder the user's file is in, so this is settled before - # OCR moves a PDF's markdown off into the cache. - saved = spec_path(source, spec) - - result = RunResult(on_stage=on_stage) - if alone is not None: - result.add_stage( - "pair", - f"no questions file named {alone}{source.suffix} beside " - f"{source.name}; converting the solutions alone", - ) - - # The rest of the pipeline reads markdown, so a PDF becomes markdown first. - frozen, _, message = _markdown( - source, cache_dir=cache_dir, settings=settings, mathpix=mathpix, fresh=fresh_ocr - ) - frozen_solutions = None - if solutions is not None: - frozen_solutions, solutions_media, said = _markdown( - solutions, - cache_dir=cache_dir, - settings=settings, - mathpix=mathpix, - fresh=fresh_ocr, - ) - message += f"; {said}" - # in2lambda freezes into one draft the documents of one directory, and - # the OCR of each PDF is cached in an entry named after its own hash. So - # the solutions markdown, and the images it refers to, are copied beside - # the questions markdown. - if frozen_solutions.parent != frozen.parent: - frozen_solutions = _copy_beside( - frozen_solutions, - solutions_media, - name=solutions.stem, - into=frozen.parent, - ) - result.add_stage("ocr", message) - - # The saved spec's own pass, where the set has one. A spec the checks fault - # is written again, and what that pass covered is what the first call is - # asked to improve on. - reused = saved.is_file() - previous = None - if reused: - draft = result.draft = package.source_add( - frozen, *([frozen_solutions] if frozen_solutions is not None else []) - ) - result.add_stage( - "freeze", - package.froze(draft, solutions.name if solutions is not None else None), - ) - result.add_stage("spec", f"reused {saved}") - result.coverage = package.spec_run(draft, saved) - result.add_stage("coverage", str(result.coverage)) - - if commands is not None and Path(commands).is_file(): - saved_commands = json.loads(Path(commands).read_text(encoding="utf-8")) - ran = package.replay(draft, saved_commands) - result.add_stage("replay", f"{ran} commands from {commands}") - - report = package.validate(draft) - # An image the saved spec marked ignore is a fault of the spec that - # in2lambda's checks say nothing about, so it sends the run into the - # rewrite loop as a faulted draft does. - dropped = [one.message for one in result.coverage.dropped] - if (report.clean and not dropped) or rounds < 1: - result.add_stage( - "validate", - package.said(report) if report.clean else "; ".join(report.errors), - ) - else: - result.add_stage( - "validate", - "; ".join(report.errors + dropped) + " — writing the set's spec again", - ) - previous = Previous( - text=saved.read_text(encoding="utf-8"), - coverage=result.coverage, - report=report, - ) - reused = False - - if not reused: - backend = backend or choose_backend(settings) - if (reason := backend.unavailable()) is not None: - raise ModelUnavailable(reason) - result.second = _second(source, cache_dir, solutions) - try: - draft, coverage, report, result.tries = iterate_spec( - frozen, - saved, - backend, - tries=tries, - on_stage=result.add_stage, - second=result.second, - previous=previous, - solutions=frozen_solutions, - solutions_name=solutions.name if solutions is not None else "", - solutions_only=alone is not None, - ) - except ModelError as error: - # Which call did not finish, for a caller that names it: a spec call - # and a fixing round both go to the same backend. - error.stage = "spec" - raise - result.draft = draft - result.coverage = coverage - for one in result.tries: - result.usage.input_tokens += one.usage.input_tokens - result.usage.output_tokens += one.usage.output_tokens - result.usage.seconds += one.usage.seconds - - # Layers 3 and 4, a round at a time. Reached only with a spec this run - # wrote, so the backend is the one that wrote it. - try: - report = _fix_rounds(draft, report, backend, rounds, result) - except ModelError as error: - error.stage = "fix" - raise - # What the corpus harness reads off the result rather than off the - # record: set here so that a run that stops for a review carries them - # too, since that return is above the record this run never writes. - result.clean = report.clean - result.reused = reused - if not report.clean: - # The first error left is the reason there is no zip. A warning is not - # one: the build goes past it, so what it says belongs in the reason of - # a run that built rather than in the reason one stopped. - result.reason = report.errors[0] - else: - # What the build will say and go on past, which the stage line prints - # and the harness's table keeps. - result.reason = "; ".join(report.warnings) - - if report.clean and review != "none": - # The run stops here: the questions the reviewer is to see, a record of - # them in the cache, and no zip until `resume` is told they are right. - # Every path in the record is absolute, as out_dir and cache_dir - # already are: the command that answers it is another process, run from - # wherever the reviewer happens to be, and a relative one would point - # at nothing from there. - waiting = Review( - mode=review, - count=sample, - source=str(source.resolve()), - spec=str(saved.resolve()), - out_dir=str(out_dir), - limit=rounds, - draft=str(draft.resolve()), - frozen=str(package.frozen_source(draft).resolve()), - reused=reused, - coverage=result.coverage, - usage=result.usage, - tries=result.tries, - second=result.second, - rounds=result.rounds, - ) - infos = package.questions(draft) - rendered, message = _render(draft, out_dir) - result.add_stage("render", message) - waiting.questions = [ - Question( - key=key, - pdf=str(rendered[key]) if key in rendered else None, - lines=infos[key].ranges, - ) - for key in choose(infos, review, sample, rng) - ] - waiting.save(cache_dir / RECORD) - result.review = waiting - result.add_stage("review", _asked(waiting, cache_dir)) - return result - - if report.clean: - result.add_stage("review", f"not asked for (mode {review})") - _build(draft, out_dir, result, report.warnings) - - record_run( - saved.parent / RECORD_NAME, - source, - reused=reused, - coverage=result.coverage, - usage=result.usage, - tries=result.tries, - second=result.second, - rounds=result.rounds, - ) - return result - - -def resume( - cache_dir: Path = DEFAULT_CACHE_DIR, - *, - verdict: str, - settings: Settings, - key: Optional[str] = None, - note: Optional[str] = None, - field: Optional[str] = None, - old: Optional[str] = None, - new: Optional[str] = None, - by: str = "reviewer", - backend: Optional[Backend] = None, - on_stage: Optional[Callable[[StageResult], None]] = None, -) -> RunResult: - """Answers the review a run left waiting, and builds once it is answered. - - Args: - cache_dir: Where the run left its review. - verdict: `approve`, `reject` or `edit`. - settings: The environment the run has available. - key: The question approved or rejected. - note: What a rejection says, which is what the fixing round is asked. - field: The field an edit changes, by its key: `q1.text`. - old: The wording that edit replaces, which is in the field once. - new: What it puts there instead. - by: Who the reviewer is, as the draft's log records their edit. - backend: The backend a rejection's fixing round calls, chosen from the - settings if absent and asked for only by a rejection. - on_stage: Called with each stage as the resume adds it, for a caller - that shows the lines of a rejection's fixing rounds while they run. - - Returns: - Each stage's line, and the zip where the last approval wrote one — - which the last approval does not where the checks fault the draft the - reviewer's own rounds and edits have left. - - Raises: - ReviewError: no review is waiting, or none of its questions is `key`. - ModelUnavailable: a rejection has no backend to answer its note with. - ModelError: a rejection's fixing round did not finish. - CommandRefused: in2lambda would not make the reviewer's edit. - """ - cache_dir = Path(cache_dir).resolve() - waiting = Review.load(cache_dir / RECORD) - draft = Path(waiting.draft) - # The result shares the review's usage and rounds rather than copying them, - # so that what a rejection's rounds cost is in the record that is saved - # below and in the run record the last approval writes. - result = RunResult( - coverage=waiting.coverage, - usage=waiting.usage, - tries=waiting.tries, - second=waiting.second, - rounds=waiting.rounds, - review=waiting, - draft=draft, - reused=waiting.reused, - on_stage=on_stage, - ) - - if verdict == "approve": - waiting.question(key).status = "approved" - if waiting.done: - # The design spec decides that build runs only after validate - # returns clean, and a reviewer's own rounds and edits have had - # the draft since the run last checked it. So the checks run - # again here, and a draft they fault is not built: the review - # stays waiting, and a rejection or an edit is what answers them. - result.add_stage("review", f"{key} approved, and that is all of them") - report = package.validate(draft) - waiting.errors = report.errors - result.clean = report.clean - result.add_stage( - "validate", - package.said(report) if report.clean else "; ".join(report.errors), - ) - if report.clean: - _build(draft, Path(waiting.out_dir), result, report.warnings) - if result.zip_path is None: - waiting.save(cache_dir / RECORD) - result.add_stage("review", _asked(waiting, cache_dir)) - return result - record_run( - Path(waiting.spec).parent / RECORD_NAME, - Path(waiting.source), - reused=waiting.reused, - coverage=waiting.coverage, - usage=waiting.usage, - tries=waiting.tries, - second=waiting.second, - rounds=waiting.rounds, - review=waiting.to_json(), - ) - (cache_dir / RECORD).unlink() - return result - waiting.save(cache_dir / RECORD) - result.add_stage("review", f"{key} approved\n{_asked(waiting, cache_dir)}") - return result - - if verdict == "reject": - question = waiting.question(key) - question.status = "rejected" - question.note = note - waiting.rejections.append({"key": key, "note": note}) - result.add_stage("review", f"{key} rejected: {note}") - if waiting.limit < 1: - # A run made with --rounds 0 has no round to answer the note with, - # so the question comes back unchanged. Saying so is the whole of - # what happens here: a backend is asked for only where a round will - # actually run, so a machine with no key can still record this. - result.add_stage( - "fix", - "no rounds left to answer the note with: the run was " - f"--rounds {waiting.limit}", - ) - report = package.validate(draft) - else: - backend = backend or choose_backend(settings) - if (reason := backend.unavailable()) is not None: - raise ModelUnavailable(reason) - # The note is a finding of its own: the checks are quiet, and it is - # what the round is for. Rounds after it answer what they leave. - try: - report = _fix_rounds( - draft, - package.validate(draft), - backend, - waiting.limit, - result, - instruction=f"The reviewer rejected {key}: {note}", - ) - except ModelError as error: - error.stage = "fix" - raise - relisted = [key] - else: - package.command( - draft, "field replace", {"field": field, "old": old, "new": new}, by=by - ) - waiting.edits.append({"field": field, "by": by}) - result.add_stage("review", f"{by} edited {field}") - report = package.validate(draft) - result.add_stage( - "validate", - package.said(report) if report.clean else "; ".join(report.errors), - ) - edited = field.split(".")[0] - relisted = [edited] if any( - one.key == edited for one in waiting.questions - ) else [] - - # What the checks make of the draft the rounds or the edit left, so that - # the listing says a draft that cannot be built cannot be built, rather - # than leaving the reviewer to find that out by approving it. - waiting.errors = report.errors - result.clean = report.clean - _relist(waiting, result, relisted) - waiting.save(cache_dir / RECORD) - result.add_stage("review", _asked(waiting, cache_dir)) - return result - - -def _markdown( - document: Path, - *, - cache_dir: Path, - settings: Settings, - mathpix: Optional[MathpixClient], - fresh: bool, -) -> tuple[Path, Optional[Path], str]: - """The markdown a document is frozen from, and the OCR stage's line for it. - - Args: - document: The file the user named, or the solutions file beside it. - cache_dir: Where the OCR of each PDF is kept. - settings: The environment the run has available. - mathpix: The client to convert with, built from the settings if absent. - fresh: Convert a PDF again even if it is already cached. - - Returns: - The markdown, which is the document itself where it is not a PDF, the - folder holding the images it refers to where OCR wrote any, and what - the OCR stage says about it. - - Raises: - MathpixError: If the PDF cannot be converted, MissingCredentials among - them. A PDF already in the cache is not converted and needs none. - """ - if document.suffix.lower() != ".pdf": - return document, None, f"not needed for {document.name}" - # The cache is asked before the client is built: a document converted once - # runs again with no credentials, which is what lets a worktree or a fork's - # CI job replay a corpus of PDFs it cannot pay for. - ocr = None if fresh else cached(document, cache_dir) - if ocr is None: - client = mathpix or MathpixClient.from_settings(settings) - ocr = ocr_pdf(document, cache_dir=cache_dir, client=client, fresh=fresh) - # A fresh pass is a restart: every stage below reads the new markdown. - if ocr.fresh: - return ocr.markdown, ocr.media, f"fresh pass, restarting from {ocr.markdown}" - return ocr.markdown, ocr.media, f"cached {ocr.markdown}" - - -def _copy_beside( - markdown: Path, media: Optional[Path], *, name: str, into: Path -) -> Path: - """Copies a second source, and the images it refers to, beside the first. - - The images cannot come across under the name their folder has, because each - PDF's OCR calls its own folder `media` and two sheets can each hold a - `plot.png`: the second copy would be the first one gone. So they arrive in a - folder named after their document, and the references in the copy are - rewritten to it. in2lambda resolves a reference from the folder the draft is - in, which is the folder copied into, so any name does. - - Args: - markdown: The markdown to copy. - media: The folder its images are in, where the document has any. - name: What the copies are named after: the solutions document's stem. - into: The folder the first source's markdown is in. - - Returns: - The copy, which is what the draft freezes as its second source. - """ - folder = f"{name}-{MEDIA_NAME}" - copied = into / f"{name}.md" - copied.write_text( - MEDIA_REFERENCE.sub( - lambda found: f"{found.group(1)}{folder}/", - markdown.read_text(encoding="utf-8"), - ), - encoding="utf-8", - ) - if media is not None and media.is_dir(): - shutil.copytree(media, into / folder, dirs_exist_ok=True) - return copied - - -def _fix_rounds( - draft: Path, - report: package.Report, - backend: Optional[Backend], - rounds: int, - result: RunResult, - instruction: Optional[str] = None, -) -> package.Report: - """The fixing rounds: the report and the source to the model, its commands - to the draft, and the checks again after each one. - - Args: - draft: The draft file. - report: What the checks found, which is what the rounds are to answer. - backend: The backend to call, already known to be available. - rounds: How many rounds there may be, from here. - result: The run so far, which each round adds its stage, its cost and - its record to. - instruction: A reviewer's note, which the first round answers as well - as the report — and which is reason for a round by itself, since a - rejection arrives with the checks already quiet. - - Returns: - What the checks found after the last round, or what they had found - already where there was no round to run. - """ - number = len(result.rounds) - limit = number + rounds - while (instruction is not None or not report.clean) and number < limit: - number += 1 - # The errors are what a round has to answer, and so what counts as - # having answered nothing. The prompt still carries every finding: a - # warning about a part nothing answers is worth a round knowing about, - # in case the sheet does hold the solution somewhere. - given = { - (one.check, one.field) - for one in report.findings - if one.level == package.ERROR - } - reply = fix_round( - draft, package.source_show(draft), report, backend, instruction - ) - instruction = None - result.usage.input_tokens += reply.usage.input_tokens - result.usage.output_tokens += reply.usage.output_tokens - result.usage.seconds += reply.usage.seconds - tokens = reply.usage.input_tokens + reply.usage.output_tokens - result.add_stage( - "fix", - f"round {number}: {summary(reply.calls)}, {tokens} tokens, " - f"{reply.usage.seconds:.1f}s", - ) - - report = package.validate(draft) - result.rounds.append( - RoundResult(number, reply.calls, reply.usage, len(report.errors)) - ) - if report.clean: - result.add_stage("validate", package.said(report)) - else: - errors = "; ".join(report.errors) - # A finding the round answered by writing a field rather than by - # quoting one — an empty field, a field the source words nowhere. - # The rounds have no command for it, so another round would be the - # same refusal, and the run ends naming the field for a person or a - # later command to quote the right source range into. - stuck = unrepaired(reply.calls) - if stuck: - result.stages.append( - StageResult( - "validate", - "; ".join(_stuck(one, report) for one in stuck) - + " — cannot be repaired by the loop, quote the source " - f"range into it; left by round {number}, no zip", - ) - ) - break - # Nothing left that the round was not already given: it answered what - # it could and left the rest, which is what it is told to do with a - # finding no range of the source answers. Another round would be the - # same prompt and the same report, so the run ends with them in it. - if all( - (one.check, one.field) in given - for one in report.findings - if one.level == package.ERROR - ): - result.add_stage( - "validate", f"{errors} — left by round {number}, no zip" - ) - break - if number == limit: - errors += f" — round limit {rounds} reached, no zip" - result.add_stage("validate", errors) - return report - - -def _build(draft: Path, out_dir: Path, result: RunResult, said: list[str]) -> None: - """Writes the set out, or says as a stage line why in2lambda would not. - - The checks passed and in2lambda still would not write the set out — an - image beside the draft that is not there, say. That is part of the run's - story rather than a fault in it, so it is a stage line like a validate - one, and the run ends without a zip. - - in2lambda warns about each warning-level finding as it builds. The validate - stage line lists the same findings, so this function repeats none of them. - A warning the validate line does not list gets a stage line of its own. - - Args: - draft: The draft file. - out_dir: Where the zip goes. - result: The run so far, which gets the build's stage line and, where - one was written, the zip — and where one was not, the refusal as - the reason, since the stage line is printed and gone. - said: The warnings the validate stage line lists, from the report that - let the build run. - """ - try: - built = package.build(draft, out_dir) - except package.BuildRefused as error: - result.add_stage("build", f"refused: {error}") - result.reason = str(error) - return - for message in built.warnings: - if message not in said: - result.add_stage("build", f"warning: {message}") - result.zip_path = built.zip_path - result.add_stage("build", str(result.zip_path)) - - -def _second( - source: Path, cache_dir: Path, solutions: Optional[Path] = None -) -> Optional[Second]: - """Another document of the set, which each candidate spec is also run over. - - The spec is saved for the whole folder, so one that covers the sheet in hand - and covers no other sheet of the set is not the spec to save. - - Each spec is run over a copy under `cache_dir`, never over the document in - the folder: running a spec over a sheet freezes it, and freezing writes that - sheet's draft again from the source, which deletes the fields a fixing round - or a reviewer wrote there and the log of the commands that wrote them. A tex - file that inputs files beside it does not find them beside the copy; where - in2lambda refuses it for that, the spec loop reports it as a document - in2lambda cannot read and judges the specs on this source. - - A PDF sibling is passed over rather than copied: converting it takes an OCR - call, and the spec loop makes no call but the model's. - - Args: - source: The file the user asked to convert, whose folder is the set. - cache_dir: Where the copy each spec is run over is written. - solutions: The solutions document of this sheet, which is in this run's - own draft and so is no other document of the set. - - Returns: - The first other document of the folder, by name, as the copy to run the - specs over or as the document the run passed over, and None where the - folder holds no other document. - """ - source = Path(source).resolve() - solutions = None if solutions is None else Path(solutions).resolve() - for path in sorted(source.parent.glob(f"*{source.suffix}")): - if path in (source, solutions) or not path.is_file(): - continue - if not package.is_document(path): - continue - if source.suffix.lower() == ".pdf": - return Second( - path.name, - passed_over="converting it takes an OCR call, and the spec " - "loop makes no call but the model's", - ) - copy = Path(cache_dir) / SECOND_NAME / path.name - try: - copy.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, copy) - except OSError as error: - # A sheet the OS will not let the run read, or a cache directory it - # will not let the run write. The other document is evidence about a - # spec and not the source the run converts, so the run goes on and - # says what it passed over, as it does for one in2lambda cannot read. - return Second(path.name, passed_over=f"copying it failed: {error}") - return Second(path.name, path=copy) - return None - - -def _stuck(field: str, report: package.Report) -> str: - """One field the rounds cannot repair, with what the report says about it.""" - said = [one.message for one in report.findings if one.field == field] - if not said: - return f"{field}: field replace was refused as writing the field" - return f"{field}: {'; '.join(said)}" - - -def _render(draft: Path, out_dir: Path) -> tuple[dict[str, Path], str]: - """Renders the draft's questions, or says why there are no pages to show.""" - try: - rendered = package.render(draft, out_dir / "render") - except package.CommandRefused as refused: - return {}, str(refused) - return rendered, ( - f"{len(rendered)} question{'' if len(rendered) == 1 else 's'} " - f"to {out_dir / 'render'}" - ) - - -def _relist(waiting: Review, result: RunResult, keys: list[str]) -> None: - """Renders again and puts the named questions back to the reviewer.""" - draft = Path(waiting.draft) - rendered, message = _render(draft, Path(waiting.out_dir)) - result.add_stage("render", message) - infos = package.questions(draft) - for key in keys: - question = waiting.question(key) - question.status = "pending" - question.pdf = str(rendered[key]) if key in rendered else None - question.lines = infos[key].ranges if key in infos else [] - - -def _asked(waiting: Review, cache_dir: Path) -> str: - """The questions still to answer, and the commands that answer them.""" - left = [one for one in waiting.questions if one.status != "approved"] - faulted = ( - "\n the checks fault the draft, so it cannot be built yet: " - f"{'; '.join(waiting.errors)}" - if waiting.errors - else "" - ) - return ( - f"mode {waiting.mode}, {len(left)} of {len(waiting.questions)} questions " - f"waiting:\n{waiting.listing()}{faulted}\n" - f" answer with `in2lambda-agent review approve Q --cache {cache_dir}`, " - '`review reject Q --note "..."` or `review edit FIELD OLD NEW`' - ) diff --git a/in2lambda_agent/review.py b/in2lambda_agent/review.py deleted file mode 100644 index 989aff8..0000000 --- a/in2lambda_agent/review.py +++ /dev/null @@ -1,306 +0,0 @@ -"""The review the design spec puts between a clean validate and the build. - -A run in `sample` or `per-question` mode stops once the checks are quiet: it -renders the questions a reviewer is to see, writes this record beside the OCR -cache, and prints them. The reviewer answers from the command line — approve, -reject with a note, or edit a field — and each of those commands reads the -record back, so the two halves of the run are one run with a file between them. - -The record holds everything the second half needs and the first half already -knew: where the draft is, what the run had cost so far, and what `record_run` -is to be told when the last question is approved. There is no reviewer state -anywhere else, and a run that is not in review builds without writing one. -""" - -import json -import random -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Any, Optional, Sequence - -from in2lambda_agent.fix import RoundResult -from in2lambda_agent.model import ToolCall, Usage -from in2lambda_agent.package import Coverage, QuestionInfo -from in2lambda_agent.spec import Second, SpecTry - -RECORD = "review.json" -"""What the pending review is called, in the run's cache directory.""" - - -class ReviewError(ValueError): - """A review command names something that is not waiting to be reviewed.""" - - -@dataclass -class Question: - """One question put to the reviewer, and what they said about it. - - Attributes: - key: The question, as the draft's fields key it: `q2`. - pdf: The PDF it was rendered to, or None where nothing rendered it. - lines: The ranges of the frozen source its fields were copied from. - status: `pending`, `approved` or `rejected`. - note: What a rejection said, which is what the fixing round is asked. - """ - - key: str - pdf: Optional[str] = None - lines: list[list[int]] = field(default_factory=list) - status: str = "pending" - note: Optional[str] = None - - -@dataclass -class Review: - """A review waiting to be answered, and the run it stopped in the middle of. - - Attributes: - mode: `sample` or `per-question`. - count: How many questions sample mode was to show. - source: The file the run converted, as the record names it. - spec: The set's spec file, whose folder the run record is in. - out_dir: Where the zip goes once every question is approved. - limit: The run's round limit, which a rejection's fixing gets again. - draft: The draft file the run left. - frozen: The source the draft was frozen from, for reading against. - reused: Whether the spec was the saved one, for the run record. - coverage: What the spec run made of the source, for the run record. - questions: Each question put to the reviewer, in order. - errors: What the checks last found, empty where they found nothing. - A reviewer's rounds and edits reach the draft between one command - and the next, so this is how the listing says that the draft as it - stands cannot be built. - rejections: Every rejection, in the order they were made. - edits: Every field the reviewer changed by hand, and who they were. - usage: What the run's model calls have cost so far. - tries: What each spec the run wrote covered and cost, for the run record. - second: The other document of the set the specs were run over, or the - one the run passed over, for the run record. - rounds: What each fixing round has done so far, the reviewer's among them. - """ - - mode: str - count: int - source: str - spec: str - out_dir: str - limit: int - draft: str - frozen: str - reused: bool - coverage: Coverage - questions: list[Question] = field(default_factory=list) - errors: list[str] = field(default_factory=list) - rejections: list[dict[str, Any]] = field(default_factory=list) - edits: list[dict[str, Any]] = field(default_factory=list) - usage: Usage = field(default_factory=Usage) - tries: list[SpecTry] = field(default_factory=list) - second: Optional[Second] = None - rounds: list[RoundResult] = field(default_factory=list) - - @property - def done(self) -> bool: - """Whether every question put to the reviewer has been approved.""" - return all(question.status == "approved" for question in self.questions) - - def question(self, key: str) -> Question: - """The question a review command names. - - Args: - key: What the reviewer called it. - - Returns: - That question of this review. - - Raises: - ReviewError: no question of that name is under review, and the - message names the ones that are. - """ - for question in self.questions: - if question.key == key: - return question - listed = ", ".join(one.key for one in self.questions) - raise ReviewError(f"{key} is not under review. This review has {listed}.") - - def listing(self) -> str: - """The questions, one line each, as the reviewer reads them.""" - return "\n".join( - f" {question.key} {question.status}: " - f"{question.pdf or 'not rendered'}, " - f"{self.frozen} lines {_lines(question.lines)}" - + (f" — {question.note}" if question.note else "") - for question in self.questions - ) - - def to_json(self) -> dict[str, Any]: - """What the run's record line says about the review. - - Returns: - The mode, every question with its verdict and note, and what the - reviewer's rejections and edits came to. - """ - return { - "mode": self.mode, - "questions": [ - {"key": one.key, "status": one.status, "note": one.note} - for one in self.questions - ], - "rejections": self.rejections, - "edits": self.edits, - } - - def save(self, path: Path) -> None: - """Writes the record where a review command will read it. - - Args: - path: The file to write, whose directory is made if it is not there. - """ - Path(path).parent.mkdir(parents=True, exist_ok=True) - state = { - **asdict(self), - "tries": [_try_json(one) for one in self.tries], - # `asdict` writes the copy the specs were run over as a Path, which - # json refuses, and the copy is of no use to the command that - # answers the review. - "second": None if self.second is None else self.second.to_json(), - "rounds": [_round_json(one) for one in self.rounds], - } - Path(path).write_text(json.dumps(state, indent=2), encoding="utf-8") - - @classmethod - def load(cls, path: Path) -> "Review": - """Reads back a review a run left waiting. - - Args: - path: The file `save` wrote. - - Returns: - The review, with what the run had spent on it. - - Raises: - ReviewError: there is no review waiting there, or what is there is - not one: a record cut short by a run that died writing it, or - one an older version of the agent left behind. - """ - path = Path(path) - if not path.is_file(): - raise ReviewError( - f"No review is waiting in {path}. A run with --review sample " - "or per-question writes one there when the checks come clean." - ) - try: - state = json.loads(path.read_text(encoding="utf-8")) - coverage = state["coverage"] - return cls( - **{ - **state, - "coverage": Coverage( - layout=coverage["layout"], - blocks=coverage["blocks"], - # JSON has no integer keys, and the layers are numbers - # everywhere else they are read. - fields={int(k): v for k, v in coverage["fields"].items()}, - ignored=coverage["ignored"], - unassigned=coverage["unassigned"], - ), - "questions": [Question(**one) for one in state["questions"]], - "usage": Usage(**state["usage"]), - "tries": [_try_from(one) for one in state["tries"]], - "second": ( - None - if state["second"] is None - else Second.from_json(state["second"]) - ), - "rounds": [_round_from(one) for one in state["rounds"]], - } - ) - except (ValueError, TypeError, KeyError) as error: - raise ReviewError( - f"{path} is not a review this run can read ({error}). Delete it " - "and run again with --review sample or per-question." - ) from None - - -def choose( - infos: dict[str, QuestionInfo], - mode: str, - count: int, - rng: Optional[random.Random] = None, -) -> list[str]: - """Which questions the reviewer is shown. - - Args: - infos: Every question of the draft, as `package.questions` reads them. - mode: `sample` or `per-question`. - count: How many a sample is at most. - rng: What picks the rest of a sample, so that a test can fix it. - - Returns: - The keys to review, in the order they are put to the reviewer: in a - sample the questions something past the spec wrote come first, since - those are the ones a reviewer is there for. - """ - if mode == "per-question": - return list(infos) - risky = [key for key, info in infos.items() if info.layer >= 3] - rest = [key for key, info in infos.items() if info.layer < 3] - chosen = risky[:count] - if len(chosen) < count: - picked = (rng or random.Random()).sample( - rest, min(count - len(chosen), len(rest)) - ) - chosen += [key for key in rest if key in set(picked)] - return chosen - - -def _lines(ranges: Sequence[Sequence[int]]) -> str: - """The lines of a question, as the listing names them.""" - if not ranges: - return "none" - return ", ".join(f"{start}-{end}" for start, end in ranges) - - -def _try_json(one: SpecTry) -> dict[str, Any]: - """One spec the run wrote, as the record keeps it between the two commands.""" - return { - "number": one.number, - "usage": asdict(one.usage), - "unassigned": one.unassigned, - "errors": one.errors, - "dropped": one.dropped, - "second": one.second, - "chosen": one.chosen, - } - - -def _try_from(saved: dict[str, Any]) -> SpecTry: - """One spec the run wrote, back out of the record for the run's own record.""" - return SpecTry( - number=saved["number"], - usage=Usage(**saved["usage"]), - unassigned=saved["unassigned"], - errors=saved["errors"], - dropped=saved["dropped"], - second=saved["second"], - chosen=saved["chosen"], - ) - - -def _round_json(one: RoundResult) -> dict[str, Any]: - """One fixing round, as the record keeps it between the two commands.""" - return { - "number": one.number, - "commands": [asdict(call) for call in one.commands], - "usage": asdict(one.usage), - "left": one.left, - } - - -def _round_from(saved: dict[str, Any]) -> RoundResult: - """One fixing round, back out of the record for the run's own record.""" - return RoundResult( - number=saved["number"], - commands=[ToolCall(**call) for call in saved["commands"]], - usage=Usage(**saved["usage"]), - left=saved["left"], - ) diff --git a/in2lambda_agent/spec.py b/in2lambda_agent/spec.py deleted file mode 100644 index e0045c8..0000000 --- a/in2lambda_agent/spec.py +++ /dev/null @@ -1,666 +0,0 @@ -"""Layer 1: the YAML spec, written once per document set and iterated into shape. - -A spec is selectors over the frozen source saying which blocks are questions, -which are parts and which are solutions, what to strip off the front of a field -and what to ignore, and which layout pairs the solutions up. `in2lambda spec -run` reads it; nothing here decides what a question is, and nothing here writes -a field. - -The spec is saved beside the source, under the name every sheet in that folder -shares, because a document set is a folder of sheets written the same way: the -next one runs the saved spec with no model call. `--spec` names another file, -which is read if it is there and written if it is not. - -A spec is the one piece of model output every sheet of a set reuses, so it is -written against what running it covers rather than blind. `iterate_spec` writes -one, runs it over this source and over another document of the set, reads the -coverage and the validation report back to the next call, and saves the spec -that left the fewest blocks unassigned, the fewest images ignored and the -fewest errors behind. -""" - -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Callable, Optional, Sequence - -import yaml - -from in2lambda_agent import package -from in2lambda_agent.fix import RoundResult -from in2lambda_agent.model import Backend, Reply, Usage -from in2lambda_agent.package import Coverage, Report - -SPEC_NAME = "in2lambda-spec.yaml" -"""What the set's spec is called, beside the sources it is for.""" - -RECORD_NAME = "in2lambda-agent-runs.jsonl" -"""One line per run, beside the spec: what it covered and what it cost.""" - -LAYOUTS = ("PartsSepSol", "PartsOneSol", "PartSolPartSol", "PartPartSolSol") -"""The four ways a document writes its solutions, and all a spec may name.""" - -SYSTEM = """\ -You write a spec for in2lambda: a small YAML file of selectors over a document \ -already parsed into blocks. It is read by `in2lambda spec run`, which fills a \ -draft's fields from it. Answer with the YAML and nothing else — no prose, no \ -code fence. - -A spec has these keys, and no others: - - question: which blocks hold a question's own text. Required. - part: which blocks hold one lettered part of a question. - solution: which blocks hold a worked solution. - ignore: which blocks are none of those and are wanted in no field. - strip: a list of regexes taken off the front of every field's text. - layout: one of PartsSepSol, PartsOneSol, PartSolPartSol, PartPartSolSol. - -A selector is an optional block type, then any number of constraints: - - Header level=2 text~'^Question' type, then an exact and a regex constraint - Para label~'^[0-9]' label is the block's first word - after Header text=Solutions, Para only blocks somewhere after such a header - -The type is the pandoc element's own name: Header, Para, ListItem, Table, -BlockQuote, CodeBlock. Leave it out to match any block. `level` is a heading's -level, `text` is everything the block says, `label` its first word. Put single -quotes round any regex with a backslash in it. - -Every block is tried against ignore, then question, then part, then solution, -whatever order the keys are written in, and is whatever the first of them says -it is. So the selectors must not overlap: if the solutions are paragraphs and -the question stems are paragraphs too, the question selector needs a constraint -that the solutions fail. - -Three things about blocks to write selectors against: - - * One block fills one field. A question's text is the block holding its stem, - not the heading above it — headings usually belong in `ignore`. - * A lettered or numbered item — `(a) ...`, `a. ...`, `1. ...` — is a - ListItem, and its marker is not part of the text a constraint matches. The - marker is still in the field's value, so `strip` is what takes it off. - * A block indented under a list item is inside it, not beside it: such a - question and its parts are one block, and there is nothing to select. - * A paragraph holding an image — `![...](...)` — is content, never `ignore`. - The figure belongs to the question or the solution it stands with, and a - set built from an ignored figure has lost it. - -A draft holds two documents where the solutions are written as a file of their -own. The questions file is the first source, with block ids `b1` onwards, and -the solutions file is the second, with ids `2/b1` onwards. The same selectors -run over both. In the solutions file, every block `part` or `solution` matches is -a solution, and the solutions answer the questions of the first source in order: -each question's parts, or the question itself where it has none. A block -`question` matches there is a marker — the `Q2.` or `## Question 2` written above -the solutions to the second question. A marker is no question of its own, and its -text reaches no field. The first marker in the file assigns the solutions after -it to the first question, the second marker to the second question, so a -`question` selector that matches the marker above one question's solutions and -not the marker above another's assigns every solution after it to the wrong -question. `layout` describes the questions file alone. - -The layout says which question or part a solution answers: - - PartsSepSol every solution together at the end, in part order: each - question with parts is answered part by part, each question - without parts is answered once. - PartsOneSol one solution to the whole question. - PartSolPartSol each part answered where it stands. - PartPartSolSol a question's parts, then their solutions in the same order. - -For a sheet whose questions are under `## Question n` headings, whose parts are -`(a)`/`(b)` items, and whose solutions are labelled `1(a)` under a `## \ -Solutions` heading: - -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\\([a-z]\\) ', '^\\d+\\([a-z]\\) '] -layout: PartsSepSol - -Every block of the source must end up in a field or be ignored: a block left -over is reported, and the run stops. Write selectors that account for all of -them. - -You may be shown the spec you last wrote, what running it covered and what the -checks found in the draft it filled, and asked for a better one. The spec is -saved for the whole set, so a document of the set other than this one is shown -as well where the folder holds one. Change the selectors that left blocks over -and keep the ones that did not.\ -""" - - -class BadSpec(ValueError): - """What the model answered with is not a spec.""" - - -@dataclass -class SpecTry: - """One spec the agent wrote, and what running it made of the set. - - Attributes: - number: 0 for the saved spec a rewrite starts from, then 1 for the first - call, 2 for the second. - usage: What the call cost, all zeroes for try 0. - unassigned: How many blocks the spec left in no field and not ignored. - errors: How many errors the checks then found in the draft it filled. - dropped: How many blocks the spec marked ignore that hold an image. - second: How many blocks the spec left in no field in another document - of the set, or None where the run ran no spec over another document - — the record's `second` says why. A spec in2lambda refuses over - that document wrote no field there, so it left every block of it in - no field. - chosen: Whether this is the spec the run saved and went on with. - """ - - number: int - usage: Usage = field(default_factory=Usage) - unassigned: int = 0 - errors: int = 0 - dropped: int = 0 - second: Optional[int] = None - chosen: bool = False - - @property - def score(self) -> int: - """What the tries are ranked by, the lowest winning. - - A block in no field is an error of the report as well as a line of the - coverage, so it counts twice. That is the same double for every try and - does not change the order they come in. - - The third term is the images the spec dropped. in2lambda's checks say - nothing about an ignored block, so a spec that ignores a figure is - scored like one that leaves a block unassigned and is written again. - """ - return self.unassigned + self.errors + self.dropped + (self.second or 0) - - -@dataclass -class Second: - """Another document of the set, which each candidate spec is also run over. - - Attributes: - name: The file name of that document, which the `set` stage line and the - run record name. - path: The copy of that document each spec is run over, or None where - in2lambda can run no spec over the document. - passed_over: Why in2lambda ran no spec over the document: the document - is a PDF, the run cannot copy it, in2lambda cannot read the copy, - or in2lambda refused the spec the run kept. None where in2lambda - ran that spec over the document. A spec in2lambda refused there and - the run then threw away is that try's alone and is not here. - """ - - name: str - path: Optional[Path] = None - passed_over: Optional[str] = None - - def to_json(self) -> dict[str, Optional[str]]: - """The document as the run record and a waiting review keep it.""" - return {"name": self.name, "passed_over": self.passed_over} - - @staticmethod - def from_json(saved: dict) -> "Second": - """The document back out of a waiting review, for the run record. - - The copy the specs were run over is not kept: the review is answered - after the spec loop has ended, and no spec is run again. - """ - return Second(name=saved["name"], passed_over=saved["passed_over"]) - - -@dataclass -class Previous: - """A spec that has been run, as the call revising it is shown it. - - Attributes: - text: The spec itself. - coverage: What running it made of this source. - report: What the checks found in the draft it filled. - second: What running it made of another document of the set, or None - where the folder holds no other document. - second_name: That document's file name. - """ - - text: str - coverage: Optional[Coverage] = None - report: Optional[Report] = None - second: Optional[Coverage] = None - second_name: str = "" - - -def spec_path(source: Path, spec: Optional[Path] = None) -> Path: - """Where this source's set keeps its spec. - - Args: - source: The file the user asked to convert, before any OCR: the set is - the folder that file is in, not the folder its markdown ended up in. - spec: A spec named on the command line, which overrides the set's own. - - Returns: - The file to read the spec from, and to write it to if it is not there. - """ - if spec is not None: - return Path(spec).resolve() - return Path(source).resolve().parent / SPEC_NAME - - -def write_spec( - shown: str, - backend: Backend, - previous: Optional[Previous] = None, - sources: int = 1, - solutions_only: bool = False, -) -> tuple[str, Reply]: - """Writes a spec for a source, in one model call with no tools. - - Args: - shown: The numbered source with block ids, as `source show` prints it. - backend: The backend to call, already known to be available. - previous: The spec run before this call and what running it covered, - where this call is a revision of that spec. - sources: How many documents the draft holds: 2 where the solutions are - a file of their own, which the prompt then says before the source. - solutions_only: Whether the one document is a file of solutions with no - questions file beside it, which the prompt then says before the - source, because the questions of such a document are the markers - written above its solutions. - - Returns: - The spec, and the reply it came in. - - Raises: - BadSpec: the reply is not YAML, is not a mapping, or names no layout - or one that is not a layout. - """ - prompt = "" - if sources > 1: - prompt = ( - "The draft holds two documents: the questions file, whose blocks " - "are `b1` onwards, and its solutions file, whose blocks are `2/b1` " - "onwards. Every solution is in the second.\n\n" - ) - elif solutions_only: - prompt = ( - "This document holds solutions and no questions, and there is no " - "second source. The marker written above each group of solutions — " - "the `Q2.` or the `## Question 2` — is the question here, and its " - "text is that question's text. Write `question` to match every " - "marker, and `solution` to match the worked solutions under it. " - "`layout` describes this file.\n\n" - ) - prompt += f"Here is the source, one line each with its block id:\n\n{shown}\n" - if previous is not None: - prompt += _revision(previous) - reply = backend.call(SYSTEM, prompt) - text = _unfenced(reply.text) - _check(text) - return text, reply - - -def _revision(previous: Previous) -> str: - """The last spec and what running it covered, as the next call is shown them.""" - said = [f"\nYour last spec for this set was:\n\n{previous.text}"] - if previous.coverage is not None: - said.append(f"\nRunning it over this source covered:\n\n{previous.coverage}\n") - # The images the spec dropped go in beside the report's errors, under the - # one heading: an ignored figure is a fault of the selectors like a block - # left in no field, and the next call answers both the same way. - found = list(previous.report.errors) if previous.report is not None else [] - if previous.coverage is not None: - found += [one.message for one in previous.coverage.dropped] - if found: - said.append("\nThe checks then found:\n\n" + "\n".join(found) + "\n") - if previous.second is not None: - left = ", ".join(previous.second.unassigned) or "no blocks" - said.append( - f"\nRunning it over {previous.second_name}, another document of this " - f"set, left {left} in no field.\n" - ) - said.append( - "\nWrite a spec that leaves fewer blocks unassigned, fewer images " - "ignored and fewer errors behind, over this source and over the rest " - "of the set.\n" - ) - return "".join(said) - - -def iterate_spec( - frozen: Path, - saved: Path, - backend: Backend, - *, - tries: int, - on_stage: Callable[[str, str], None], - second: Optional[Second] = None, - previous: Optional[Previous] = None, - solutions: Optional[Path] = None, - solutions_name: str = "", - solutions_only: bool = False, -) -> tuple[Path, Coverage, Report, list[SpecTry]]: - """Writes the set's spec up to `tries` times and saves the best of them. - - Each call after the first is shown the spec before it, the coverage line, - the errors the checks found, the images the spec marked ignore and the - blocks the spec left over in another document of the set. The loop stops at - a spec that scores zero, since a further call has nothing to improve. - - Args: - frozen: The markdown, tex or docx file each spec is run over. - saved: The set's spec file, which every try writes and the chosen spec - is left in. - backend: The backend to call, already known to be available. - tries: How many specs may be written. - on_stage: Called with the `(stage, message)` of each line as the loop - makes it, so that the caller prints a line while the loop is still - running. It is a function of two strings rather than - `RunResult.add_stage` itself, because `pipeline` imports `spec`. - second: Another document of the set, run to say whether a spec covers - the set rather than this one sheet of it, or one the run passed - over, which the `set` line and the record name. A document - in2lambda cannot read becomes a document passed over. A document - in2lambda reads and refuses the spec over stays, and the next try - is run over that document as well. - previous: The saved spec and what running it covered, where this loop - is the rewrite of a spec the checks faulted. The saved spec is run - over the other document first, so that it is recorded as try 0 and - the first call reads what it left there; the spec it names is being - replaced, so it is not one of the tries chosen from. - solutions: The markdown of the solutions document, frozen into the same - draft as the second source, where the sheet has one. - solutions_name: That document's file name, which the freeze line names. - solutions_only: Whether `frozen` is a file of solutions with no - questions file beside it, which every call is told. - - Returns: - The draft the chosen spec filled, what that spec covered, what the - checks found in the draft, and what each try did. Each line the loop - prints went to `on_stage` as the loop made it. - - Raises: - ModelError: a call did not finish. - BadSpec: what the model answered with is not a spec. - SpecRejected: in2lambda will not run a spec this loop wrote. - SourceError: in2lambda cannot freeze or check this source. - - Every error leaving this function puts the spec the set had before the loop - back, since the spec of a try the loop never chose is not one to save. - """ - made: list[SpecTry] = [] - if second is not None and second.passed_over is not None: - on_stage("set", f"{second.name} passed over: {second.passed_over}") - if previous is not None: - # The saved spec is still the file on disk, so running it over the other - # document says what it left there. Try 0 records that, and the first - # call is asked to improve on the set rather than on this sheet alone. - over_second, left_over = _over_second(second, saved, on_stage) - previous.second = over_second - previous.second_name = second.name if over_second is not None else "" - made.append( - SpecTry( - number=0, - unassigned=len(previous.coverage.unassigned), - errors=len(previous.report.errors), - dropped=len(previous.coverage.dropped), - second=left_over, - ) - ) - # What the set's spec said before this loop wrote over it. Every try writes - # the file, and no try has been chosen until the loop ends, so a loop that - # raises puts the old spec back: a try's spec left beside the sources is - # read by every later run over the set, which then makes no call, until - # someone deletes the file by hand. - replaced = saved.read_text(encoding="utf-8") if saved.is_file() else None - - # Each try's spec, and what the other document was left as after that try - # ran over it: the record is to say what became of the other document under - # the spec the loop kept, not under a later try it threw away. - best: Optional[tuple[SpecTry, str, Optional[str]]] = None - more = [solutions] if solutions is not None else [] - try: - for number in range(1, tries + 1): - draft = package.source_add(frozen, *more) - on_stage("freeze", package.froze(draft, solutions_name)) - text, reply = write_spec( - package.source_show(draft), - backend, - previous, - sources=2 if solutions is not None else 1, - solutions_only=solutions_only, - ) - saved.write_text(text, encoding="utf-8") - tokens = reply.usage.input_tokens + reply.usage.output_tokens - on_stage( - "spec", - f"wrote {saved} via {reply.backend}, {tokens} tokens, " - f"{reply.usage.seconds:.1f}s (try {number} of {tries})", - ) - coverage, report = _run(draft, saved, on_stage) - over_second, left_over = _over_second(second, saved, on_stage) - one = SpecTry( - number=number, - usage=reply.usage, - unassigned=len(coverage.unassigned), - errors=len(report.errors), - dropped=len(coverage.dropped), - second=left_over, - ) - made.append(one) - if best is None or one.score < best[0].score: - best = (one, text, second.passed_over if second is not None else None) - if one.score == 0: - break - previous = Previous( - text=text, - coverage=coverage, - report=report, - second=over_second, - second_name=second.name if over_second is not None else "", - ) - except Exception: - if replaced is None: - saved.unlink(missing_ok=True) - else: - saved.write_text(replaced, encoding="utf-8") - raise - - chosen, text, passed_over = best - chosen.chosen = True - if second is not None and second.path is not None: - # A document in2lambda could not read is passed over for good, whatever - # try found that out. A spec it refused there is that try's alone: the - # record is to say what became of the document under the spec the loop - # kept. - second.passed_over = passed_over - if len([one for one in made if one.number]) > 1: - on_stage("spec", f"kept try {chosen.number} of {tries}") - if chosen.number != made[-1].number: - # A later try covered the set less well, so the chosen spec is written - # and run again: the draft the run goes on with is the one that spec - # filled, not the one the last try left. - saved.write_text(text, encoding="utf-8") - draft = package.source_add(frozen, *more) - on_stage("freeze", package.froze(draft, solutions_name)) - coverage, report = _run(draft, saved, on_stage) - return draft, coverage, report, made - - -def _over_second( - second: Optional[Second], saved: Path, on_stage: Callable[[str, str], None] -) -> tuple[Optional[Coverage], Optional[int]]: - """Runs the spec now in `saved` over the set's other document. - - Returns: - The coverage the spec made of the other document, and how many blocks - it left in no field there. Both None where the run has no other - document to run a spec over. Where in2lambda refuses the spec over the - other document, there is no coverage to show the next call, and the - count is every block of that document. - """ - if second is None or second.path is None: - return None, None - try: - draft = package.source_add(second.path) - except package.SourceError as error: - # A folder holds files that are not documents — a Word lock file beside - # a docx — and in2lambda refuses them. in2lambda also refuses a tex - # file that inputs the figures beside it, since the copy in the cache - # has no figures beside it. The other document is evidence about a - # spec and not the source the run converts, so the run continues and - # judges the tries on this source. The next try runs over no other - # document, and the record names the document the run passed over. - second.passed_over = str(error) - second.path = None - on_stage("set", f"{second.name} cannot be read: {error}") - return None, None - try: - coverage = package.spec_run(draft, saved) - except package.SpecRejected as error: - # in2lambda runs this spec over this source and refuses it over the - # other document. The next spec the run writes may run over both - # documents, so the copy stays and the next try is run over the other - # document as well. - second.passed_over = f"in2lambda refused the spec: {error}" - on_stage("set", f"{second.name}: {second.passed_over}") - return None, package.blocks(draft) - second.passed_over = None - on_stage("set", f"{second.name}: {coverage}") - return coverage, len(coverage.unassigned) - - -def _run( - draft: Path, saved: Path, on_stage: Callable[[str, str], None] -) -> tuple[Coverage, Report]: - """Runs one spec over one draft and reports the two lines it prints.""" - coverage = package.spec_run(draft, saved) - on_stage("coverage", str(coverage)) - report = package.validate(draft) - on_stage( - "validate", package.said(report) if report.clean else "; ".join(report.errors) - ) - return coverage, report - - -def record_run( - path: Path, - source: Path, - *, - reused: bool, - coverage: Coverage, - usage: Usage, - tries: Sequence[SpecTry] = (), - second: Optional[Second] = None, - rounds: Sequence[RoundResult] = (), - review: Optional[dict] = None, -) -> None: - """Appends one line about a run to the set's record. - - The design spec's test plan is run over the corpus and reads these: what - share of a document a spec covers, whether the set's spec was reused, what - the model calls cost, and how many rounds of fixing the checks took — or how - many they took without ever coming clean. - - Args: - path: The record file, beside the spec. - source: The file the run converted. - reused: Whether the spec was the saved one rather than a new call. - coverage: What the spec run made of the source. - usage: What the run's model calls cost, all zeroes where there were none. - tries: What each spec the run wrote covered and cost, in order, and - empty where the run reused the set's saved spec. - second: The other document of the set each spec was run over, and why - in2lambda ran no spec over the document where it ran none. None - where the folder holds no other document, and where the run reused - the saved spec and ran no loop, which `reused` on the same line - says. - rounds: What each round of fixing did, in order, and empty where the - draft came clean out of the spec alone. - review: What a reviewer made of the set, as `Review.to_json` says it, - and absent where the run was not reviewed. The design spec's test - plan counts the rejections a mode drew, which is how a document set - earns its way to mode none. - """ - line = { - "source": str(source), - "reused": reused, - "layout": coverage.layout, - "blocks": coverage.blocks, - "fields": coverage.fields, - "ignored": coverage.ignored, - "unassigned": coverage.unassigned, - "input_tokens": usage.input_tokens, - "output_tokens": usage.output_tokens, - "seconds": round(usage.seconds, 3), - "iterations": [ - { - "try": one.number, - "input_tokens": one.usage.input_tokens, - "output_tokens": one.usage.output_tokens, - "seconds": round(one.usage.seconds, 3), - "unassigned": one.unassigned, - "errors": one.errors, - "dropped": one.dropped, - "second": one.second, - "chosen": one.chosen, - } - for one in tries - ], - "second": None if second is None else second.to_json(), - "rounds": [ - { - "round": one.number, - "input_tokens": one.usage.input_tokens, - "output_tokens": one.usage.output_tokens, - "seconds": round(one.usage.seconds, 3), - "commands": [call.name for call in one.commands], - "left": one.left, - } - for one in rounds - ], - } - if review is not None: - line["review"] = review - with Path(path).open("a", encoding="utf-8") as record: - record.write(json.dumps(line) + "\n") - - -def _unfenced(text: str) -> str: - """The spec out of a reply, past a code fence the model wrapped it in. - - Asking for the YAML alone is not the same as getting it, and a fence is the - one thing a model adds often enough to be worth taking off rather than - refusing over. - """ - lines = text.strip().splitlines() - if lines and lines[0].startswith("```"): - lines = lines[1:] - if lines and lines[-1].strip() == "```": - lines = lines[:-1] - return "\n".join(lines).strip() + "\n" - - -def _check(text: str) -> None: - """Refuses a spec that in2lambda would refuse, saying which part of it. - - Only what is cheap to say better here: the layout is the field the agent - chooses and in2lambda cannot suggest an alternative for, and a reply that - is not a mapping at all is a reply that is not a spec. Everything else a - spec can get wrong is `spec run`'s to report, against the line it is on. - """ - try: - loaded = yaml.safe_load(text) - except yaml.YAMLError as error: - raise BadSpec(f"The spec is not YAML: {error}") from None - if not isinstance(loaded, dict): - raise BadSpec(f"A spec is a mapping of selectors, which {text!r} is not.") - if "layout" not in loaded: - raise BadSpec( - "The spec names no layout, and a layout is what pairs a solution " - f"to the part it answers. One of {', '.join(LAYOUTS)}." - ) - if loaded["layout"] not in LAYOUTS: - raise BadSpec( - f"{loaded['layout']!r} is not a layout. One of {', '.join(LAYOUTS)}." - ) diff --git a/in2lambda_agent/sweep.py b/in2lambda_agent/sweep.py index cca24b6..7870b69 100644 --- a/in2lambda_agent/sweep.py +++ b/in2lambda_agent/sweep.py @@ -24,7 +24,7 @@ from pathlib import Path from typing import Optional, Sequence -from in2lambda_agent import package, pair, pipeline, routes +from in2lambda_agent import ocr, pair, routes from in2lambda_agent.model import Backend, choose_backend from in2lambda_agent.settings import Settings @@ -113,14 +113,14 @@ def _sheets( of the PDFs alone therefore has no such sheet to run. A tex file with no document body is a drawing or a preamble rather than a - sheet, as `package.is_document` reads one, and is left out: converting one + sheet, as `pair.is_document` reads one, and is left out: converting one makes two model calls and returns a set of no questions. """ wanted = {"." + one.lower().lstrip(".") for one in suffixes} return [ (sheet, solutions) for sheet, solutions in pair.pairs_in(folder) - if sheet.suffix.lower() in wanted and package.is_document(sheet) + if sheet.suffix.lower() in wanted and pair.is_document(sheet) ] @@ -224,7 +224,7 @@ def sweep( suffixes: Sequence[str] = DEFAULT_SUFFIXES, results: Path = DEFAULT_RESULTS, work: Path = DEFAULT_WORK_DIR, - cache: Path = pipeline.DEFAULT_CACHE_DIR, + cache: Path = ocr.DEFAULT_CACHE_DIR, settings: Optional[Settings] = None, backend: Optional[Backend] = None, ) -> list[Row]: diff --git a/in2lambda_agent/targets.py b/in2lambda_agent/targets.py index c31520f..5297353 100644 --- a/in2lambda_agent/targets.py +++ b/in2lambda_agent/targets.py @@ -271,6 +271,7 @@ def run_one( settings: Optional[Settings] = None, backend: Optional[Backend] = None, fresh: bool = False, + replay: bool = False, ) -> Result: """Converts one target and compares what came out with its export. @@ -284,6 +285,9 @@ def run_one( if absent. fresh: Read the document again rather than converting the reply saved beside the filter, which is how a target is given a new reading. + replay: Refuse a target whose filter or reply is not saved rather than + paying for one, so that the run reads what is committed and makes + no call that reads the document. Returns: The target's result. Nothing a target raises leaves this function: what @@ -299,6 +303,15 @@ def run_one( # the document's structure, so route B cannot run over a scanned target: # it converts through route A alone, and no filter is written for it. lua = None if target.questions.suffix.lower() == ".pdf" else saved / FILTER_NAME + if replay: + absent = [one for one in (reply, lua) if one is not None and not one.is_file()] + if absent: + return Result( + name=target.name, + error=f"{absent[0]} is not saved, and a replay makes no call that " + f"reads a document. `in2lambda-agent targets ROOT --filters " + f"{filters}` writes it.", + ) route_a = None try: if reply.is_file() and not fresh: @@ -368,6 +381,7 @@ def run( settings: Optional[Settings] = None, backend: Optional[Backend] = None, fresh: bool = False, + replay: bool = False, ) -> list[Result]: """Runs every target under a root, printing each one's report as it finishes. @@ -381,6 +395,8 @@ def run( backend: The backend to write the filters with. fresh: Read every document again rather than converting the saved replies. + replay: Refuse a target whose filter or reply is not saved rather than + paying for one. Returns: One result per target, in the order they ran. @@ -396,6 +412,7 @@ def run( settings=settings, backend=backend, fresh=fresh, + replay=replay, ) for line in result.report(): print(line) diff --git a/in2lambda_agent/ui/server.py b/in2lambda_agent/ui/server.py index 0f1f4bf..e5d1b1e 100644 --- a/in2lambda_agent/ui/server.py +++ b/in2lambda_agent/ui/server.py @@ -35,7 +35,7 @@ from starlette.responses import FileResponse, JSONResponse, Response, StreamingResponse from starlette.routing import Route -from in2lambda_agent import corpus, pair, pipeline, routes +from in2lambda_agent import ocr, pair, routes from in2lambda_agent.mathpix import MathpixError from in2lambda_agent.model import ModelError, ModelUnavailable, choose_backend from in2lambda_agent.settings import Settings, load_settings @@ -271,7 +271,7 @@ def build_app( corpus_dir: Optional[Path] = None, *, settings: Optional[Settings] = None, - cache_dir: Path = pipeline.DEFAULT_CACHE_DIR, + cache_dir: Path = ocr.DEFAULT_CACHE_DIR, ) -> Starlette: """The page and its endpoints, over one runner. @@ -308,7 +308,7 @@ async def sources(request: Request) -> Response: for path in sorted(where.iterdir(), key=lambda one: one.name.lower()): if path.is_dir(): folders.append(path) - elif path.suffix.lower().lstrip(".") in SUFFIXES and corpus.is_document( + elif path.suffix.lower().lstrip(".") in SUFFIXES and pair.is_document( path ): documents.append(path) diff --git a/poetry.lock b/poetry.lock index bb7695f..512692e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1721,4 +1721,4 @@ ui = ["starlette", "uvicorn"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "ae14de7279e16e6cab61007ac0ae124713cb2a2207786f347cb9f74e2a5071ca" +content-hash = "dfe3be05f1088a4b0759641a3e5d20af580bbf817cd5ebd24a64476f01b3ab66" diff --git a/pyproject.toml b/pyproject.toml index 44c0d55..5d09618 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,6 @@ in2lambda = { git = "https://github.com/lambda-feedback/in2lambda.git", rev = "0 "convert", ] } python-dotenv = "^1.0" -# The agent reads the spec the model writes before handing it to in2lambda, so it -# parses YAML itself rather than through whatever in2lambda happens to use. -pyyaml = "^6.0" # One per model backend, all of them imported only by in2lambda_agent.model. claude-agent-sdk = "^0.2.157" anthropic = "^1.7.0" diff --git a/tests/fixtures/faulty-spec.yaml b/tests/fixtures/faulty-spec.yaml deleted file mode 100644 index b7d6947..0000000 --- a/tests/fixtures/faulty-spec.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# A spec `in2lambda spec run` accepts, over a sheet carrying the ticket's three -# faults — the kind OCR leaves behind. Two of them the checks report, and those -# are what a fixing round is given: -# -# * Question 2's stem and its part (a) ran together with no blank line, so -# pandoc made them one block; it starts with maths, so `question` misses it -# and nothing else claims it. One uncovered block, to split and then quote. -# * Question 2's solution is labelled `2` with no part letter, which `solution` -# misses: one more uncovered block, to quote. -# -# The third is the dropped brace in `\mathbf{B` on that same solution line. It -# is reported once the solution has been quoted into a field: the report now -# runs `in2lambda.validation` over the set the draft describes — the delimiters, -# KaTeX and the PDF compile the design spec lists — and what those find arrives -# as a finding under the check `problem`, named by the field it is in. So a -# round can repair it by reading the text it has just quoted, or by the finding -# that names it once it has; `field replace` is exercised either way. -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para label~'^\d+\([a-z]\)' -strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/tests/fixtures/faulty-unsolved.md b/tests/fixtures/faulty-unsolved.md deleted file mode 100644 index d940c14..0000000 --- a/tests/fixtures/faulty-unsolved.md +++ /dev/null @@ -1,24 +0,0 @@ -# Tutorial Sheet 3 - -## Question 1 - -A ball is thrown straight up at $20\,\mathrm{m/s}$. - -(a) Find the greatest height it reaches. - -(b) Find its time of flight. - -(c) Sketch the trajectory. - -## Question 2 - -$\nabla \cdot \mathbf{B} = 0$ for a dipole field. -(a) Show this by taking the divergence term by term. - -## Solutions - -1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ - -1(b) $t = 2v/g = 4.08\,\mathrm{s}$ - -2 Write $\mathbf{B$ in components and differentiate. diff --git a/tests/fixtures/faulty.md b/tests/fixtures/faulty.md deleted file mode 100644 index 8f1b6e7..0000000 --- a/tests/fixtures/faulty.md +++ /dev/null @@ -1,22 +0,0 @@ -# Tutorial Sheet 3 - -## Question 1 - -A ball is thrown straight up at $20\,\mathrm{m/s}$. - -(a) Find the greatest height it reaches. - -(b) Find its time of flight. - -## Question 2 - -$\nabla \cdot \mathbf{B} = 0$ for a dipole field. -(a) Show this by taking the divergence term by term. - -## Solutions - -1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ - -1(b) $t = 2v/g = 4.08\,\mathrm{s}$ - -2 Write $\mathbf{B$ in components and differentiate. diff --git a/tests/fixtures/figure-paragraph-spec.yaml b/tests/fixtures/figure-paragraph-spec.yaml deleted file mode 100644 index d2f48ee..0000000 --- a/tests/fixtures/figure-paragraph-spec.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# A spec for figure-paragraph.md that matches the figure's caption with its -# `ignore` selector, so the image is dropped and the set is built without it. -# Everything else the spec covers, and the checks find nothing: the dropped -# image is the one fault of it. -ignore: [Header, "Para text~'Figure 1'"] -question: Para text~'^A ball' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/tests/fixtures/figure-paragraph.md b/tests/fixtures/figure-paragraph.md deleted file mode 100644 index cbe2f1a..0000000 --- a/tests/fixtures/figure-paragraph.md +++ /dev/null @@ -1,18 +0,0 @@ -# Tutorial Sheet 1 - -## Question 1 - -A ball is thrown straight up at $20\,\mathrm{m/s}$. - -![the ball](figures/ball.png) -Figure 1: the ball leaving the hand. - -(a) Find the greatest height it reaches. - -(b) Find its time of flight. - -## Solutions - -1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ - -1(b) $t = 2v/g = 4.08\,\mathrm{s}$ diff --git a/tests/fixtures/figure.md b/tests/fixtures/figure.md deleted file mode 100644 index e7ede4f..0000000 --- a/tests/fixtures/figure.md +++ /dev/null @@ -1,27 +0,0 @@ -# Tutorial Sheet 1 - -## Question 1 - -A ball is thrown straight up at $20\,\mathrm{m/s}$. ![the ball](figures/ball.png) - -(a) Find the greatest height it reaches. - -(b) Find its time of flight. - -## Question 2 - -A block of mass $m$ rests on a slope of angle $\theta$. - -(a) Name the three forces acting on the block. - -(b) Find the least coefficient of friction that holds it still. - -## Solutions - -1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ - -1(b) $t = 2v/g = 4.08\,\mathrm{s}$ - -2(a) Weight, the normal reaction, and friction along the slope. - -2(b) $\mu = \tan\theta$ diff --git a/tests/fixtures/lone-spec.yaml b/tests/fixtures/lone-spec.yaml deleted file mode 100644 index ab95b21..0000000 --- a/tests/fixtures/lone-spec.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# The spec the fake backend answers with for a file of solutions that has no -# questions file beside it. The marker above each solution is the question, so -# the `question` selector matches the markers and the `solution` selector the -# paragraphs under them. -ignore: Header -question: Para text~'^Q\d' -solution: after Header text=Solutions, Para -layout: PartsOneSol diff --git a/tests/fixtures/lone_solutions.md b/tests/fixtures/lone_solutions.md deleted file mode 100644 index 00757cf..0000000 --- a/tests/fixtures/lone_solutions.md +++ /dev/null @@ -1,11 +0,0 @@ -# Tutorial Sheet 5: answers - -## Solutions - -Q1. - -$\omega = v / r$ - -Q2. - -$T = 2\pi\sqrt{m/k}$ diff --git a/tests/fixtures/marked_solutions.md b/tests/fixtures/marked_solutions.md deleted file mode 100644 index 4b9549e..0000000 --- a/tests/fixtures/marked_solutions.md +++ /dev/null @@ -1,13 +0,0 @@ -# Tutorial Sheet 3: answers - -## Solutions - -Q1. - -(a) $\omega = v / r$ - -Q2. - -(a) $T = 2\pi\sqrt{m/k}$ - -(b) $v = A\sqrt{k/m}$ diff --git a/tests/fixtures/paired-spec.yaml b/tests/fixtures/paired-spec.yaml deleted file mode 100644 index 0dae92a..0000000 --- a/tests/fixtures/paired-spec.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# The spec the fake backend answers with for a sheet whose solutions are a file -# of their own. The same selectors run over both sources: the questions file -# holds no `## Solutions` heading, so nothing in it is a solution, and every -# solution comes from the second source. -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/tests/fixtures/questions-only.md b/tests/fixtures/questions-only.md deleted file mode 100644 index e000e3f..0000000 --- a/tests/fixtures/questions-only.md +++ /dev/null @@ -1,17 +0,0 @@ -# Tutorial Sheet 1 - -## Question 1 - -A ball is thrown straight up at $20\,\mathrm{m/s}$. - -(a) Find the greatest height it reaches. - -(b) Find its time of flight. - -## Question 2 - -A block of mass $m$ rests on a slope of angle $\theta$. - -(a) Name the three forces acting on the block. - -(b) Find the least coefficient of friction that holds it still. diff --git a/tests/fixtures/sheet-spec.yaml b/tests/fixtures/sheet-spec.yaml deleted file mode 100644 index e2eb6c3..0000000 --- a/tests/fixtures/sheet-spec.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# The spec the fake backend answers with: one that `in2lambda spec run` accepts -# for either sheet in fixtures/, which is what a document set's spec is for. -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/tests/fixtures/tex-figure-spec.yaml b/tests/fixtures/tex-figure-spec.yaml deleted file mode 100644 index 3543598..0000000 --- a/tests/fixtures/tex-figure-spec.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# The spec for the sheet with a figure: the question is the paragraph that -# names the image, its parts are the items of the list under it, and its -# solutions are under the `Solutions` heading. A solution begins with its -# label, `1(a)`, so no solution matches `question`. -ignore: Header -question: Para text~'^[A-Z]' -part: ListItem -solution: after Header text=Solutions, Para -strip: ['^\d+\([a-z]\) '] -layout: PartsSepSol diff --git a/tests/fixtures/tex-figure.tex b/tests/fixtures/tex-figure.tex deleted file mode 100644 index f5fc264..0000000 --- a/tests/fixtures/tex-figure.tex +++ /dev/null @@ -1,26 +0,0 @@ -% Synthetic, in the shape of the corpus's sheets that carry a figure: a -% question whose text names an image in a folder beside the sheet, and its -% solutions under a heading. Nothing here is copied from ExampleContents. -\documentclass[12pt]{article} -\usepackage{amsmath} -\usepackage{graphicx} - -\begin{document} - -\section*{Problem Sheet 1: Statics} - -The beam below carries a load $W$ at its midpoint. -\includegraphics[width=0.2\textwidth]{figures/ball.png} - -\begin{enumerate} -\item Find the reaction at each support. -\item Find the bending moment at the midpoint. -\end{enumerate} - -\section*{Solutions} - -1(a) $W/2$ at each support. - -1(b) $M = WL/4$ - -\end{document} diff --git a/tests/fixtures/tex-sheet-spec.yaml b/tests/fixtures/tex-sheet-spec.yaml deleted file mode 100644 index ad39e4c..0000000 --- a/tests/fixtures/tex-sheet-spec.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# The spec for the tex set: a sheet whose questions are the items of one -# enumerate, with its title the only heading and no solutions in the file. -ignore: Header -question: ListItem -strip: ['^\d+\.\s+'] -layout: PartsSepSol diff --git a/tests/test_cli.py b/tests/test_cli.py index ea954ea..39e1beb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ """The command line the design spec describes.""" -import getpass +import argparse import subprocess import sys from pathlib import Path @@ -9,79 +9,12 @@ import pytest from conftest import FakeBackend, FakeMathpix -from in2lambda_agent import cli, compare, gate, pipeline, routes, sweep, targets -from in2lambda_agent.cli import build_parser, main, reviewer_name +from in2lambda_agent import cli, compare, gate, routes, sweep, targets +from in2lambda_agent.cli import build_parser, main from in2lambda_agent.model import ModelUnavailable, Usage from in2lambda_agent.settings import Settings -def test_defaults(): - args = build_parser().parse_args(["run", "sheet.md"]) - - assert args.source == Path("sheet.md") - assert args.spec is None - assert args.review == "none" - assert args.rounds == 3 - assert args.out == Path("out") - assert args.cache == Path(".in2lambda-agent") - assert args.fresh_ocr is False - assert args.sample == 3 - assert args.tries == 3 - - -def test_every_option(): - args = build_parser().parse_args( - [ - "run", - "sheet.md", - "--spec", - "sheet.yaml", - "--review", - "per-question", - "--rounds", - "5", - "--tries", - "2", - "--sample", - "2", - "--cache", - "cached", - "--fresh-ocr", - "--out", - "somewhere", - ] - ) - - assert args.spec == Path("sheet.yaml") - assert args.review == "per-question" - assert args.rounds == 5 - assert args.out == Path("somewhere") - assert args.cache == Path("cached") - assert args.fresh_ocr is True - assert args.sample == 2 - assert args.tries == 2 - - -@pytest.mark.parametrize("mode", ["none", "sample", "per-question"]) -def test_review_modes(mode): - assert build_parser().parse_args(["run", "s.md", "--review", mode]).review == mode - - -def test_an_unknown_review_mode_is_rejected(): - with pytest.raises(SystemExit): - build_parser().parse_args(["run", "s.md", "--review", "everything"]) - - -@pytest.mark.parametrize("count", ["0", "-1"]) -def test_a_sample_of_no_questions_is_refused(count, capsys): - # It would stop the run, write a record with nothing in it to approve, and - # never build: there would be no way on from there but to delete the record. - with pytest.raises(SystemExit): - build_parser().parse_args(["run", "sheet.md", "--sample", count]) - - assert "at least one question" in capsys.readouterr().err - - # --- convert --------------------------------------------------------------- @@ -124,21 +57,14 @@ def test_convert_every_option(): assert written.filter is None -@pytest.mark.parametrize("command", ["convert", "run"]) -def test_a_filter_and_a_written_filter_together_are_refused(command): +def test_a_filter_and_a_written_filter_together_are_refused(): # One run has one route B filter: either the file named or the file written. with pytest.raises(SystemExit): build_parser().parse_args( - [command, "sheet.md", "--filter", "set.lua", "--write-filter"] + ["convert", "sheet.md", "--filter", "set.lua", "--write-filter"] ) -def test_run_converts_through_both_routes_unless_the_spec_route_is_asked_for(): - assert build_parser().parse_args(["run", "sheet.md"]).route == "direct" - assert build_parser().parse_args(["run", "s.md", "--route", "spec"]).route == "spec" - assert build_parser().parse_args(["run", "s.md", "--solutions", "s2.md"]).solutions - - def converted(zip_path=None, **counts): """What a monkeypatched `routes.convert` answers with.""" return routes.Converted( @@ -215,19 +141,18 @@ def test_convert_takes_the_solutions_document_beside_the_document( assert given["solutions"] == tmp_path / "sheet_solutions.md" -@pytest.mark.parametrize("command", ["convert", "run"]) def test_convert_named_by_its_solutions_document_converts_the_pair( - command, tmp_path, backend, monkeypatch, capsys + tmp_path, backend, monkeypatch, capsys ): - # Naming either half of a pair converts the pair, and the set is named after the - # questions document, as the spec route has always done. + # Naming either half of a pair converts the pair, and the set is named after + # the questions document. (tmp_path / "Worksheet_1.md").write_text("x") (tmp_path / "Worksheet_1_solutions.md").write_text("x") given = {} monkeypatch.setattr(routes, "convert", records(given, converted(tmp_path / "s.zip"))) code = main( - [command, str(tmp_path / "Worksheet_1_solutions.md"), "--out", str(tmp_path)] + ["convert", str(tmp_path / "Worksheet_1_solutions.md"), "--out", str(tmp_path)] ) assert code == 0 @@ -336,17 +261,6 @@ def test_convert_names_a_file_that_is_not_there(missing, tmp_path, backend, caps assert ("sol.md" if missing == "solutions" else "sheet.md") in printed.err -def test_run_without_a_route_converts_the_document(tmp_path, backend, monkeypatch): - given = {} - monkeypatch.setattr(routes, "convert", records(given, converted(tmp_path / "s.zip"))) - monkeypatch.setattr( - pipeline, "run", lambda *a, **k: pytest.fail("the spec route ran") - ) - - assert main(["run", str(tmp_path / "sheet.md"), "--out", str(tmp_path)]) == 0 - assert given["document"] == tmp_path / "sheet.md" - - def test_the_written_filter_is_kept_in_the_out_directory( tmp_path, backend, monkeypatch ): @@ -433,45 +347,6 @@ def test_convert_reports_a_reply_that_is_not_a_list_of_questions( assert "fields" not in printed.out and "build" not in printed.out -@pytest.mark.parametrize( - "given, message", - [ - ( - ["--review", "per-question"], - "--review is an option of the spec route; add --route spec", - ), - ( - ["--spec", "set.yaml"], - "--spec is an option of the spec route; add --route spec", - ), - ( - ["--route", "spec", "--write-filter"], - "--write-filter is an option of the direct route; drop --route spec", - ), - ( - ["--route", "spec", "--solutions", "sol.md"], - "--solutions is an option of the direct route; drop --route spec", - ), - ], -) -def test_an_option_of_the_other_route_is_refused(given, message, monkeypatch, capsys): - # Under the route it does not belong to the option would be parsed and - # thrown away: a saved spec ignored and written again by two model calls, a - # review never stopped for, a filter never written. So the run says so, and - # says so before it has paid for anything. - monkeypatch.setattr( - cli, "choose_backend", lambda settings: pytest.fail("a model was asked for") - ) - monkeypatch.setattr( - pipeline, "run", lambda *a, **k: pytest.fail("the spec route ran") - ) - - code = main(["run", "sheet.md", *given]) - - assert code == 1 - assert capsys.readouterr().err.strip() == f"in2lambda-agent: {message}" - - def test_corpus_defaults(): args = build_parser().parse_args(["corpus", "ExampleContents"]) @@ -618,22 +493,34 @@ def record(*args, **kwargs): def test_gate_defaults(): - args = build_parser().parse_args(["gate", "gate-baseline.json"]) + args = build_parser().parse_args( + ["gate", "ci-corpus/targets", "--filters", "ci-corpus/filters"] + ) assert args.command == "gate" - assert args.baseline == Path("gate-baseline.json") - assert args.record is False + assert args.root == Path("ci-corpus/targets") + assert args.paths == [] + assert args.filters == Path("ci-corpus/filters") assert args.cache == Path.home() / ".cache" / "in2lambda-agent" # Chosen when the command runs, so that two runs do not share a directory. assert args.work is None +def test_a_gate_with_no_filter_tree_is_refused(): + # There is nothing saved to replay without one, and a gate that wrote what + # is missing would be making the calls it exists not to make. + with pytest.raises(SystemExit): + build_parser().parse_args(["gate", "ci-corpus/targets"]) + + def test_gate_every_option(): args = build_parser().parse_args( [ "gate", - "saved.json", - "--record", + "ci-corpus/targets", + "sheet", + "--filters", + "saved", "--cache", "cached", "--work", @@ -641,63 +528,46 @@ def test_gate_every_option(): ] ) - assert args.record is True + assert args.paths == [Path("sheet")] + assert args.filters == Path("saved") assert args.cache == Path("cached") assert args.work == Path("working") -def test_a_gate_that_passes_exits_zero(tmp_path, monkeypatch, capsys): - path = written_baseline(tmp_path) - report = gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) - monkeypatch.setattr(gate, "run", lambda *args, **kwargs: report) - - code = main(["gate", str(path)]) - - assert code == 0 - assert "tex" in capsys.readouterr().out - - -def test_a_gate_that_fails_exits_one_and_says_what_the_folder_built( - tmp_path, monkeypatch, capsys +def test_a_gate_with_no_new_difference_passes_and_says_where_it_worked( + monkeypatch, capsys ): - path = written_baseline(tmp_path) - summary = gate.Summary(built=1, counts={"faulted": 1}, recorded=2) - monkeypatch.setattr( - gate, "run", lambda *args, **kwargs: gate.Report(folders={"tex": summary}) - ) - - code = main(["gate", str(path)]) - - assert code == 1 - assert "tex" in capsys.readouterr().out - - -def test_recording_writes_the_baseline_and_exits_zero(tmp_path, monkeypatch): - path = written_baseline(tmp_path) + given = {} - def record(baseline, **kwargs): - baseline.folders["tex"].built = 2 - return gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + def record(root, **kwargs): + given.update(root=root, **kwargs) + return [targets.Result(name="sheet")] monkeypatch.setattr(gate, "run", record) - code = main(["gate", str(path), "--record"]) + code = main(["gate", "ci-corpus/targets", "--filters", "ci-corpus/filters"]) + printed = capsys.readouterr().out assert code == 0 - assert gate.read_baseline(path).folders["tex"].built == 2 + assert given["root"] == Path("ci-corpus/targets") + assert given["filters"] == Path("ci-corpus/filters") + assert given["cache"] == gate.DEFAULT_CACHE_DIR + # The directory is printed and not deleted, so the sets can be read after. + assert printed.startswith("work ") + assert printed.endswith("1 target, 0 new differences\n") -def written_baseline(tmp_path): - """A baseline file on disk, for the gate command to read.""" - path = tmp_path / "baseline.json" - gate.write_baseline( - gate.Baseline( - specs=Path("corpus-specs"), - folders={"tex": gate.Folder(root=tmp_path / "corpus", suffixes=["tex"])}, - ), - path, +def test_a_target_the_gate_could_not_replay_fails_the_run(monkeypatch, capsys): + monkeypatch.setattr( + gate, + "run", + lambda *args, **kwargs: [ + targets.Result(name="sheet", error="reply.json is not saved") + ], ) - return path + + assert main(["gate", "ci-corpus/targets", "--filters", "saved"]) == 1 + assert "1 did not run" in capsys.readouterr().out def test_compare_defaults(): @@ -805,105 +675,29 @@ def test_a_subcommand_is_required(): build_parser().parse_args([]) -def test_approving_a_question(): - args = build_parser().parse_args(["review", "approve", "q2", "--cache", "cached"]) - - assert (args.command, args.verdict, args.question) == ("review", "approve", "q2") - assert args.cache == Path("cached") - - -def test_rejecting_a_question_carries_a_note(): - args = build_parser().parse_args( - ["review", "reject", "q2", "--note", "part (b) is missing"] - ) - - assert (args.verdict, args.question, args.note) == ( - "reject", - "q2", - "part (b) is missing", - ) - assert args.cache == Path(".in2lambda-agent") - - -def test_a_rejection_without_a_note_is_refused(): - with pytest.raises(SystemExit): - build_parser().parse_args(["review", "reject", "q2"]) - - -def test_an_edit_names_the_field_the_wording_and_the_reviewer(): - args = build_parser().parse_args( - ["review", "edit", "q1.text", "m/s", "m/s^2", "--by", "ada"] +def test_the_commands_are_the_ones_the_agent_has(): + # The spec route's `run` and `review` are gone, and the parser is where a + # user finds that out. + action = next( + one + for one in build_parser()._actions + if isinstance(one, argparse._SubParsersAction) ) - assert (args.verdict, args.field, args.old, args.new) == ( - "edit", - "q1.text", - "m/s", - "m/s^2", - ) - assert args.by == "ada" - - -def test_an_edit_is_by_whoever_is_logged_in_unless_they_say(): - args = build_parser().parse_args(["review", "edit", "q1.text", "a", "b"]) - - # Nothing is asked of the system while the arguments are being parsed: the - # name is resolved on the review branch and nowhere else. - assert args.by is None - assert reviewer_name(args.by) == getpass.getuser() - assert reviewer_name("ada") == "ada" - - -def test_an_edit_is_by_the_reviewer_where_there_is_no_login_name(monkeypatch): - monkeypatch.setattr( - getpass, "getuser", lambda: (_ for _ in ()).throw(OSError("no passwd entry")) - ) - - assert reviewer_name(None) == "reviewer" - - -def test_a_run_parses_where_there_is_no_login_name(monkeypatch, tmp_path): - # A container started with `--user 1001` and no LOGNAME: `run` never wants - # a reviewer's name, so it must not be asked for one to get to the parser. - monkeypatch.setattr( - getpass, "getuser", lambda: (_ for _ in ()).throw(OSError("no passwd entry")) - ) - called = {} - - def record(source, **given): - called["source"] = source - return pipeline.RunResult(zip_path=tmp_path / "set.zip") - - monkeypatch.setattr(pipeline, "run", record) - - assert main(["run", "sheet.md", "--route", "spec"]) == 0 - assert called["source"] == Path("sheet.md") - - -def test_how_many_specs_may_be_written_reaches_the_run(monkeypatch, tmp_path): - given = {} - - def record(source, **passed): - given.update(passed) - return pipeline.RunResult(zip_path=tmp_path / "set.zip") - - monkeypatch.setattr(pipeline, "run", record) - - assert main(["run", "sheet.md", "--route", "spec", "--tries", "5"]) == 0 - assert given["tries"] == 5 - - -@pytest.mark.parametrize("count", ["0", "-1"]) -def test_a_run_that_may_write_no_spec_is_refused(count, capsys): - with pytest.raises(SystemExit): - build_parser().parse_args(["run", "s.md", "--tries", count]) - - assert "at least one spec" in capsys.readouterr().err + assert list(action.choices) == [ + "convert", + "corpus", + "targets", + "gate", + "compare", + "ui", + ] -def test_a_verdict_is_required(): +@pytest.mark.parametrize("gone", ["run", "review"]) +def test_the_spec_routes_commands_are_refused(gone): with pytest.raises(SystemExit): - build_parser().parse_args(["review"]) + build_parser().parse_args([gone, "sheet.md"]) def test_ui_defaults_and_every_option(): diff --git a/tests/test_corpus.py b/tests/test_corpus.py deleted file mode 100644 index 7164d0f..0000000 --- a/tests/test_corpus.py +++ /dev/null @@ -1,688 +0,0 @@ -"""The sweep over a corpus: one row per document, and a replay with no model in it.""" - -import json -import shutil -from dataclasses import asdict -from pathlib import Path - -import pytest -from conftest import FakeBackend -from test_pipeline import FAULTY_SPEC, FIXES, LONE_SPEC, PAIRED_SPEC, SPEC, TEX_SPEC - -from in2lambda_agent import corpus, package, pipeline -from in2lambda_agent.model import ModelError -from in2lambda_agent.settings import Settings -from in2lambda_agent.spec import SPEC_NAME - -FIXTURES = Path(__file__).parent / "fixtures" -TEX_FIGURE_SPEC = (FIXTURES / "tex-figure-spec.yaml").read_text() - - -def make_set(root, folder, names): - """One document set of the miniature corpus: a folder of sheets.""" - made = root / folder - made.mkdir(parents=True) - for name in names: - shutil.copy(FIXTURES / name, made / name) - return made - - -@pytest.fixture -def root(tmp_path): - """A corpus of two sets, a markdown one and a tex one.""" - made = tmp_path / "corpus" - make_set(made, "sheets", ["sheet.md", "sheet-2.md"]) - make_set(made, "tex", ["tex-sheet.tex", "tex-sheet-2.tex"]) - return made - - -def sweep(root, tmp_path, **kwargs): - """A sweep with its three directories under tmp_path rather than the corpus.""" - return corpus.sweep( - root, - results=tmp_path / "results.csv", - work=tmp_path / "work", - specs=kwargs.pop("specs", tmp_path / "specs"), - settings=Settings(), - **kwargs, - ) - - -def contents(folder): - """Every file under a folder, by its path relative to it.""" - return sorted( - path.relative_to(folder).as_posix() - for path in folder.rglob("*") - if path.is_file() - ) - - -def test_a_fresh_sweep_calls_once_per_set_and_the_rows_say_which(root, tmp_path): - backend = FakeBackend(SPEC, TEX_SPEC) - before = contents(root) - - rows = sweep(root, tmp_path, backend=backend) - - # One call per set, and the sheets of a set run together and in order, so - # the first of them writes the spec and the rest reuse it. - assert len(backend.calls) == 2 - assert [row.source for row in rows] == [ - "sheets/sheet-2.md", - "sheets/sheet.md", - "tex/tex-sheet-2.tex", - "tex/tex-sheet.tex", - ] - assert [row.set for row in rows] == ["sheets", "sheets", "tex", "tex"] - assert [row.spec for row in rows] == ["wrote", "reused", "wrote", "reused"] - assert [row.outcome for row in rows] == ["built"] * 4 - # A zip was written, so the reason says what the build went past rather - # than why there is none: nothing for the sheets, whose solutions are on - # them, and the no-solution warnings for the tex set, whose are not. - assert [row.reason for row in rows[:2]] == ["", ""] - assert all("has no solution" in row.reason for row in rows[2:]) - # The specs are kept in a tree mirroring the corpus, which is what makes a - # later sweep a replay. - assert (tmp_path / "specs" / "sheets" / SPEC_NAME).read_text() == SPEC - assert (tmp_path / "specs" / "tex" / SPEC_NAME).read_text() == TEX_SPEC - # And the corpus is untouched: every run happened in the work directory. - assert contents(root) == before - - -def test_the_row_says_what_the_spec_made_of_the_document(root, tmp_path): - rows = sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC)) - sheet = next(row for row in rows if row.source == "sheets/sheet.md") - - assert (sheet.layout, sheet.blocks, sheet.fields) == ("PartsSepSol", 14, 10) - assert (sheet.layer1, sheet.layer3, sheet.edited) == (10, 0, 0) - assert (sheet.unassigned, sheet.rounds) == (0, 0) - assert (sheet.review, sheet.rejections) == ("none", 0) - # The reused run made no call; the wall clock still ran. - assert sheet.input_tokens == sheet.output_tokens == 0 - assert sheet.model_seconds == 0.0 and sheet.wall_seconds > 0 - - -def test_the_sweeps_cache_is_where_each_run_looks_for_its_ocr( - root, tmp_path, monkeypatch -): - given = [] - - def record(source, **kwargs): - given.append(kwargs["cache_dir"]) - raise RuntimeError("as far as this goes") - - monkeypatch.setattr(pipeline, "run", record) - - rows = sweep(root, tmp_path, cache=tmp_path / "shared") - - assert given == [tmp_path / "shared"] * len(rows) - - -def test_a_saved_spec_and_its_source_replay_with_no_model_call(root, tmp_path): - for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): - saved = tmp_path / "specs" / folder / SPEC_NAME - saved.parent.mkdir(parents=True) - saved.write_text(text) - backend = FakeBackend(reason="a replay makes no call") - - rows = sweep(root, tmp_path, replay=True, backend=backend) - again = sweep(root, tmp_path, replay=True, backend=backend) - - assert backend.calls == [] - assert [row.outcome for row in rows] == ["built"] * 4 - assert [row.spec for row in rows] == ["reused"] * 4 - assert all(row.input_tokens == row.output_tokens == 0 for row in rows) - # Deterministic: a saved spec plus its source is the same draft every time, - # and only the clocks differ between two replays of it. - assert [without_clocks(row) for row in rows] == [ - without_clocks(row) for row in again - ] - - -def test_a_sweep_keeps_each_documents_log_beside_the_sets_spec(tmp_path): - root = tmp_path / "corpus" - make_set(root, "faulty", ["faulty.md"]) - - sweep(root, tmp_path, tries=1, backend=FakeBackend(FAULTY_SPEC, FIXES)) - saved = tmp_path / "specs" / "faulty" / f"faulty.md{corpus.COMMANDS_SUFFIX}" - draft = package.draft_of(tmp_path / "work" / "faulty" / "faulty.md") - - # The round's commands, and not the spec run before them: a replay runs the - # set's spec itself. - assert json.loads(saved.read_text()) == package.fix_log(draft) - assert [one["command"] for one in json.loads(saved.read_text())] == [ - "split block", - "question add", - "part add", - "question solution", - "field replace", - ] - # The log and nothing beside it: the draft's `fields`, which hold every - # field's captured text, stay in the work directory. - assert "fields" not in saved.read_text() - - -def test_a_document_that_took_no_round_keeps_an_empty_log(root, tmp_path): - sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC)) - saved = tmp_path / "specs" / "sheets" / f"sheet.md{corpus.COMMANDS_SUFFIX}" - - # Written all the same: a replay that finds no file there reads it as a - # document no sweep has run. - assert json.loads(saved.read_text()) == [] - - -def test_a_replay_runs_the_saved_log_and_builds_what_the_rounds_repaired( - tmp_path, monkeypatch -): - root = tmp_path / "corpus" - make_set(root, "faulty", ["faulty.md"]) - (swept,) = sweep(root, tmp_path, tries=1, backend=FakeBackend(FAULTY_SPEC, FIXES)) - monkeypatch.setattr( - pipeline, - "choose_backend", - lambda settings: pytest.fail("a replay chose a backend"), - ) - - (row,) = sweep(root, tmp_path, replay=True, backend=None) - - # The sweep took a round to build this document, and the replay builds it - # with the same fields and no model call at all. - assert (swept.outcome, row.outcome) == ("built", "built") - assert (row.rounds, row.input_tokens, row.output_tokens) == (0, 0, 0) - assert (row.layer3, row.edited) == (swept.layer3, swept.edited) == (3, 1) - - -def test_a_log_in2lambda_refuses_names_the_command_in_the_rows_reason(tmp_path): - root = tmp_path / "corpus" - make_set(root, "faulty", ["faulty.md"]) - sweep(root, tmp_path, tries=1, backend=FakeBackend(FAULTY_SPEC, FIXES)) - saved = tmp_path / "specs" / "faulty" / f"faulty.md{corpus.COMMANDS_SUFFIX}" - written = json.loads(saved.read_text()) - written[0]["args"]["block"] = "b99" - saved.write_text(json.dumps(written)) - - (row,) = sweep(root, tmp_path, replay=True, backend=None) - - assert row.outcome == "replay refused" - assert "command 1 of 5, split block" in row.reason - - -def test_a_replay_with_nothing_saved_says_so_and_still_makes_no_call(root, tmp_path): - backend = FakeBackend(reason="a replay makes no call") - - rows = sweep(root, tmp_path, replay=True, specs=tmp_path / "none", backend=backend) - - assert backend.calls == [] - assert [row.outcome for row in rows] == ["no spec"] * 4 - assert not (tmp_path / "none" / "sheets" / SPEC_NAME).exists() - - -def test_a_replay_refuses_the_call_even_with_no_backend_handed_to_it( - root, tmp_path, monkeypatch -): - # The path the CLI runs: it hands the sweep no backend, so without one of - # its own a replay over a set with nothing saved would reach the settings' - # backend and spend the call the replay promises not to make. - monkeypatch.setattr( - pipeline, - "choose_backend", - lambda settings: pytest.fail("a replay chose a backend"), - ) - - rows = sweep(root, tmp_path, replay=True, specs=tmp_path / "none", backend=None) - - assert [row.outcome for row in rows] == ["no spec"] * 4 - - -def test_the_table_is_written_with_the_columns_in_order(root, tmp_path): - sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC)) - written = (tmp_path / "results.csv").read_text().splitlines() - - assert written[0] == ",".join(corpus.COLUMNS) - assert [line.split(",")[0] for line in written[1:]] == [ - "sheets/sheet-2.md", - "sheets/sheet.md", - "tex/tex-sheet-2.tex", - "tex/tex-sheet.tex", - ] - - -def test_the_rounds_a_document_took_are_counted_by_layer(tmp_path): - root = tmp_path / "corpus" - make_set(root, "faulty", ["faulty.md"]) - - (row,) = sweep(root, tmp_path, tries=1, backend=FakeBackend(FAULTY_SPEC, FIXES)) - - # The spec's own fields, then the three the round quoted out of the source, - # one of which it went on to edit. - assert row.outcome == "built" - assert (row.layer3, row.edited, row.rounds) == (3, 1, 1) - assert row.layer1 > 0 and row.layer2 == row.layer4 == 0 - # The column the shares are taken against is the whole draft's, not the - # spec run's: a document that took a round writes fields after it. - assert row.fields == row.layer1 + row.layer3 - assert row.input_tokens > 0 and row.output_tokens > 0 - - -def test_a_refused_build_is_its_own_outcome_and_not_a_faulted_draft( - tmp_path, monkeypatch -): - # Two sets a row can tell apart only by what the checks said: the sheet's - # spec covers it and its report comes clean, and in2lambda still will not - # write it out, while the faulty sheet's own checks fault and a replay has - # no round to answer them with. The refusal is stubbed because every - # refusal in2lambda has today — a missing image among them — is now a - # finding of the report instead, which is a faulted draft and not this. - root = tmp_path / "corpus" - make_set(root, "sheets", ["sheet.md"]) - make_set(root, "faulty", ["faulty.md"]) - for folder, text in (("sheets", SPEC), ("faulty", FAULTY_SPEC)): - saved = tmp_path / "specs" / folder / SPEC_NAME - saved.parent.mkdir(parents=True) - saved.write_text(text) - monkeypatch.setattr( - pipeline.package, - "build", - lambda draft, out_dir: (_ for _ in ()).throw( - corpus.package.BuildRefused("figures/ball.png is not beside the draft") - ), - ) - - faulted, refused = sweep(root, tmp_path, replay=True) - - assert (faulted.source, faulted.outcome) == ("faulty/faulty.md", "faulted") - assert (refused.source, refused.outcome) == ("sheets/sheet.md", "build refused") - # The draft was made and the checks came clean: the export is what stopped, - # which the rounds column would otherwise read as a spec that never covered - # the sheet. - assert (refused.rounds, refused.spec) == (0, "reused") - assert refused.layer1 > 0 - # And each says why, in the words of what stopped it: the refusal itself, - # without the stage line's prefix, and one of the findings rather than the - # whole line the validate stage joined them into. - assert "figures/ball.png" in refused.reason - assert not refused.reason.startswith("refused: ") - assert faulted.reason and "; " not in faulted.reason - - -def test_a_questions_only_set_is_built_and_its_warnings_are_the_reason(tmp_path): - # The sheets whose solutions are not on them, which a sweep has to report - # as built rather than faulted — with what the build went past in the one - # column that says it. - root = tmp_path / "corpus" - make_set(root, "unanswered", ["questions-only.md"]) - saved = tmp_path / "specs" / "unanswered" / SPEC_NAME - saved.parent.mkdir(parents=True) - saved.write_text(SPEC) - - (row,) = sweep(root, tmp_path, replay=True) - - assert (row.outcome, row.rounds, row.spec) == ("built", 0, "reused") - assert row.reason.count("has no solution") == 4 - assert "\n" not in row.reason - - -def test_a_spec_in2lambda_refuses_says_so_in_the_row(root, tmp_path, monkeypatch): - # A spec the package will not run, which is a row rather than the end of the - # sweep — and the message is the only thing that says which set's spec. - running = corpus.package.spec_run - - def refuse(draft, spec): - if "tex" in str(spec): - raise corpus.SpecRejected("selector `question` matches\nno node") - return running(draft, spec) - - monkeypatch.setattr(pipeline.package, "spec_run", refuse) - - rows = sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC, TEX_SPEC)) - rejected = [row for row in rows if row.set == "tex"] - - assert [row.outcome for row in rejected] == ["spec rejected"] * 2 - # One line, whatever the message did with its own. - assert all( - row.reason == "selector `question` matches no node" for row in rejected - ) - - -def test_a_replay_with_nothing_saved_says_why_in_the_row(root, tmp_path): - rows = sweep( - root, - tmp_path, - replay=True, - specs=tmp_path / "none", - backend=FakeBackend(reason="a replay makes no call"), - ) - - assert [row.outcome for row in rows] == ["no spec"] * 4 - assert all(row.reason == corpus.NoModel().unavailable() for row in rows) - - -def test_a_set_that_cannot_be_staged_is_a_row_and_the_table_is_still_written( - root, tmp_path, monkeypatch -): - # A folder the copy cannot read. Monkeypatched rather than made, since what - # a permission bit does depends on who is running the tests. - staging = corpus.stage - - def refuse(root, folder, work, suffixes): - if folder.name == "sheets": - raise PermissionError(folder) - return staging(root, folder, work, suffixes) - - monkeypatch.setattr(corpus, "stage", refuse) - - rows = sweep(root, tmp_path, backend=FakeBackend(TEX_SPEC)) - - # Every sheet of the set is a row, since none of them ran, and the set after - # it runs as it would have. - assert [(row.source, row.outcome) for row in rows] == [ - ("sheets/sheet-2.md", "error: PermissionError"), - ("sheets/sheet.md", "error: PermissionError"), - ("tex/tex-sheet-2.tex", "built"), - ("tex/tex-sheet.tex", "built"), - ] - # What the copy raised, so the folder it could not read is in the table. - assert all("sheets" in row.reason for row in rows[:2]) - written = (tmp_path / "results.csv").read_text().splitlines() - assert len(written) == 5 - - -def test_a_file_that_cannot_be_read_is_a_row_and_the_table_is_still_written( - root, tmp_path, monkeypatch -): - # A file that goes, or that cannot be opened, while the sweep is deciding - # whether it is a document at all. Monkeypatched rather than made, since - # what a permission bit does depends on who is running the tests. - reading = corpus.is_document - - def refuse(path): - if path.name == "tex-sheet.tex": - raise PermissionError(path) - return reading(path) - - monkeypatch.setattr(corpus, "is_document", refuse) - - rows = sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC)) - - # The one file is a row, the sheets beside it ran, and the table was written. - assert [(row.source, row.outcome) for row in rows] == [ - ("sheets/sheet-2.md", "built"), - ("sheets/sheet.md", "built"), - ("tex/tex-sheet-2.tex", "built"), - ("tex/tex-sheet.tex", "error: PermissionError"), - ] - assert "tex-sheet.tex" in rows[-1].reason - assert len((tmp_path / "results.csv").read_text().splitlines()) == 5 - - -@pytest.mark.parametrize( - "paths", - [(Path("sheets"),), (Path("."), Path("sheets")), (Path("sheets"), Path("sheets"))], -) -def test_overlapping_paths_name_a_document_once(root, tmp_path, paths): - # `.` holds `sheets`, and a folder may be named twice: either way the same - # document run twice would be two rows disagreeing about its spec, one - # saying it wrote it and the other that it reused it. - shutil.rmtree(root / "tex") - - rows = sweep(root, tmp_path, paths=paths, backend=FakeBackend(SPEC)) - - assert [(row.source, row.spec) for row in rows] == [ - ("sheets/sheet-2.md", "wrote"), - ("sheets/sheet.md", "reused"), - ] - - -def test_a_document_that_raises_is_a_row_and_not_the_end_of_the_sweep( - root, tmp_path -): - # A file that says it is a docx and is not: pandoc refuses it, and the sweep - # is over the sets after it as well as the one it is in. - broken = root / "broken" - broken.mkdir() - (broken / "notes.docx").write_bytes(b"not really a docx") - - rows = sweep(root, tmp_path, suffixes=("md", "docx"), backend=FakeBackend(SPEC)) - - assert rows[0].source == "broken/notes.docx" - assert rows[0].outcome.startswith("error: ") - assert [row.outcome for row in rows[1:]] == ["built", "built"] - assert [row.source for row in rows[1:]] == ["sheets/sheet-2.md", "sheets/sheet.md"] - - -def test_a_spec_call_that_did_not_finish_names_the_call_and_the_cause(tmp_path): - root = tmp_path / "corpus" - make_set(root, "broken", ["sheet.md"]) - make_set(root, "sheets", ["sheet.md"]) - stopped = ModelError( - "Claude Code returned an error result: Reached maximum number of turns (8)" - ) - - rows = sweep(root, tmp_path, backend=FakeBackend(stopped, SPEC)) - - assert (rows[0].outcome, rows[0].reason) == ("spec failed", str(stopped)) - # And the set after it still runs. - assert (rows[1].source, rows[1].outcome) == ("sheets/sheet.md", "built") - - -def test_a_fixing_round_that_did_not_finish_says_so_and_not_spec_failed(tmp_path): - root = tmp_path / "corpus" - make_set(root, "faulty", ["faulty.md"]) - stopped = ModelError("the agent-sdk backend stopped on error_during_execution: None") - - # The one spec call answers, so what did not finish is the round the checks - # ask for. - (row,) = sweep(root, tmp_path, tries=1, backend=FakeBackend(FAULTY_SPEC, stopped)) - - assert (row.outcome, row.reason) == ("fix failed", str(stopped)) - - -def test_a_staged_set_leaves_behind_what_no_run_reads(root, tmp_path): - (root / "sheets" / "scan.pdf").write_bytes(b"%PDF-1.4 most of what a corpus weighs") - (root / "sheets" / "sources.zip").write_bytes(b"PK the rest of what it weighs") - (root / "sheets" / "figures").mkdir() - (root / "sheets" / "figures" / "plot.png").write_bytes(b"PNG") - (root / "sheets" / "figures" / "page.pdf").write_bytes(b"%PDF-1.4") - # What a run in the corpus itself would have left: a draft beside the sheet - # it was frozen from, and a cache directory — here under the figures, to - # show that the depth it sits at makes no difference. - (root / "sheets" / ("sheet" + package.DRAFT_SUFFIX)).write_text("{}") - cache = root / "sheets" / "figures" / pipeline.DEFAULT_CACHE_DIR.name - cache.mkdir() - (cache / "sheet.md").write_text("# an OCR conversion from a run before") - - staged = corpus.stage(root, root / "sheets", tmp_path / "work", ("md",)) - - # The figures come along, since the sheets refer to them; the PDFs do not, - # in the folder itself or under it, and neither does anything an earlier - # run wrote. - assert contents(staged) == ["figures/plot.png", "sheet-2.md", "sheet.md"] - - -def test_a_set_with_figures_in_a_subfolder_sweeps_to_built(tmp_path): - # What PHYS40002's sheets do: name an image in a folder beside them. The - # folder has to come along, or the build faults on a file it cannot find. - # The caption among the figures is a document, so the folder is a set of - # its own when the sweep reaches it — and that is no reason to leave it - # behind when the sheet that names its image is staged. - made = tmp_path / "corpus" - folder = make_set(made, "beams", ["tex-figure.tex"]) - (folder / "figures").mkdir() - shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") - (folder / "figures" / "caption.md").write_text("# Figure 1\n\nA loaded beam.\n") - - rows = sweep(made, tmp_path, backend=FakeBackend(TEX_FIGURE_SPEC, TEX_FIGURE_SPEC)) - - (row,) = [one for one in rows if one.source == "beams/tex-figure.tex"] - # The build compiles the set as the PDF generator does, so it is `built` - # only because the image was there to load. - assert row.outcome == "built", row.reason - assert "figures/ball.png" in contents(tmp_path / "work" / "beams") - - -def test_the_corpus_root_is_a_set_of_its_own_and_wipes_nothing_but_itself( - root, tmp_path -): - # A corpus with documents loose at its top, ExampleContents among them: the - # set's folder is the root, whose copy would otherwise be the work directory - # and take the sets already staged beside it — and anything else a - # user-named --work holds — with it when it is emptied. - shutil.copy(FIXTURES / "sheet.md", root / "loose.md") - (root / "figures").mkdir() - (root / "figures" / "plot.png").write_bytes(b"PNG") - work = tmp_path / "work" - already = corpus.stage(root, root / "sheets", work, ("md", "tex")) - (work / "not-the-sweep's.txt").write_text("a user's own --work") - - staged = corpus.stage(root, root, work, ("md", "tex")) - corpus.stage(root, root, work, ("md", "tex")) - - assert staged.parent == work and staged.name == corpus.ROOT_SET - # Everything under the root, the sets beneath it included: they are staged - # again under their own names when the sweep reaches them, and a second - # copy of them here costs less than deciding per folder what to bring. - assert contents(staged) == [ - "figures/plot.png", - "loose.md", - "sheets/sheet-2.md", - "sheets/sheet.md", - "tex/tex-sheet-2.tex", - "tex/tex-sheet.tex", - ] - assert contents(already) == ["sheet-2.md", "sheet.md"] - assert (work / "not-the-sweep's.txt").exists() - - -def test_a_work_directory_inside_the_corpus_is_not_copied_into_itself(root, tmp_path): - # The default --work sits under the corpus root, which is a set of its own: - # its copy would hold the work directory, which holds that copy, and so on - # until the path is too long for the filesystem. So the work directory is - # left behind by where it is, whatever the user named it. - work = root / "work" - corpus.stage(root, root / "sheets", work, ("md",)) - - staged = corpus.stage(root, root, work, ("md",)) - - assert contents(staged) == [ - "sheets/sheet-2.md", - "sheets/sheet.md", - "tex/tex-sheet-2.tex", - "tex/tex-sheet.tex", - ] - - -TIKZ = """\\begin{tikzpicture} - \\draw[->] (0,0) -- (4,0) node[right] {$x$}; -\\end{tikzpicture} -""" - - -def test_a_tex_file_that_is_a_drawing_is_skipped_and_comes_with_its_set( - root, tmp_path -): - # What the first sweep of PHYS40002 made a set of: a figures/ folder of TikZ - # sources, staged and built on its own, and left out of the sheets that - # input it — so every sheet with a figure was refused for a missing image. - figures = root / "tex" / "figures" - figures.mkdir() - (figures / "tunnel-potential.tex").write_text(TIKZ) - - backend = FakeBackend(SPEC, TEX_SPEC) - rows = sweep(root, tmp_path, backend=backend) - - skipped = next(row for row in rows if "tunnel-potential" in row.source) - assert (skipped.source, skipped.set) == ( - "tex/figures/tunnel-potential.tex", - "tex/figures", - ) - assert (skipped.outcome, skipped.reason) == ("skipped", "no \\begin{document}") - # Nothing was frozen on its account: no draft beside it and no zip, and — - # since staging the set wipes the work folder the figures sit in, which is - # where those would have been — no spec written for it and no call made. - staged_figures = tmp_path / "work" / "tex" / "figures" - assert not corpus.package.draft_of( - staged_figures / "tunnel-potential.tex" - ).exists() - assert not (staged_figures / "out").exists() - assert not (tmp_path / "specs" / "tex" / "figures").exists() - assert len(backend.calls) == 2 - # And the sheets that input it have it. - assert "figures/tunnel-potential.tex" in contents(tmp_path / "work" / "tex") - assert [row.outcome for row in rows if row.set == "tex"] == ["built"] * 2 - - -def test_a_tex_file_is_a_document_only_with_a_begin_document_in_it(root): - assert corpus.is_document(root / "sheets" / "sheet.md") - assert corpus.is_document(root / "tex" / "tex-sheet.tex") - drawing = root / "tex" / "tunnel-potential.tex" - drawing.write_text(TIKZ) - assert not corpus.is_document(drawing) - - -def test_only_the_named_folders_are_run(root): - assert [path.name for path in corpus.documents(root, ["tex"])] == [ - "tex-sheet-2.tex", - "tex-sheet.tex", - ] - - -def without_clocks(row): - """A row without the two timings, which are what differs between two runs.""" - return { - key: value - for key, value in asdict(row).items() - if key not in ("model_seconds", "wall_seconds") - } - - -def test_a_sheet_and_its_solutions_file_are_one_row(tmp_path): - # UCL_MechEng writes each worksheet as two documents: Worksheet_1.pdf and - # Worksheet_1_solutions.pdf beside it. - root = tmp_path / "corpus" - folder = root / "worksheets" - folder.mkdir(parents=True) - shutil.copy(FIXTURES / "paired.md", folder / "Sheet_3.md") - shutil.copy(FIXTURES / "paired_solutions.md", folder / "Sheet_3_solutions.md") - backend = FakeBackend(PAIRED_SPEC) - - rows = sweep(root, tmp_path, backend=backend) - - assert [(row.source, row.set, row.outcome) for row in rows] == [ - ("worksheets/Sheet_3.md", "worksheets", "built") - ] - assert len(backend.calls) == 1 - # The solutions file was frozen as the draft's second source, so the four - # part solutions came out of it. - draft = corpus.package.draft_of(tmp_path / "work" / "worksheets" / "Sheet_3.md") - frozen = json.loads(draft.read_text()) - assert [one["source"] for one in frozen["sources"]] == [ - "Sheet_3.md", - "Sheet_3_solutions.md", - ] - assert [ - key - for key, written in frozen["fields"].items() - if written.get("source") == 2 and not key.endswith(".ignore") - ] == [ - "q1.p1.solution", - "q1.p2.solution", - "q2.p1.solution", - "q2.p2.solution", - ] - - -def test_solutions_with_no_questions_beside_them_are_converted_alone(tmp_path): - root = tmp_path / "corpus" - folder = root / "worksheets" - folder.mkdir(parents=True) - shutil.copy(FIXTURES / "lone_solutions.md", folder / "Tutorial_2_Solutions.md") - backend = FakeBackend(LONE_SPEC) - - rows = sweep(root, tmp_path, backend=backend) - - # A document of solutions is a set of its own, so the sweep runs it and - # records the row it built. - assert [(row.source, row.set, row.outcome) for row in rows] == [ - ("worksheets/Tutorial_2_Solutions.md", "worksheets", "built") - ] - assert len(backend.calls) == 1 - assert (tmp_path / "work" / "worksheets" / "Tutorial_2_Solutions.md").is_file() diff --git a/tests/test_docs.py b/tests/test_docs.py index 9bb3150..13e5d50 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,18 +1,16 @@ """The documentation against the code it describes. -A flag or a stage added to the code is documented in the same change, or one of -these fails and names the one that is not. +An option added to the command line is documented in the same change, or one of +these fails and names the option that is not. """ import argparse -import ast from pathlib import Path from in2lambda_agent.cli import build_parser ROOT = Path(__file__).resolve().parent.parent README = (ROOT / "README.md").read_text(encoding="utf-8") -HOW_IT_WORKS = (ROOT / "docs" / "how-it-works.md").read_text(encoding="utf-8") WORKFLOW = (ROOT / ".github" / "workflows" / "gate.yml").read_text(encoding="utf-8") @@ -33,62 +31,17 @@ def _options(parser: argparse.ArgumentParser) -> set[str]: return found - {"--help"} -def _stage_names() -> set[str]: - """Every name a stage is reported under. - - A run records a stage in one of two ways: `RESULT.add_stage(name, message)`, - or, in the spec loop, the `on_stage(name, message)` callback the loop is - given and `pipeline.run` answers with `add_stage`. So this reads both files - and both forms: an `add_stage` attribute call, and a call of the bare name - `on_stage`, which is the parameter rather than `RunResult.on_stage`. - - A stage name that is not a string constant is one this test cannot read, so - it fails naming the file and the line. - """ - names = set() - for module in ("pipeline.py", "spec.py"): - path = ROOT / "in2lambda_agent" / module - for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): - if not isinstance(node, ast.Call) or not node.args: - continue - records = ( - isinstance(node.func, ast.Attribute) and node.func.attr == "add_stage" - ) or (isinstance(node.func, ast.Name) and node.func.id == "on_stage") - if not records: - continue - assert isinstance(node.args[0], ast.Constant), ( - f"{module} line {node.lineno}: the stage name is not a string, " - "so this test cannot tell which stage it is" - ) - names.add(node.args[0].value) - return names - - def test_readme_names_every_option(): missing = [one for one in sorted(_options(build_parser())) if one not in README] assert not missing -def test_how_it_works_names_every_stage(): - names = _stage_names() - assert len(names) == 12 - # The stage's own section, rather than the name anywhere on the page: a - # column of the corpus table shares a name with a stage. - missing = [one for one in sorted(names) if f"### `{one}`" not in HOW_IT_WORKS] - assert not missing - - -def test_the_readme_compiles_the_ci_corpus_pdf_as_the_workflow_does(): - # The repository does not hold ci-corpus/pdf, so the reader compiles the - # PDF before the gate reads it. The README and the workflow give the same - # command, because another command writes other bytes, and the PDF's bytes - # are the key the OCR cache reads under. - for line in ( - "SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1", - "xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex", - ): - assert line in WORKFLOW - assert line in README +def test_the_readme_runs_the_gate_as_the_workflow_does(): + # The gate is a check on a branch, so the command a reader runs and the + # command CI runs must be the one command. + command = "in2lambda-agent gate ci-corpus/targets --filters ci-corpus/filters" + assert command in WORKFLOW + assert command in README def test_no_github_check_is_required_on_main(): @@ -100,23 +53,9 @@ def test_no_github_check_is_required_on_main(): assert "branch-protection" not in text -def test_the_readme_names_the_private_baseline_by_an_absolute_path(): - # The workbench check runs in a worktree, and the worktree holds neither - # the baseline for ExampleContents nor the specs it names. - check = next( - line - for line in README.splitlines() - if "in2lambda-agent gate " in line and "gate-baseline.json" in line - ) - path = check.split("in2lambda-agent gate ", 1)[1].split()[0] - assert path.startswith("/") - assert path.endswith("/corpus-specs/gate-baseline.json") - - def test_the_workflow_installs_a_pandoc_of_its_own(): - # ubuntu-24.04 packages pandoc 3.1.3, under which every ci-corpus document - # faults on a block no selector reaches. The job installs a pinned - # release, the one the baselines were recorded under. + # ubuntu-24.04 packages pandoc 3.1.3, under which the tests fail. The job + # installs a pinned release, the one the committed reply was written under. apt = WORKFLOW.split("apt-get install", 1)[1].split("\n\n", 1)[0] assert "pandoc" not in apt assert "https://github.com/jgm/pandoc/releases/download/" in WORKFLOW diff --git a/tests/test_fix.py b/tests/test_fix.py deleted file mode 100644 index b7d6adb..0000000 --- a/tests/test_fix.py +++ /dev/null @@ -1,474 +0,0 @@ -"""The tools a fixing round has, and what running one leaves in the draft's log.""" - -import json -import shutil -from pathlib import Path - -import pytest -from conftest import FakeBackend - -from in2lambda_agent import fix, package -from in2lambda_agent.model import ToolCall - -FIXTURES = Path(__file__).parent / "fixtures" - - -@pytest.fixture -def draft(tmp_path): - """A draft of the faulty sheet, with the spec's layer 1 fields in it. - - The same starting point a fixing round is given: q1 written with both its - parts and their solutions, and two blocks of the source in no field at all. - """ - folder = tmp_path / "sheets" - folder.mkdir() - for name in ("faulty.md", "faulty-spec.yaml"): - shutil.copy(FIXTURES / name, folder / name) - written = package.source_add(folder / "faulty.md") - package.spec_run(written, folder / "faulty-spec.yaml") - return written - - -def run(draft, name, arguments): - """Calls one tool the way a backend's loop calls it, and returns the result.""" - tool = next(one for one in fix.tools(draft) if one.name == name) - return tool.run(arguments) - - -def commands(draft): - """The name of every command the draft records having been built by.""" - return [entry["command"] for entry in package.command_log(draft)] - - -def test_the_tools_are_the_packages_draft_commands(draft): - # Underscored, which is the only shape a tool name may have on the wire. - assert [one.name for one in fix.tools(draft)] == [ - "mark_ignore", - "question_add", - "part_add", - "question_solution", - "field_replace", - "split_block", - "field_set", - "part_solution", - ] - - -@pytest.mark.parametrize( - "tool, arguments, wrote", - [ - ("mark_ignore", {"block": "b11"}, "b11.ignore"), - ("question_add", {"text": "b11"}, "q2.text"), - ("part_add", {"question": "q1", "text": "b11"}, "q1.p3.text"), - ("question_solution", {"question": "q1", "text": "s22"}, "q1.solution"), - ( - "field_replace", - {"field": "q1.text", "old": "ball", "new": "stone"}, - "q1.text", - ), - ("split_block", {"block": "b7", "at": 14}, "b7a and b7b"), - ("field_set", {"field": "q1.text", "text": "s22"}, "q1.text"), - ], -) -def test_each_tool_runs_its_command_and_in2lambda_logs_it( - draft, tool, arguments, wrote -): - result = run(draft, tool, arguments) - - assert result == f"{tool.replace('_', ' ')} wrote {wrote}" - # The log is in2lambda's, written as it applied the command: the agent keeps - # no record of its own, and this is the one the fix has to be visible in. - assert package.command_log(draft)[-1] == { - "command": tool.replace("_", " "), - "args": arguments, - "by": package.BY, - } - - -def test_a_field_set_writes_a_written_field_from_the_lines_it_names(draft): - # The finding it answers: a field that is empty, or that took the wrong - # lines. What the field held before is dropped, so the lines it came from - # are in no field and the next round is told about them. - before = json.loads(draft.read_text())["fields"]["q1.text"]["ranges"] - - result = run(draft, "field_set", {"field": "q1.text", "text": "s22"}) - written = json.loads(draft.read_text())["fields"]["q1.text"] - - assert result == "field set wrote q1.text" - assert before == [[5, 5]] - assert (written["ranges"], written["layer"], written["edited"]) == ( - [[22, 22]], - 3, - False, - ) - - -def test_a_part_solution_answers_one_part_rather_than_the_question(draft): - # A sheet that writes a solution under each part: the part is added from - # the lines holding it, and then answered on its own. - run(draft, "part_add", {"question": "q1", "text": "b7"}) - - result = run(draft, "part_solution", {"part": "q1.p3", "text": "s22"}) - written = json.loads(draft.read_text())["fields"]["q1.p3.solution"] - - assert result == "part solution wrote q1.p3.solution" - assert (written["ranges"], written["layer"], written["edited"]) == ( - [[22, 22]], - 3, - False, - ) - assert package.command_log(draft)[-1] == { - "command": "part solution", - "args": {"part": "q1.p3", "text": "s22"}, - "by": package.BY, - } - - -def test_a_part_solution_typed_out_is_layer_4(draft): - run(draft, "part_add", {"question": "q1", "text": "b7"}) - - result = run(draft, "part_solution", {"part": "q1.p3", "literal": "Term by term."}) - written = json.loads(draft.read_text())["fields"]["q1.p3.solution"] - - assert result == "part solution wrote q1.p3.solution" - assert (written["layer"], written["edited"]) == (4, True) - - -def test_a_part_solution_longer_than_a_repair_is_refused_before_it_is_written(draft): - typed = "x" * (fix.LITERAL_MAX + 1) - run(draft, "part_add", {"question": "q1", "text": "b7"}) - - result = run(draft, "part_solution", {"part": "q1.p3", "literal": typed}) - - assert result.startswith("part solution was refused: ") - assert f"literal is {len(typed)} characters" in result - assert commands(draft) == ["spec run", "part add"] - - -def test_the_system_prompt_names_the_two_commands_that_write_a_written_field(): - assert "part_solution one part's own worked solution" in fix.SYSTEM - assert "field_set quotes other lines into a field" in fix.SYSTEM - assert "It has no literal" in fix.SYSTEM - - -@pytest.mark.parametrize( - "arguments, layer, edited", - [ - ({"text": "b11"}, 3, False), - ({"literal": "Show that the field is solenoidal."}, 4, True), - ], -) -def test_copying_is_layer_3_and_typing_is_layer_4(draft, arguments, layer, edited): - run(draft, "question_add", arguments) - fields = json.loads(draft.read_text())["fields"] - - # in2lambda decides the layer from which argument it was given, so a field's - # provenance follows from copying or typing rather than from anything said. - assert (fields["q2.text"]["layer"], fields["q2.text"]["edited"]) == (layer, edited) - - -def test_the_layers_are_counted_off_the_draft(draft): - # The spec's own fields, and then one quoted out of the source and one typed. - spec_fields = package.layers(draft) - run(draft, "question_add", {"text": "b11"}) - run(draft, "part_add", {"question": "q2", "literal": "Typed out."}) - - assert spec_fields == { - "layer1": 5, - "layer2": 0, - "layer3": 0, - "layer4": 0, - "edited": 0, - } - assert package.layers(draft) == { - "layer1": 5, - "layer2": 0, - "layer3": 1, - "layer4": 1, - "edited": 1, - } - - -def test_a_literal_the_length_of_a_repair_is_written(draft): - typed = "x" * fix.LITERAL_MAX - - result = run(draft, "question_solution", {"question": "q1", "literal": typed}) - - assert result == "question solution wrote q1.solution" - assert commands(draft) == ["spec run", "question solution"] - - -def test_a_literal_longer_than_a_repair_is_refused_before_it_is_written(draft): - typed = "x" * (fix.LITERAL_MAX + 1) - - result = run(draft, "question_solution", {"question": "q1", "literal": typed}) - - assert result.startswith("question solution was refused: ") - assert f"literal is {len(typed)} characters" in result - assert f"at most {fix.LITERAL_MAX} may be typed" in result - # Refused by the agent rather than by in2lambda, so the draft never sees it: - # what the source does not hold is a finding to report, not a field to fill. - assert commands(draft) == ["spec run"] - - -@pytest.mark.parametrize( - "old, regex", - [ - ("", False), - ("WHOLE", False), - (".*", True), - ("^.*$", True), - ], -) -def test_a_field_replace_that_writes_the_whole_field_is_refused(draft, old, regex): - # The one command whose typed argument could become the field's whole text: - # `old` matching nothing or everything makes `new` the field, which is the - # model writing what the document does not say. - value = package.field_value(draft, "q1.text") - arguments = { - "field": "q1.text", - "old": value if old == "WHOLE" else old, - "new": "A ship's resistance force.", - } - if regex: - arguments["regex"] = True - - result = run(draft, "field_replace", arguments) - - assert result.startswith("field replace was refused: q1.text ") - assert "does not write a field" in result - assert commands(draft) == ["spec run"] - - -def test_an_empty_field_is_not_written_by_a_regex_that_matches_it(draft): - # The refusal the ticket is about: `q4.text (lines 32-33) is empty` answered - # with `--old '^$' --regex`, which makes `new` the field's whole text. An - # empty field is repaired by quoting the source range into it instead. - run(draft, "question_add", {"literal": ""}) - - result = run( - draft, - "field_replace", - {"field": "q2.text", "old": "^$", "new": "A ship's hull.", "regex": True}, - ) - - assert result.startswith("field replace was refused: q2.text ") - assert "does not write a field" in result - assert commands(draft) == ["spec run", "question add"] - - -def test_a_field_replace_repairing_wording_is_still_written(draft): - result = run( - draft, "field_replace", {"field": "q1.text", "old": "ball", "new": "stone"} - ) - - assert result == "field replace wrote q1.text" - assert commands(draft) == ["spec run", "field replace"] - - -def test_a_replacement_longer_than_a_repair_is_refused_before_it_is_written(draft): - typed = "x" * (fix.LITERAL_MAX + 1) - - result = run( - draft, "field_replace", {"field": "q1.text", "old": "ball", "new": typed} - ) - - assert result.startswith("field replace was refused: ") - assert f"new is {len(typed)} characters" in result - assert f"at most {fix.LITERAL_MAX} may be typed" in result - assert commands(draft) == ["spec run"] - - -def test_a_replacement_the_length_of_a_repair_is_written(draft): - typed = "x" * fix.LITERAL_MAX - - result = run( - draft, "field_replace", {"field": "q1.text", "old": "ball", "new": typed} - ) - - assert result == "field replace wrote q1.text" - assert commands(draft) == ["spec run", "field replace"] - - -def test_a_field_replace_naming_no_field_of_the_draft_is_left_to_in2lambda(draft): - result = run( - draft, "field_replace", {"field": "q9.text", "old": "", "new": "Anything."} - ) - - assert result.startswith("field replace was refused: ") - assert "does not write a field" not in result - assert commands(draft) == ["spec run"] - - -def test_the_replacement_is_capped_as_a_literal_is(draft): - tool = next(one for one in fix.tools(draft) if one.name == "field_replace") - - assert tool.parameters["properties"]["new"]["maxLength"] == fix.LITERAL_MAX - assert "not be empty or the whole of the field" in tool.description - assert f"at most {fix.LITERAL_MAX} characters" in tool.description - - -def test_the_system_prompt_says_a_field_replace_repairs_rather_than_writes(): - assert "it may not\n be empty or the whole of the field" in ( - fix.SYSTEM - ) - assert "does not write a field" in fix.SYSTEM - assert f"at most {fix.LITERAL_MAX} characters, as a literal is" in fix.SYSTEM - - -@pytest.mark.parametrize( - "arguments, named", - [ - ({"field": "q1.text", "old": "", "new": "Typed out."}, ["q1.text"]), - ({"field": "q1.text", "old": "ball", "new": "stone"}, []), - ({"field": "q9.text", "old": "ball", "new": "stone"}, []), - ], -) -def test_unrepaired_names_the_field_a_write_was_refused_over(draft, arguments, named): - # What the pipeline reads to tell a finding the loop cannot repair from one - # it answered, or from a command in2lambda refused for its own reasons. - result = run(draft, "field_replace", arguments) - - assert fix.unrepaired([ToolCall("field_replace", arguments, result)]) == named - - -def test_unrepaired_says_nothing_of_a_refusal_that_is_not_a_write(draft): - result = run(draft, "mark_ignore", {"block": "b99"}) - - assert fix.unrepaired([ToolCall("mark_ignore", {"block": "b99"}, result)]) == [] - - -def test_a_literal_that_is_not_text_at_all_is_left_to_in2lambda(draft): - # What a backend that fills every parameter in sends: the range it means - # beside a null for the one it does not. The cap measures a string or - # nothing, so this is a copy like any other rather than the round ending on - # a length it cannot take. - result = run(draft, "question_add", {"text": "b11", "literal": None}) - fields = json.loads(draft.read_text())["fields"] - - assert result == "question add wrote q2.text" - assert (fields["q2.text"]["layer"], fields["q2.text"]["edited"]) == (3, False) - - -def test_a_literal_of_the_wrong_shape_is_refused_by_in2lambda(draft): - result = run(draft, "question_solution", {"question": "q1", "literal": 12}) - - assert result.startswith("question solution was refused: ") - assert "rather than a name" in result - assert commands(draft) == ["spec run"] - - -def test_every_tool_that_types_says_how_little_it_may_type(draft): - typing = [ - one for one in fix.tools(draft) if "literal" in one.parameters["properties"] - ] - - # `field_set` is not among them: in2lambda takes no literal for it, so a - # field it writes says what the source says and nothing else. - assert [one.name for one in typing] == [ - "question_add", - "part_add", - "question_solution", - "part_solution", - ] - assert all( - one.parameters["properties"]["literal"]["maxLength"] == fix.LITERAL_MAX - for one in typing - ) - - -def test_the_system_prompt_says_what_is_left_alone_rather_than_written(): - # The cap stops a long invention; this is what stops a short one, and what - # tells a round that a finding it cannot answer is an answer in itself. - assert "A finding no range of the source can answer is left as it is." in fix.SYSTEM - assert "never for writing a solution" in fix.SYSTEM - assert f"at most {fix.LITERAL_MAX} characters" in fix.SYSTEM - - -def test_a_refusal_comes_back_as_the_tools_result(draft): - result = run(draft, "mark_ignore", {"block": "b99"}) - - assert result.startswith("mark ignore was refused: ") - assert "no block b99" in result - # And nothing is recorded as having happened: a command in2lambda refused is - # not a command the draft was built by. - assert commands(draft) == ["spec run"] - - -def test_lines_another_field_has_taken_are_refused_rather_than_written_twice(draft): - result = run(draft, "question_add", {"text": "b3"}) - - assert "was refused" in result and "q1.text" in result - assert commands(draft) == ["spec run"] - - -def test_the_prompt_carries_the_source_and_every_finding(draft): - backend = FakeBackend([]) - shown = package.source_show(draft) - report = package.validate(draft) - - fix.fix_round(draft, shown, report, backend) - system, prompt = backend.calls[0] - - assert system == fix.SYSTEM - assert shown in prompt - for finding in report.findings: - # One line of the prompt each: the level, the check, the field and the - # lines it is about, and then the sentence. Named before the sentence - # rather than only inside it, so that what a round is given does not - # depend on how whichever check found it happens to word itself. - (line,) = [one for one in prompt.splitlines() if finding.message in one] - assert line.startswith( - f"- {finding.level} {finding.check} {finding.field} " - ) - assert all(f"{start}-{end}" in line for start, end in finding.ranges) - - -def test_a_round_runs_the_commands_the_model_asks_for(draft): - backend = FakeBackend( - [ - ("split_block", {"block": "b7", "at": 14}), - ("question_add", {"text": "b7a"}), - ] - ) - - reply = fix.fix_round( - draft, package.source_show(draft), package.validate(draft), backend - ) - - assert [call.name for call in reply.calls] == ["split_block", "question_add"] - assert reply.calls[1].result == "question add wrote q2.text" - assert commands(draft) == ["spec run", "split block", "question add"] - - -def test_the_halves_of_a_split_block_are_shown_to_the_next_round(draft): - run(draft, "split_block", {"block": "b7", "at": 14}) - shown = package.source_show(draft) - - # The source is read again each round, so that a block the last round cut in - # two is in front of the model under the ids it can quote. - assert "b7a" in shown and "b7b" in shown and "b7 " not in shown - - -@pytest.mark.parametrize( - "calls, expected", - [ - ([], "no commands"), - ([("mark_ignore", {"block": "b9"})], "1 command (mark ignore b9)"), - ( - [ - ("mark_ignore", {"block": "b9"}), - ("question_solution", {"question": "q2", "text": "b11"}), - ], - "2 commands (mark ignore b9, question solution q2)", - ), - ( - [("part_solution", {"part": "q1.p3", "text": "s22"})], - "1 command (part solution q1.p3)", - ), - ], -) -def test_the_summary_names_what_each_command_was_about(calls, expected): - made = [ToolCall(name, arguments, "") for name, arguments in calls] - - assert fix.summary(made) == expected diff --git a/tests/test_gate.py b/tests/test_gate.py index 28e2b80..2568a6f 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -1,399 +1,121 @@ -"""The merge gate: a replay over a corpus, checked against a recorded baseline.""" +"""The merge gate: every target replayed against its export, and no call. + +The targets themselves are tested in test_targets; what is tested here is that +the gate replays them — a target with nothing saved is an error rather than a +call — and that the one target the repository commits comes back clean under a +backend that refuses to answer. +""" import json from pathlib import Path import pytest -from test_corpus import make_set -from test_pipeline import SPEC, TEX_SPEC - -from in2lambda_agent import corpus, gate, pair -from in2lambda_agent.gate import Baseline, Folder -from in2lambda_agent.settings import Settings -from in2lambda_agent.spec import SPEC_NAME - -@pytest.fixture -def corpus_root(tmp_path): - """A corpus of two folders, with a saved spec each so a replay builds.""" - made = tmp_path / "corpus" - make_set(made, "sheets", ["sheet.md", "sheet-2.md"]) - make_set(made, "tex", ["tex-sheet.tex", "tex-sheet-2.tex"]) - for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): - saved = tmp_path / "specs" / folder / SPEC_NAME - saved.parent.mkdir(parents=True) - saved.write_text(text) - return made +from conftest import FakeBackend +from test_targets import REPLY, fake_convert, make_target +from in2lambda_agent import gate, targets -@pytest.fixture -def baseline(corpus_root, tmp_path): - """The baseline as it is written by hand, before anything is recorded.""" - return Baseline( - specs=tmp_path / "specs", - folders={ - "sheets": Folder(root=corpus_root, suffixes=["md"]), - "tex": Folder(root=corpus_root, suffixes=["tex"]), - }, - ) +CI_TARGETS = Path(__file__).parent.parent / "ci-corpus" / "targets" +CI_FILTERS = Path(__file__).parent.parent / "ci-corpus" / "filters" -def run(baseline, tmp_path, **kwargs): - """A gate run with its cache and work directory under tmp_path.""" - return gate.run( - baseline, - cache=tmp_path / "cache", - work=tmp_path / "gate", - settings=Settings(), - **kwargs, - ) +def save(filters, name, *, reply=True, lua=True): + """What a target must have saved for the gate to replay it.""" + saved = Path(filters) / name + saved.mkdir(parents=True, exist_ok=True) + if reply: + (saved / targets.REPLY_NAME).write_text(json.dumps(REPLY)) + if lua: + (saved / targets.FILTER_NAME).write_text("-- filter") + return saved -def test_recording_fills_the_counts_and_leaves_what_was_written_by_hand( - baseline, corpus_root, tmp_path +def test_a_saved_target_is_replayed_and_its_new_differences_come_back( + tmp_path, monkeypatch ): - report = run(baseline, tmp_path, record=True) - - assert not report.failed - assert baseline.folders["sheets"].built == 2 - assert baseline.folders["tex"].built == 2 - assert baseline.folders["tex"].documents == { - "tex/tex-sheet.tex": "built", - "tex/tex-sheet-2.tex": "built", - } - # The hand-written half is the run's to read, not to write over. - assert baseline.folders["tex"].root == corpus_root - assert baseline.folders["tex"].suffixes == ["tex"] - - -def test_a_recorded_baseline_passes_the_run_that_recorded_it(baseline, tmp_path): - run(baseline, tmp_path, record=True) - - report = run(baseline, tmp_path) - - assert not report.failed - assert [one.built for one in report.folders.values()] == [2, 2] - - -def test_the_gate_replays_the_logs_kept_beside_the_specs(baseline, tmp_path): - # A log in2lambda refuses, so that the outcome says whether the gate ran it. - saved = tmp_path / "specs" / "tex" / f"tex-sheet.tex{corpus.COMMANDS_SUFFIX}" - saved.write_text( - json.dumps([{"command": "mark ignore", "args": {"block": "b99"}, "by": "ada"}]) + calls = fake_convert(monkeypatch) + make_target(tmp_path / "corpus", "ME2") + save(tmp_path / "filters", "ME2") + + results = gate.run( + tmp_path / "corpus", + filters=tmp_path / "filters", + work=tmp_path / "work", + cache=tmp_path / "cache", + backend=FakeBackend(), ) - run(baseline, tmp_path, record=True) - - # The gate reads the specs from its own copy of the tree, and the logs are - # copied with them. - copied = tmp_path / "gate" / "specs" / "tex" / saved.name - assert copied.is_file() - assert baseline.folders["tex"].documents["tex/tex-sheet.tex"] == "replay refused" - assert baseline.folders["tex"].documents["tex/tex-sheet-2.tex"] == "built" - - -def test_a_folder_that_builds_fewer_than_recorded_fails(baseline, tmp_path): - run(baseline, tmp_path, record=True) - baseline.folders["tex"].built += 1 - - report = run(baseline, tmp_path) - - assert report.failed - assert report.folders["tex"].failed and not report.folders["sheets"].failed - - -def test_a_folder_whose_documents_no_longer_build_fails(baseline, tmp_path): - run(baseline, tmp_path, record=True) - # The spec a replay has nothing to replay without. - (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() - - report = run(baseline, tmp_path) - - assert report.failed - summary = report.folders["tex"] - assert (summary.built, summary.recorded) == (0, 2) - assert summary.counts == {"no spec": 2} - assert sorted(one.document for one in summary.regressions) == [ - "tex/tex-sheet-2.tex", - "tex/tex-sheet.tex", - ] - + assert [one.name for one in results] == ["ME2"] + assert results[0].error is None + assert results[0].new == results[0].differences and results[0].new + # The saved reply was converted rather than asked for again. + assert calls[0]["route_a"] == REPLY -def test_a_folder_the_baseline_records_no_count_for_passes(baseline, tmp_path): - run(baseline, tmp_path, record=True) - # A folder added to the gate before it replays to a build worth defending: - # `--record` writes both halves, so neither is there yet. - baseline.folders["tex"].built = None - baseline.folders["tex"].documents = {} - (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() - report = run(baseline, tmp_path) - - assert not report.failed - assert report.folders["tex"].built == 0 - - -def test_a_document_the_baseline_does_not_know_is_not_a_failure( - baseline, corpus_root, tmp_path +def test_a_target_with_nothing_saved_is_an_error_and_makes_no_call( + tmp_path, monkeypatch ): - run(baseline, tmp_path, record=True) - (corpus_root / "sheets" / "sheet-3.md").write_text( - (corpus_root / "sheets" / "sheet.md").read_text() - ) + calls = fake_convert(monkeypatch) + make_target(tmp_path / "corpus", "ME2") - report = run(baseline, tmp_path) + results = gate.run( + tmp_path / "corpus", + filters=tmp_path / "filters", + work=tmp_path / "work", + cache=tmp_path / "cache", + backend=FakeBackend(), + ) - assert not report.failed - assert report.folders["sheets"].built == 3 + assert targets.REPLY_NAME in results[0].error + assert calls == [] -def test_a_document_the_corpus_no_longer_holds_is_a_regression( - baseline, corpus_root, tmp_path +def test_the_run_writes_nothing_under_the_directory_it_was_run_in( + tmp_path, monkeypatch ): - run(baseline, tmp_path, record=True) - (corpus_root / "tex" / "tex-sheet-2.tex").unlink() - - report = run(baseline, tmp_path) - - assert report.failed - regression = report.folders["tex"].regressions[0] - assert (regression.document, regression.current) == ( - "tex/tex-sheet-2.tex", - gate.MISSING, + fake_convert(monkeypatch) + make_target(tmp_path / "corpus", "ME2") + save(tmp_path / "filters", "ME2") + here = tmp_path / "here" + here.mkdir() + monkeypatch.chdir(here) + + gate.run( + tmp_path / "corpus", + filters=tmp_path / "filters", + work=tmp_path / "work", + cache=tmp_path / "cache", + backend=FakeBackend(), ) + assert list(here.iterdir()) == [] + assert (tmp_path / "work" / "ME2").is_dir() -def test_a_baseline_of_no_builds_still_notices_a_document_that_did_worse( - baseline, tmp_path -): - # The check that gives a corpus where nothing builds teeth: the built count - # is 0 on both sides, so only the document's own outcome says anything. - baseline.folders["tex"].built = 0 - baseline.folders["tex"].documents = {"tex/tex-sheet.tex": "faulted"} - (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() - - report = run(baseline, tmp_path) - summary = report.folders["tex"] - assert summary.built == summary.recorded == 0 - assert report.failed - assert [one.document for one in summary.regressions] == ["tex/tex-sheet.tex"] +def test_the_cache_is_shared_between_worktrees(): + # Outside every worktree, so that a PDF converted on one branch is not + # converted again on the next. + assert gate.DEFAULT_CACHE_DIR == Path.home() / ".cache" / "in2lambda-agent" -@pytest.mark.parametrize( - ("current", "recorded", "expected"), - [ - ("built", "built", False), - ("built", "faulted", False), - ("skipped", "built", True), - # A document read and faulted, then not read at all. - ("skipped", "faulted", True), - ("build refused", "built", True), - ("faulted", "built", True), - ("faulted", "build refused", True), - ("no spec", "faulted", True), - (gate.MISSING, "built", True), - (gate.MISSING, "no spec", False), - ], +@pytest.mark.skipif( + not (CI_TARGETS / "sheet" / "set_Sheet").is_dir(), reason="the CI target" ) -def test_what_counts_as_worse(current, recorded, expected): - assert gate.worse(current, recorded) is expected - - -def test_the_folder_line_says_what_each_outcome_came_to(baseline, tmp_path): - run(baseline, tmp_path, record=True) - summary = run(baseline, tmp_path).folders["sheets"] - - line = gate.folder_line("sheets", summary) - - assert line.split() == [ - "sheets", - "built", - "2", - "faulted", - "0", - "build", - "refused", - "0", - "skipped", - "0", - "(baseline", - "built", - "2)", - ] - - -def test_the_folder_line_names_an_outcome_of_its_own(baseline, tmp_path): - summary = gate.Summary(built=1, counts={"no spec": 2}, recorded=3) - - line = gate.folder_line("tex", summary) - - assert "no spec 2" in line and "(baseline built 3)" in line - - -def test_the_folder_line_names_each_document_that_did_worse(): - summary = gate.Summary( - built=0, - counts={"faulted": 1}, - recorded=0, - regressions=[ - gate.Regression("tex/sheet-3.tex", "built", "faulted", "KaTeX"), - gate.Regression("tex/sheet-4.tex", "built", gate.MISSING), - ], - ) - - first, second, third = gate.folder_line("tex", summary).splitlines() - - assert first.startswith("tex") - assert second == " worse tex/sheet-3.tex built -> faulted: KaTeX" - assert third == " worse tex/sheet-4.tex built -> missing" - - -def test_the_folder_line_says_where_no_count_is_recorded(baseline, tmp_path): - line = gate.folder_line("tex", gate.Summary(built=1)) - - assert line.endswith("(not recorded)") - - -def test_the_baseline_survives_being_written_and_read(baseline, tmp_path): - run(baseline, tmp_path, record=True) - baseline.folders["tex"].built = None - path = tmp_path / "baseline.json" - - gate.write_baseline(baseline, path) - read = gate.read_baseline(path) - - assert (read.specs, read.folders) == (baseline.specs, baseline.folders) - # The directory the paths are read from is the file's own and is not a - # field of the file. - assert read.directory == tmp_path - - -@pytest.fixture -def beside(tmp_path): - """A baseline naming its specs and its corpus relative to its own directory.""" - made = tmp_path / "beside" - (made / "sheets").mkdir(parents=True) - (made / "sheets" / SPEC_NAME).write_text(SPEC) - make_set(made / "corpus", "sheets", ["sheet.md"]) - (made / "baseline.json").write_text( - json.dumps( - { - "specs": ".", - "folders": {"sheets": {"root": "corpus", "suffixes": ["md"]}}, - } - ) +def test_the_committed_target_replays_clean_with_no_backend_to_call(tmp_path): + # What CI runs, and the whole of it: the sheet, the filter and the reply the + # repository commits, compared with the set in2lambda's own writer wrote + # from that reply. The backend raises if anything asks it to answer, so a + # run that needs a model call fails here rather than in a job with no key. + results = gate.run( + CI_TARGETS, + filters=CI_FILTERS, + work=tmp_path / "work", + cache=tmp_path / "cache", + backend=FakeBackend(AssertionError("the gate made a model call")), ) - return made / "baseline.json" - - -def test_the_specs_and_a_relative_root_are_read_beside_the_baseline( - beside, tmp_path, monkeypatch -): - # The private baseline sits beside the specs it names, outside the - # repository, and the workbench check runs the gate in a worktree. Reading - # the file's paths from its own directory gives the same run from any - # directory. - elsewhere = tmp_path / "elsewhere" - elsewhere.mkdir() - monkeypatch.chdir(elsewhere) - - report = run(gate.read_baseline(beside), tmp_path) - - assert report.folders["sheets"].built == 1 - - -def test_recording_writes_the_paths_as_they_were_written(beside, tmp_path): - read = gate.read_baseline(beside) - - run(read, tmp_path, record=True) - gate.write_baseline(read, beside) - - written = json.loads(beside.read_text()) - assert written["specs"] == "." - assert written["folders"]["sheets"]["root"] == "corpus" - assert written["folders"]["sheets"]["built"] == 1 - - -def test_a_run_writes_nothing_under_the_directory_it_was_run_from( - baseline, tmp_path, monkeypatch -): - ran_from = tmp_path / "empty" - ran_from.mkdir() - monkeypatch.chdir(ran_from) - - run(baseline, tmp_path) - - assert list(ran_from.iterdir()) == [] - - -def test_a_run_leaves_the_committed_spec_tree_as_it_found_it(baseline, tmp_path): - # A sweep writes a record of each run beside the spec it read. The specs - # are the repository's, so the gate reads a copy and the records land there. - specs = tmp_path / "specs" - before = sorted(path.relative_to(specs) for path in specs.rglob("*")) - - run(baseline, tmp_path) - - assert sorted(path.relative_to(specs) for path in specs.rglob("*")) == before - - -def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): - # Shared between worktrees on purpose: OCR already fetched for a PDF is not - # fetched again on the next branch. - assert gate.DEFAULT_CACHE_DIR == Path.home() / ".cache" / "in2lambda-agent" - - -REPOSITORY = Path(__file__).resolve().parent.parent - - -def test_the_committed_baseline_names_its_folders_and_their_specs(): - baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") - - assert set(baseline.folders) == {"tex", "docx", "pdf"} - for name, folder in baseline.folders.items(): - # The spec each folder replays, at the path the sweep reads it from. - assert (REPOSITORY / baseline.specs / name / SPEC_NAME).is_file() - # Recorded, however low: a count of none is what the gate defends. - assert folder.built is not None - assert folder.documents - - -def test_the_committed_baseline_names_no_path_outside_the_repository(): - # An absolute root is a path on one machine, and under ExampleContents it - # is also the name of a folder of private documents. - baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") - - assert not baseline.specs.is_absolute() - assert all(not folder.root.is_absolute() for folder in baseline.folders.values()) - - -def test_the_private_corpus_keeps_its_specs_and_its_baseline_out_of_the_repository(): - # The specs for ExampleContents quote the headings of private documents, - # and the baseline beside them records those documents' file names and the - # path of the corpus on one machine. The workbench check names that - # baseline by its absolute path instead of reading it from the worktree. - assert "corpus-specs/" in (REPOSITORY / ".gitignore").read_text() - assert not (REPOSITORY / "gate-baseline.json").exists() - - -def test_the_ci_corpus_pairs_a_solutions_document_with_its_questions(): - # The corpus exists to exercise the separate-solutions document, which it - # does only when `pair` matches the file's name. Name it so that it does - # not — solutions-2.tex rather than sheet-2-solutions.tex — and the sweep - # reads it as a sheet of its own and the path is never run. - assert pair.solutions_beside(REPOSITORY / "ci-corpus/tex/sheet-2.tex") is not None - - -def test_no_document_of_the_ci_corpus_is_a_solutions_file(): - # A solutions document is frozen as the second source of the questions - # document beside it, so a sweep gives it no row of its own. - baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") - named = [ - document - for folder in baseline.folders.values() - for document in folder.documents - if pair.questions_stem(Path(document)) is not None - ] - assert named == [] + assert [one.name for one in results] == ["sheet"] + assert results[0].error is None + assert results[0].new == [] + assert results[0].flags == 0 diff --git a/tests/test_model_live.py b/tests/test_model_live.py index 355d899..545dc61 100644 --- a/tests/test_model_live.py +++ b/tests/test_model_live.py @@ -6,21 +6,16 @@ """ import os -import shutil -from pathlib import Path import pytest -from in2lambda_agent import package from in2lambda_agent.model import ( AgentSDKBackend, AnthropicBackend, OpenRouterBackend, Tool, - choose_backend, ) from in2lambda_agent.settings import load_settings -from in2lambda_agent.spec import LAYOUTS, SPEC_NAME, write_spec ADD = Tool( name="add", @@ -67,26 +62,3 @@ def test_a_trivial_tool_call(backend): assert reply.usage.input_tokens > 0 assert reply.usage.output_tokens > 0 assert reply.usage.seconds > 0 - - -def test_one_call_writes_a_spec_in2lambda_will_run(tmp_path): - """The agent's own call: the numbered source in, a spec `spec run` takes out.""" - if os.environ.get("IN2LAMBDA_AGENT_LIVE") != "1": - pytest.skip("set IN2LAMBDA_AGENT_LIVE=1 to make real model calls") - - backend = choose_backend(load_settings()) - if reason := backend.unavailable(): - pytest.skip(reason) - - shutil.copy(Path(__file__).parent / "fixtures" / "sheet.md", tmp_path) - draft_dir = package.source_add(tmp_path / "sheet.md") - text, _ = write_spec(package.source_show(draft_dir), backend) - spec = tmp_path / SPEC_NAME - spec.write_text(text) - - coverage = package.spec_run(draft_dir, spec) - - assert coverage.layout in LAYOUTS - # At least the sheet's two questions: how much more it covered is what the - # corpus run records rather than what one call is held to. - assert coverage.fields.get(1, 0) >= 2 diff --git a/tests/test_package.py b/tests/test_package.py deleted file mode 100644 index 0bb56bb..0000000 --- a/tests/test_package.py +++ /dev/null @@ -1,320 +0,0 @@ -"""What the package module reads back out of a draft, and what it cannot yet.""" - -import json -import shutil -import warnings -from pathlib import Path - -import in2lambda.draft.export -import pytest -from in2lambda.source import ConversionToolsMissing -from in2lambda.validation.pdf import missing_tools - -from in2lambda_agent import package - -FIXTURES = Path(__file__).parent / "fixtures" - - -@pytest.fixture -def draft(tmp_path): - """A frozen sheet with its spec run over it: ten fields, all layer 1.""" - folder = tmp_path / "sheets" - folder.mkdir() - shutil.copy(FIXTURES / "sheet.md", folder / "sheet.md") - spec = FIXTURES / "sheet-spec.yaml" - written = package.source_add(folder / "sheet.md") - package.spec_run(written, spec) - return written - - -def test_the_questions_are_numbered_with_the_lines_they_came_from(draft): - found = package.questions(draft) - - assert list(found) == ["q1", "q2"] - # The question's own text, its two parts and their two solutions. - assert found["q1"].ranges == [[5, 5], [7, 7], [9, 9], [21, 21], [23, 23]] - assert found["q1"].layer == 1 - - -def test_a_field_written_past_the_spec_raises_its_questions_layer(draft): - package.command( - draft, - "field replace", - {"field": "q2.text", "old": "A block", "new": "A crate"}, - ) - - found = package.questions(draft) - - # Layer 4, the typed-out edit, is what sample review picks on; the question - # nothing touched is still the spec's. - assert found["q2"].layer == 4 - assert found["q1"].layer == 1 - - -def test_a_field_typed_out_has_no_lines_of_the_source_behind_it(draft): - package.command( - draft, "question add", {"literal": "Show that the field is solenoidal."} - ) - written = json.loads(draft.read_text())["fields"]["q3.text"] - - # What in2lambda records for a field no range of the source backs: the key - # is there and empty rather than absent. `questions` reads it with a - # default all the same, so a version that leaves it out reads the same way. - assert (written["layer"], written["edited"]) == (4, True) - assert written["ranges"] == [] - assert package.questions(draft)["q3"] == package.QuestionInfo("q3", 4, []) - - -def test_a_reviewers_edit_is_logged_under_their_name(draft): - package.command( - draft, - "field replace", - {"field": "q1.text", "old": "ball", "new": "stone"}, - by="ada", - ) - fields = json.loads(draft.read_text())["fields"] - - assert package.command_log(draft)[-1]["by"] == "ada" - assert fields["q1.text"]["edited"] is True - assert "stone" in fields["q1.text"]["value"] - - -def test_the_fix_log_is_the_commands_after_the_spec_run(draft): - package.command( - draft, "field replace", {"field": "q1.text", "old": "ball", "new": "stone"} - ) - - assert package.command_log(draft)[0]["command"] == "spec run" - assert [one["command"] for one in package.fix_log(draft)] == ["field replace"] - - -def test_a_saved_log_replays_in_the_order_it_was_saved(draft, tmp_path): - saved = [ - { - "command": "field replace", - "args": {"field": "q1.text", "old": "ball", "new": "stone"}, - "by": package.BY, - }, - { - "command": "field replace", - "args": {"field": "q1.text", "old": "stone", "new": "brick"}, - "by": package.BY, - }, - ] - again = package.source_add(package.frozen_source(draft)) - package.spec_run(again, FIXTURES / "sheet-spec.yaml") - - assert package.replay(again, saved) == 2 - assert "brick" in package.field_value(again, "q1.text") - - -def test_a_refused_command_names_its_place_in_the_log(draft): - saved = [ - { - "command": "field replace", - "args": {"field": "q1.text", "old": "ball", "new": "stone"}, - "by": package.BY, - }, - {"command": "mark ignore", "args": {"block": "b99"}, "by": package.BY}, - { - "command": "field replace", - "args": {"field": "q1.text", "old": "stone", "new": "brick"}, - "by": package.BY, - }, - ] - - with pytest.raises(package.CommandRefused, match="command 2 of 3, mark ignore"): - package.replay(draft, saved) - - # The command before the refusal was applied, and the one after it was not. - assert "stone" in package.field_value(draft, "q1.text") - - -def test_a_part_with_no_solution_is_a_warning_the_report_is_still_clean_for( - tmp_path, -): - # The same set's spec over a sheet whose solutions are not on it: nothing - # the checks find stops a build, so the report is clean with four warnings - # in it rather than four errors. - folder = tmp_path / "questions-only" - folder.mkdir() - shutil.copy(FIXTURES / "questions-only.md", folder / "questions-only.md") - written = package.source_add(folder / "questions-only.md") - package.spec_run(written, FIXTURES / "sheet-spec.yaml") - - report = package.validate(written) - - assert (report.clean, report.errors) == (True, []) - assert len(report.warnings) == 4 - assert all(one.level == "warning" for one in report.findings) - assert all(one.check == "no-solution" for one in report.findings) - assert report.warnings == [one.message for one in report.findings] - - -def beside_its_image(tmp_path, name: str) -> Path: - """A sheet copied into a folder of its own, beside the image it refers to.""" - folder = tmp_path / Path(name).stem - (folder / "figures").mkdir(parents=True) - shutil.copy(FIXTURES / name, folder / name) - shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") - return folder / name - - -def test_a_block_marked_ignore_whose_lines_hold_an_image_is_reported(tmp_path): - # The spec marks the figure's paragraph ignored by matching its caption. - # Nothing in2lambda checks says so, and the set built from the draft holds - # no image. - written = package.source_add(beside_its_image(tmp_path, "figure-paragraph.md")) - package.spec_run(written, FIXTURES / "figure-paragraph-spec.yaml") - - (dropped,) = package.ignored_images(written) - - assert (dropped.check, dropped.level) == ("coverage", package.ERROR) - assert (dropped.field, dropped.ranges) == ("b4", [[7, 8]]) - assert dropped.message == "b4 (lines 7-8) holds an image and is marked ignore." - - -def test_an_image_inside_a_question_is_nothing_to_report(tmp_path): - written = package.source_add(beside_its_image(tmp_path, "figure.md")) - package.spec_run(written, FIXTURES / "sheet-spec.yaml") - - assert package.ignored_images(written) == [] - - -def test_a_source_whose_bytes_are_not_text_has_no_ignored_image_to_read(tmp_path): - # A docx source is frozen as itself, so the file beside the draft is a zip. - # Reading it for a `![` is not what it is for, and must not end the run. - written = package.source_add(beside_its_image(tmp_path, "figure-paragraph.md")) - package.spec_run(written, FIXTURES / "figure-paragraph-spec.yaml") - package.frozen_source(written).write_bytes((FIXTURES / "ball.png").read_bytes()) - - assert package.ignored_images(written) == [] - - -def test_the_coverage_line_names_the_images_the_spec_dropped(tmp_path): - written = package.source_add(beside_its_image(tmp_path, "figure-paragraph.md")) - - coverage = package.spec_run(written, FIXTURES / "figure-paragraph-spec.yaml") - - assert str(coverage).endswith("; 1 image dropped: b4 (lines 7-8)") - - -def test_the_coverage_line_of_a_spec_that_dropped_none_is_unchanged(tmp_path): - written = package.source_add(beside_its_image(tmp_path, "figure.md")) - - coverage = package.spec_run(written, FIXTURES / "sheet-spec.yaml") - - assert str(coverage).endswith("4 ignored, none unassigned") - - -def test_a_second_source_is_frozen_into_the_same_draft(tmp_path): - folder = tmp_path / "sheets" - folder.mkdir() - for name in ("paired.md", "paired_solutions.md"): - shutil.copy(FIXTURES / name, folder / name) - - written = package.source_add(folder / "paired.md", folder / "paired_solutions.md") - - assert written == folder / "paired.draft.json" - frozen = json.loads(written.read_text())["sources"] - assert [one["source"] for one in frozen] == ["paired.md", "paired_solutions.md"] - # The second source's blocks carry its number, which is how a command and a - # spec-writing prompt address them. - assert all(one["id"].startswith("2/") for one in frozen[1]["blocks"]) - - -def test_the_warnings_a_build_says_are_returned_rather_than_printed(tmp_path): - # The same sheet without its solutions: in2lambda builds it and warns about - # each unanswered part as it goes. Those are the messages the validate line - # already lists, which is why the pipeline prints none of them again. - folder = tmp_path / "questions-only" - folder.mkdir() - shutil.copy(FIXTURES / "questions-only.md", folder / "questions-only.md") - written = package.source_add(folder / "questions-only.md") - package.spec_run(written, FIXTURES / "sheet-spec.yaml") - report = package.validate(written) - - with warnings.catch_warnings(record=True) as escaped: - warnings.simplefilter("always") - built = package.build(written, tmp_path / "out") - - assert escaped == [] - assert built.zip_path.is_file() - assert built.warnings == report.warnings - - -def test_the_frozen_source_is_named_from_the_draft(draft): - assert package.frozen_source(draft).name == "sheet.md" - assert package.frozen_source(draft).is_file() - - -def test_rendering_names_the_pdf_written_for_each_question( - draft, tmp_path, monkeypatch -): - out = tmp_path / "render" - # in2lambda numbers each file with the question's place in the set, counting - # from zero, which is what the key comes from: a question the compiler gave - # up on is left out of the list, so counting the list would number the rest - # wrongly. - monkeypatch.setattr( - in2lambda.draft.export, - "render", - lambda written, directory: [ - Path(directory) / "question_001_Question_2.pdf", - Path(directory) / "question_000_Question_1.pdf", - ], - ) - - assert package.render(draft, out) == { - "q1": out / "question_000_Question_1.pdf", - "q2": out / "question_001_Question_2.pdf", - } - - -def test_rendering_that_cannot_compile_is_refused_with_in2lambdas_reason( - draft, tmp_path, monkeypatch -): - def missing(written, directory): - raise ConversionToolsMissing("Rendering questions needs xelatex.") - - monkeypatch.setattr(in2lambda.draft.export, "render", missing) - - with pytest.raises(package.CommandRefused, match="needs xelatex"): - package.render(draft, tmp_path / "render") - - -@pytest.mark.skipif(bool(missing_tools()), reason="needs pandoc and xelatex") -def test_rendering_writes_a_pdf_for_each_question_of_the_draft(draft, tmp_path): - rendered = package.render(draft, tmp_path / "render") - - assert list(rendered) == ["q1", "q2"] - assert all(path.parent == tmp_path / "render" for path in rendered.values()) - assert all(path.is_file() for path in rendered.values()) - - -@pytest.mark.skipif(bool(missing_tools()), reason="needs pandoc and xelatex") -def test_a_question_the_compiler_gives_up_on_leaves_the_rest_keyed_as_they_were( - draft, tmp_path -): - # TeX stops on a file it cannot find before it has typeset anything, so q1 - # has no page while q2 still does. in2lambda leaves it out of the list it - # returns and names the files it did write after the question's place in - # the set, which is what the key must come from: counting the list would - # hand q2's page to q1 and leave q2 reading `not rendered`. - package.command( - draft, - "field replace", - { - "field": "q1.text", - "old": r"A ball is thrown straight up at $20\,\mathrm{m/s}$.", - "new": r"\input{no-such-file-at-all}", - }, - ) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - rendered = package.render(draft, tmp_path / "render") - - assert rendered == {"q2": tmp_path / "render" / "question_001_Question_2.pdf"} - assert rendered["q2"].is_file() - assert not (tmp_path / "render" / "question_000_Question_1.pdf").exists() diff --git a/tests/test_pair.py b/tests/test_pair.py index 3318294..f791f66 100644 --- a/tests/test_pair.py +++ b/tests/test_pair.py @@ -170,3 +170,16 @@ def test_solutions_with_no_questions_run_alone(tmp_path): solutions.write_bytes(b"%PDF") assert pair.of(solutions) == (solutions, None) + + +def test_a_tex_file_is_a_document_only_with_a_begin_document_in_it(tmp_path): + sheet = tmp_path / "sheet.tex" + sheet.write_text("\\begin{document}\nQuestion 1\n\\end{document}\n") + drawing = tmp_path / "tunnel-potential.tex" + drawing.write_text("\\begin{tikzpicture}\n \\draw (0,0) -- (4,0);\n\\end{tikzpicture}\n") + markdown = tmp_path / "sheet.md" + markdown.write_text("# Question 1\n") + + assert pair.is_document(sheet) + assert pair.is_document(markdown) + assert not pair.is_document(drawing) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py deleted file mode 100644 index 65ffbf1..0000000 --- a/tests/test_pipeline.py +++ /dev/null @@ -1,2520 +0,0 @@ -"""The end-to-end run: a markdown or PDF source in, a Lambda Feedback zip out.""" - -import json -import os -import random -import shutil -import warnings -import zipfile -from pathlib import Path - -import in2lambda.draft.export -import pytest -from conftest import PNG, FakeBackend, FakeMathpix -from in2lambda.source import ConversionToolsMissing -from in2lambda.validation.pdf import missing_tools - -from in2lambda_agent import package, pipeline -from in2lambda_agent.cli import main -from in2lambda_agent.model import ModelUnavailable -from in2lambda_agent.ocr import ocr_pdf -from in2lambda_agent.package import SourceError, SpecRejected -from in2lambda_agent.review import ReviewError -from in2lambda_agent.settings import Settings -from in2lambda_agent.spec import RECORD_NAME, SPEC_NAME, BadSpec - -# The command line of the spec route, which these tests cover. `run` without -# `--route spec` converts the document through the two routes of routes.py. -RUN = ["run", "--route", "spec"] - -FIXTURES = Path(__file__).parent / "fixtures" -SOURCE = FIXTURES / "sheet.md" -SPEC = (FIXTURES / "sheet-spec.yaml").read_text() -PAIRED_SPEC = (FIXTURES / "paired-spec.yaml").read_text() -LONE_SPEC = (FIXTURES / "lone-spec.yaml").read_text() -TEX_SPEC = (FIXTURES / "tex-sheet-spec.yaml").read_text() -FAULTY_SPEC = (FIXTURES / "faulty-spec.yaml").read_text() -IGNORES_THE_FIGURE = (FIXTURES / "figure-paragraph-spec.yaml").read_text() - -# What a model would run over the faulty sheet: the merged block cut in two and -# each half quoted, the solution the spec's selector missed given to the question -# it answers, and then the brace the OCR dropped out of that solution put back. -# The three quotations are layer 3; the replacement is the layer 4 edit, which -# in2lambda marks on the field rather than moving where it came from. -# -# A run scripted with these passes `tries=1`: the spec loop stops at one spec, -# and the replies after it are the round's commands rather than another spec. -FIXES = [ - ("split_block", {"block": "b7", "at": 14}), - ("question_add", {"text": "b7a"}), - ("part_add", {"question": "q2", "text": "b7b"}), - ("question_solution", {"question": "q2", "text": "b11"}), - ( - "field_replace", - {"field": "q2.solution", "old": r"\mathbf{B$", "new": r"\mathbf{B}$"}, - ), -] - -# The same fixes over the sheet with an unanswerable part in it, where the extra -# part has pushed every block and line below it along: the merged block is b8 at -# lines 15-16, and the solution the selector missed is b12. -UNSOLVED_FIXES = [ - ("split_block", {"block": "b8", "at": 16}), - ("question_add", {"text": "b8a"}), - ("part_add", {"question": "q2", "text": "b8b"}), - ("question_solution", {"question": "q2", "text": "b12"}), - ( - "field_replace", - {"field": "q2.solution", "old": r"\mathbf{B$", "new": r"\mathbf{B}$"}, - ), -] - -# The same spec with no `part` selector, so it runs but leaves every lettered -# part in no field: a saved spec the checks have something to say about. -PARTLESS_SPEC = "\n".join( - line for line in SPEC.splitlines() if not line.startswith("part:") -) - -# And with no `solution` selector, which leaves the four solution paragraphs in -# no field: fewer blocks over than PARTLESS_SPEC leaves, so the spec loop keeps -# this one of the two. -SOLUTIONLESS_SPEC = ( - "\n".join(line for line in SPEC.splitlines() if not line.startswith("solution:")) - + "\n" -) - -# A spec whose question selector is keyed to the wording of the first sheet — -# "A ball", "A block" — and so covers it completely while leaving the second -# sheet's stem, "A car brakes...", in no field. -FIRST_SHEET_SPEC = SPEC.replace("text~'^[A-Z]'", "text~'^A b'") - -# And one whose layout reads the solutions as a run of parts followed by a run -# of solutions, which covers sheet-2.md completely. It is the spec the two -# tests below have in2lambda refuse over the other sheet. -SECOND_SHEET_SPEC = SPEC.replace("PartsSepSol", "PartPartSolSol") - - -def refused_over_the_other_sheet(monkeypatch, text): - """Has in2lambda refuse one of the loop's specs over the set's other sheet. - - in2lambda reports a block two of a layout's fields would hold rather than - refusing the spec over the document holding it, so no spec these fixtures - can hold draws a refusal out of it over one sheet and not the other. It - still refuses a spec it cannot read there, and a run still has to score the - try that drew the refusal, so the refusal is put in here. - - Args: - monkeypatch: The test's own. - text: The spec of the try to refuse, as the loop writes it to the file. - """ - ran = package.spec_run - - def refusing(draft, spec): - # The copy of the other sheet, which the run freezes under its cache. - if "second" in Path(draft).parts and Path(spec).read_text() == text: - raise SpecRejected("the spec cannot be read over this document") - return ran(draft, spec) - - monkeypatch.setattr(package, "spec_run", refusing) - - -def drafted(folder, name): - """The draft a run over one sheet of a folder left, which is named after it.""" - return package.draft_of(folder / name) - - -@pytest.fixture -def sheets(tmp_path): - """A document set: two sheets written the same way, in a folder of their own.""" - folder = tmp_path / "sheets" - folder.mkdir() - for name in ("sheet.md", "sheet-2.md"): - shutil.copy(FIXTURES / name, folder / name) - return folder - - -@pytest.fixture -def faulty(tmp_path): - """A folder holding the sheet with three faults seeded in it, and nothing else. - - No spec beside it, so the run writes one — which keeps the rewrite a saved - spec gets out of the way of what the fixing rounds do. - """ - folder = tmp_path / "faulty" - folder.mkdir() - shutil.copy(FIXTURES / "faulty.md", folder / "faulty.md") - return folder - - -@pytest.fixture -def unsolved(tmp_path): - """The same sheet with a part the document never answers: q1 has a (c). - - Everything else about it is the faulty sheet, so a round has the two - findings it can answer and one it cannot. - """ - folder = tmp_path / "unsolved" - folder.mkdir() - shutil.copy(FIXTURES / "faulty-unsolved.md", folder / "faulty-unsolved.md") - return folder - - -@pytest.fixture -def questions_only(tmp_path): - """A sheet whose solutions are not on it, with the set's spec beside it. - - Nothing answers any of its four parts, so every check but `no-solution` is - quiet and that one is a warning: the set is written with them said. - """ - folder = tmp_path / "questions-only" - folder.mkdir() - shutil.copy(FIXTURES / "questions-only.md", folder / "questions-only.md") - (folder / SPEC_NAME).write_text(SPEC) - return folder - - -@pytest.fixture -def tex_sheets(tmp_path): - """A set of tex sheets, which is the shape the corpus keeps its sets in.""" - folder = tmp_path / "tex" - folder.mkdir() - for name in ("tex-sheet.tex", "tex-sheet-2.tex"): - shutil.copy(FIXTURES / name, folder / name) - return folder - - -@pytest.fixture -def figures(tmp_path): - """A sheet whose first question refers to an image, with the image beside it. - - The spec the set already has covers it: the reference sits in the same - block as the question's text. The image is a real PNG rather than a few - bytes named like one, because the checks now compile the set as the PDF - generator does, and a file xelatex cannot load is an error of its own. - """ - folder = tmp_path / "figures" - (folder / "figures").mkdir(parents=True) - shutil.copy(FIXTURES / "figure.md", folder / "figure.md") - shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") - (folder / SPEC_NAME).write_text(SPEC) - return folder - - -@pytest.fixture -def figure_paragraph(tmp_path): - """A sheet whose figure is a paragraph of its own, under a spec that drops it. - - Mathpix writes a figure this way: the image line and then its caption, apart - from the question they illustrate. The saved spec matches that caption with - its `ignore` selector, so the run's first draft is one in2lambda's checks - have nothing to say about and the image is in no field all the same. - """ - folder = tmp_path / "figure-paragraph" - (folder / "figures").mkdir(parents=True) - shutil.copy(FIXTURES / "figure-paragraph.md", folder / "figure-paragraph.md") - shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") - (folder / SPEC_NAME).write_text(IGNORES_THE_FIGURE) - return folder - - -@pytest.fixture -def paired(tmp_path): - """A sheet whose solutions are written as a file of their own beside it.""" - folder = tmp_path / "paired" - folder.mkdir() - for name in ("paired.md", "paired_solutions.md"): - shutil.copy(FIXTURES / name, folder / name) - return folder - - -@pytest.fixture -def marked(tmp_path): - """A pair whose solutions file writes a marker above each group of solutions. - - A solutions document written to stand on its own repeats the question it is - answering, so the `question` selector matches in the second source as well - as the first. `marked_solutions.md` leaves question 1's part (b) - unanswered, so the marker `Q2.` is what sends the solutions under it to - question 2 rather than to question 1's remaining part. - """ - folder = tmp_path / "marked" - folder.mkdir() - shutil.copy(FIXTURES / "paired.md", folder / "marked.md") - shutil.copy(FIXTURES / "marked_solutions.md", folder / "marked_solutions.md") - return folder - - -class PairedMathpix: - """A Mathpix client answering with the fixture each PDF is named after. - - Each conversion holds a figure, as a scanned sheet does, and both call it - `media/plot.png`: the pair of names that must not become one file when the - solutions are copied beside the questions. - """ - - def __init__(self): - self.calls: list[Path] = [] - - def convert(self, pdf: Path, media_dir: Path) -> str: - self.calls.append(Path(pdf)) - media_dir.mkdir(parents=True, exist_ok=True) - (media_dir / "plot.png").write_bytes(PNG) - lines = (FIXTURES / f"{Path(pdf).stem}.md").read_text().splitlines() - # Inside the first paragraph, which is a field of the draft: an image of - # a block the spec assigns to nothing is a coverage error, not a figure. - first = next( - number - for number, line in enumerate(lines) - if line and not line.startswith("#") - ) - lines[first] += " ![a plot](media/plot.png)" - return "\n".join(lines) + "\n" - - -def test_one_model_call_writes_the_sets_spec_and_the_run_builds(sheets, tmp_path): - backend = FakeBackend(SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - - assert len(backend.calls) == 1 - assert (sheets / SPEC_NAME).read_text() == SPEC - assert result.zip_path is not None and result.zip_path.exists() - # A zip was written, so there is nothing to say about why one was not. - assert result.reason == "" - - -def test_the_zip_holds_what_the_spec_made_of_the_source(sheets, tmp_path): - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(SPEC), - ) - - zip_file = zipfile.ZipFile(result.zip_path) - assert zip_file.namelist() == [ - "question_000_Question_1.json", - "question_001_Question_2.json", - "set_set.json", - ] - question = json.loads(zip_file.read("question_000_Question_1.json")) - # The parts with their markers stripped, and the solutions the layout - # paired to them out of the separate solutions section. - assert [part["content"] for part in question["parts"]] == [ - "Find the greatest height it reaches.", - "Find its time of flight.", - ] - assert [part["workedSolution"]["content"] for part in question["parts"]] == [ - "$h = v^2 / 2g = 20.4\\,\\mathrm{m}$", - "$t = 2v/g = 4.08\\,\\mathrm{s}$", - ] - - -def test_a_second_sheet_in_the_set_reuses_the_spec_with_no_call(sheets, tmp_path): - backend = FakeBackend(SPEC) - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "first", - settings=Settings(), - backend=backend, - ) - - result = pipeline.run( - sheets / "sheet-2.md", - out_dir=tmp_path / "second", - settings=Settings(), - backend=backend, - ) - spec = next(stage for stage in result.stages if stage.name == "spec") - - assert len(backend.calls) == 1 - assert spec.message == f"reused {sheets / SPEC_NAME}" - assert result.zip_path.exists() - - -def test_a_tex_set_is_frozen_to_markdown_and_shares_one_spec(tex_sheets, tmp_path): - backend = FakeBackend(TEX_SPEC) - - first = pipeline.run( - tex_sheets / "tex-sheet.tex", - out_dir=tmp_path / "first", - settings=Settings(), - backend=backend, - ) - second = pipeline.run( - tex_sheets / "tex-sheet-2.tex", - out_dir=tmp_path / "second", - settings=Settings(), - backend=backend, - ) - spec = next(stage for stage in second.stages if stage.name == "spec") - - # `source add` converts a tex sheet to markdown beside it and freezes that, - # so a folder of tex sheets is a set of markdown sources by the time the - # spec is written, and one spec does for all of them. - assert (tex_sheets / "tex-sheet.md").exists() - assert len(backend.calls) == 1 - assert spec.message == f"reused {tex_sheets / SPEC_NAME}" - assert (first.coverage.fields, second.coverage.fields) == ({1: 2}, {1: 3}) - assert first.zip_path.exists() and second.zip_path.exists() - - -def test_the_coverage_is_reported_and_recorded_beside_the_spec(sheets, tmp_path): - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(SPEC), - ) - coverage = next(stage for stage in result.stages if stage.name == "coverage") - - assert result.coverage.layout == "PartsSepSol" - assert result.coverage.unassigned == [] - # The sheet's fourteen blocks: four headings ignored, ten in fields. - assert (result.coverage.blocks, result.coverage.fields) == (14, {1: 10}) - assert coverage.message == ( - "PartsSepSol: 14 blocks, 10 fields at layer 1, 4 ignored, none unassigned" - ) - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - recorded = json.loads(line) - assert recorded["layout"] == "PartsSepSol" - assert (recorded["blocks"], recorded["fields"]) == (14, {"1": 10}) - assert recorded["unassigned"] == [] and recorded["reused"] is False - assert recorded["output_tokens"] == result.usage.output_tokens > 0 - # A spec that covers its source is the whole run: nothing was left to fix. - assert recorded["rounds"] == [] - - -def test_a_named_spec_is_read_from_where_it_was_named(sheets, tmp_path): - elsewhere = tmp_path / "mine.yaml" - elsewhere.write_text(SPEC) - backend = FakeBackend() - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - spec=elsewhere, - backend=backend, - ) - spec = next(stage for stage in result.stages if stage.name == "spec") - - assert backend.calls == [] - assert spec.message == f"reused {elsewhere}" - assert not (sheets / SPEC_NAME).exists() - assert result.zip_path.exists() - - -def test_a_named_spec_that_is_not_there_yet_is_written_there(sheets, tmp_path): - elsewhere = tmp_path / "mine.yaml" - - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - spec=elsewhere, - backend=FakeBackend(SPEC), - ) - - assert elsewhere.read_text() == SPEC - - -def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path): - backend = FakeBackend("question: NotAnElement\nlayout: PartsSepSol\n") - - with pytest.raises(SpecRejected, match="not a pandoc element"): - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - - assert not (tmp_path / "out").exists() - # And the set is not wedged: a spec in2lambda refused is not left for the - # next run to read instead of making a call of its own. - assert not (sheets / SPEC_NAME).exists() - - again = FakeBackend(SPEC) - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=again, - ) - - assert len(again.calls) == 1 - assert result.zip_path.exists() - - -def test_a_saved_spec_that_drops_an_image_is_written_again_and_the_run_goes_on( - figure_paragraph, tmp_path -): - # The checks find nothing in the draft the saved spec filled: the image it - # marked ignore is the only fault of it, and it is the one the run writes - # the spec again over. The rewrite keeps ignoring the figure, so the run - # reports the drop on its coverage line and builds the set without it. - backend = FakeBackend(IGNORES_THE_FIGURE) - - result = pipeline.run( - figure_paragraph / "figure-paragraph.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - validated = [stage.message for stage in result.stages if stage.name == "validate"] - covered = [stage.message for stage in result.stages if stage.name == "coverage"] - - assert validated[0] == ( - "b4 (lines 7-8) holds an image and is marked ignore. — " - "writing the set's spec again" - ) - assert len(backend.calls) == 1 - assert "b4 (lines 7-8) holds an image" in backend.calls[0][1] - assert covered[-1].endswith("; 1 image dropped: b4 (lines 7-8)") - assert validated[-1] == "nothing to report" - assert result.reused is False - assert result.zip_path is not None and result.zip_path.exists() - - -def test_a_rewrite_in2lambda_will_not_run_leaves_the_saved_spec_alone( - sheets, tmp_path -): - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - backend = FakeBackend("question: NotAnElement\nlayout: PartsSepSol\n") - - with pytest.raises(SpecRejected, match="not a pandoc element"): - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - - # The spec the rewrite was meant to improve on still runs, whatever the - # checks had to say about it; the one that does not run is gone. - assert (sheets / SPEC_NAME).read_text() == PARTLESS_SPEC - - -def test_a_try_the_loop_never_chose_is_not_left_beside_the_sources(sheets, tmp_path): - # The first call answers with a spec that runs but covers little, the second - # with something that is not a spec at all. The set keeps the spec it had: - # try 1 was written to the file, and no try was ever chosen. - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - backend = FakeBackend(SOLUTIONLESS_SPEC, "I would rather not.") - - with pytest.raises(BadSpec): - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - - assert len(backend.calls) == 2 - assert (sheets / SPEC_NAME).read_text() == PARTLESS_SPEC - - -def test_a_document_of_the_set_in2lambda_cannot_read_is_passed_over( - sheets, tmp_path, monkeypatch -): - # A folder holds files that are not documents — a Word lock file beside a - # docx — and in2lambda refuses them. The run reports the file and converts - # the source it was asked for. - freeze = package.source_add - - def refuse(source): - if Path(source).name == "sheet-2.md": - raise SourceError("pandoc could not read sheet-2.md") - return freeze(source) - - monkeypatch.setattr(package, "source_add", refuse) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(SPEC), - ) - (over_set,) = [stage for stage in result.stages if stage.name == "set"] - - assert over_set.message == ( - "sheet-2.md cannot be read: pandoc could not read sheet-2.md" - ) - assert result.zip_path is not None and result.zip_path.exists() - # The try is then judged on this source alone, and the record says which - # document the run passed over and why, rather than reading as a set of one. - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - (one,) = record["iterations"] - assert one["second"] is None and one["chosen"] is True - assert record["second"] == { - "name": "sheet-2.md", - "passed_over": "pandoc could not read sheet-2.md", - } - - -def test_a_sheet_of_the_set_that_has_a_draft_of_its_own_is_left_alone( - sheets, tmp_path -): - # A run over sheet-2.md left the draft beside it, holding that sheet's - # fields and the log of the commands that wrote them. Freezing sheet-2.md - # to try a spec over it would write the draft again from the source and - # delete both, so the loop runs each spec over a copy in the cache and the - # draft beside the sheet is still the one that run wrote. - elsewhere = tmp_path / "other-spec.yaml" - elsewhere.write_text(SPEC) - second = package.source_add(sheets / "sheet-2.md") - package.spec_run(second, elsewhere) - before = second.read_text() - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - backend=FakeBackend(SPEC), - ) - (over_set,) = [stage for stage in result.stages if stage.name == "set"] - - assert second.read_text() == before - assert (tmp_path / "cache" / "second" / "sheet-2.md").is_file() - assert result.zip_path is not None and result.zip_path.exists() - # The sheet is evidence about the spec all the same: the copy is run over, - # the line names the sheet, and the try carries what it left there. - assert over_set.message.startswith("sheet-2.md: ") - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - (one,) = record["iterations"] - assert one["second"] == 0 - assert record["second"] == {"name": "sheet-2.md", "passed_over": None} - - -def test_a_saved_spec_the_checks_fault_is_written_again_once(sheets, tmp_path): - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - backend = FakeBackend(SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - names = [stage.name for stage in result.stages] - first, second = (stage for stage in result.stages if stage.name == "spec") - - assert len(backend.calls) == 1 - # The report the first pass left is what the second was asked to answer. - assert "is in no field" in backend.calls[0][1] - assert first.message.startswith("reused ") - assert second.message.startswith(f"wrote {sheets / SPEC_NAME}") - assert names.count("validate") == 2 - assert (sheets / SPEC_NAME).read_text() == SPEC - assert result.zip_path.exists() - - -def test_a_fresh_spec_the_checks_fault_stops_the_run_with_no_zip(sheets, tmp_path): - backend = FakeBackend(PARTLESS_SPEC) - - # No rounds, so the report is where the run ends: what the rounds make of a - # report is below. - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=0, - tries=1, - backend=backend, - ) - stages = {stage.name: stage.message for stage in result.stages} - - assert len(backend.calls) == 1 - assert "is in no field and not marked ignore" in stages["validate"] - # The blocks left over are named by block id, as the coverage line and the - # record say they are: the sheet's four lettered parts, with the two - # solutions that had no part to pair with. - assert result.coverage.unassigned == ["b4", "b5", "b8", "b9", "b13", "b14"] - assert "b4, b5, b8, b9, b13, b14 unassigned" in stages["coverage"] - assert result.zip_path is None - assert result.clean is False - assert not (tmp_path / "out").exists() - - -def test_the_spec_is_written_again_against_what_running_the_last_one_covered( - sheets, tmp_path -): - # Three specs, none of them clean: the second leaves four blocks over where - # the first and third leave six, so it is the one the set keeps. - backend = FakeBackend(PARTLESS_SPEC, SOLUTIONLESS_SPEC, PARTLESS_SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=0, - backend=backend, - ) - kept = [stage for stage in result.stages if stage.name == "spec"][-1] - fields = json.loads(drafted(sheets, "sheet.md").read_text())["fields"] - - assert len(backend.calls) == 3 - # What the first spec covered is in the second call's prompt, by the block - # ids the coverage line names and the sentences the checks wrote. - assert "b4, b5, b8, b9, b13, b14 unassigned" in backend.calls[1][1] - assert "b4 (lines 7-7) is in no field" in backend.calls[1][1] - assert "sheet-2.md, another document of this set" in backend.calls[1][1] - assert kept.message == "kept try 2 of 3" - # The spec beside the sources, the coverage the run carries and the draft on - # disk are all the second try's, which the third try wrote over and the loop - # ran again. - assert (sheets / SPEC_NAME).read_text() == SOLUTIONLESS_SPEC - assert result.coverage.unassigned == ["b11", "b12", "b13", "b14"] - assert "q1.p1.text" in fields - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - iterations = json.loads(line)["iterations"] - assert [one["try"] for one in iterations] == [1, 2, 3] - assert all(one["input_tokens"] > 0 and one["seconds"] > 0 for one in iterations) - assert [one["unassigned"] for one in iterations] == [6, 4, 6] - assert [one["chosen"] for one in iterations] == [False, True, False] - - -def test_on_stage_sees_each_line_of_the_spec_loop_as_it_is_made(sheets, tmp_path): - # The page that shows a run sends each line to the browser as the run adds - # it, so the spec loop reports a line when it makes it rather than every - # line at the end. What the backend has been told when it is called is what - # says which of the two happened. - watched: list[tuple[str, str]] = [] - seen: list[list[str]] = [] - - class Watching(FakeBackend): - def call(self, system, prompt, tools=(), images=()): - seen.append([name for name, _ in watched]) - return super().call(system, prompt, tools, images) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=0, - tries=2, - backend=Watching(PARTLESS_SPEC, SPEC), - on_stage=lambda stage: watched.append((stage.name, stage.message)), - ) - - # The first call is made once the source is frozen, and the second once the - # first spec has been run over this sheet and over the other one. - assert seen[0] == ["ocr", "freeze"] - assert seen[1] == [ - "ocr", - "freeze", - "spec", - "coverage", - "validate", - "set", - "freeze", - ] - assert watched == [(stage.name, stage.message) for stage in result.stages] - - -def test_a_rewrite_reads_the_other_sheet_of_a_set_whose_sheets_all_have_drafts( - sheets, tmp_path -): - # The sweep's third sheet: every other sheet of the set has been run and has - # a draft beside it, and the saved spec faults this one. The rewrite is the - # call this ticket added try 0 for, so try 0 is run over the other sheet as - # well and the first call reads what the saved spec left there. - elsewhere = tmp_path / "other-spec.yaml" - elsewhere.write_text(SPEC) - second = package.source_add(sheets / "sheet-2.md") - package.spec_run(second, elsewhere) - before = second.read_text() - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - backend = FakeBackend(FIRST_SHEET_SPEC, SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - tries=2, - backend=backend, - ) - - # The saved spec leaves b4, b5 and b8 of sheet-2.md in no field, and the - # first call is asked for a spec that does not. - assert "another document of this set, left b4, b5, b8" in backend.calls[0][1] - assert (sheets / SPEC_NAME).read_text() == SPEC - assert second.read_text() == before - assert result.zip_path is not None and result.zip_path.exists() - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - # Try 0 is the saved spec, try 1 covers this sheet alone and try 2 the set. - assert [one["try"] for one in record["iterations"]] == [0, 1, 2] - assert [one["second"] for one in record["iterations"]] == [3, 5, 0] - assert record["second"] == {"name": "sheet-2.md", "passed_over": None} - - -def test_a_spec_in2lambda_refuses_over_the_other_sheet_is_scored_on_that( - sheets, tmp_path, monkeypatch -): - # The first spec covers this sheet, and in2lambda refuses it over the other - # sheet. in2lambda reads the other sheet, so the copy of it stays and the - # next try is run over it as well, and the refused try scores as leaving - # every block of it in no field. - refused_over_the_other_sheet(monkeypatch, SECOND_SHEET_SPEC) - backend = FakeBackend(SECOND_SHEET_SPEC, SPEC) - - result = pipeline.run( - sheets / "sheet-2.md", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - tries=2, - backend=backend, - ) - refused, ran = [stage for stage in result.stages if stage.name == "set"] - - assert refused.message.startswith("sheet.md: in2lambda refused the spec: ") - assert ran.message.startswith("sheet.md: ") - assert (sheets / SPEC_NAME).read_text() == SPEC - assert result.zip_path is not None and result.zip_path.exists() - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - # The refused try covers this sheet completely, so the other sheet is what - # carried the loop on to the try that covers both sheets. The refused spec - # wrote no field in sheet.md, so it left all 14 of its blocks in no field. - assert [one["unassigned"] for one in record["iterations"]] == [0, 0] - assert [one["second"] for one in record["iterations"]] == [14, 0] - assert [one["chosen"] for one in record["iterations"]] == [False, True] - # The spec the run kept ran over the document, so the record does not say - # the document was passed over. - assert record["second"] == {"name": "sheet.md", "passed_over": None} - - -def test_a_refusal_that_went_with_a_try_the_run_threw_away_is_not_recorded( - sheets, tmp_path, monkeypatch -): - # The second spec covers this sheet and in2lambda refuses it over the other - # sheet, so the run keeps the first, which ran over both. What the record - # says became of the other sheet is what the kept spec made of it, not what - # the try after it did. - refused_over_the_other_sheet(monkeypatch, SECOND_SHEET_SPEC) - backend = FakeBackend(SOLUTIONLESS_SPEC, SECOND_SHEET_SPEC) - - result = pipeline.run( - sheets / "sheet-2.md", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - rounds=0, - tries=2, - backend=backend, - ) - ran, refused = [stage for stage in result.stages if stage.name == "set"] - kept = [stage for stage in result.stages if stage.name == "spec"][-1] - - # The stage lines are the log of the loop, so both tries are still in them. - assert ran.message.startswith("sheet.md: PartsSepSol") - assert refused.message.startswith("sheet.md: in2lambda refused the spec: ") - assert kept.message == "kept try 1 of 2" - assert (sheets / SPEC_NAME).read_text() == SOLUTIONLESS_SPEC - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - assert [one["chosen"] for one in record["iterations"]] == [True, False] - assert record["second"] == {"name": "sheet.md", "passed_over": None} - - -def test_the_sheets_own_solutions_file_is_no_other_document_of_the_set( - paired, tmp_path -): - # The solutions file is the second source of this run's own draft, so it is - # not another document for a spec to be judged over, and the folder holds - # nothing else. - result = pipeline.run( - paired / "paired.md", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - backend=FakeBackend(PAIRED_SPEC), - ) - - assert not [stage for stage in result.stages if stage.name == "set"] - assert not (tmp_path / "cache" / "second").exists() - (line,) = (paired / RECORD_NAME).read_text().splitlines() - assert json.loads(line)["second"] is None - - -@pytest.mark.skipif(os.geteuid() == 0, reason="root may read any file") -def test_a_document_of_the_set_the_run_cannot_copy_is_passed_over(sheets, tmp_path): - # The copy is what keeps the spec loop off the set's own drafts, so a sheet - # the OS will not let the run copy is one to pass over, as one in2lambda - # cannot read is, rather than one that ends the run with no spec written. - (sheets / "sheet-2.md").chmod(0o000) - try: - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - backend=FakeBackend(SPEC), - ) - finally: - (sheets / "sheet-2.md").chmod(0o644) - (over_set,) = [stage for stage in result.stages if stage.name == "set"] - - assert over_set.message.startswith("sheet-2.md passed over: copying it failed: ") - assert result.zip_path is not None and result.zip_path.exists() - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - assert record["second"]["name"] == "sheet-2.md" - assert record["second"]["passed_over"].startswith("copying it failed: ") - (one,) = record["iterations"] - assert one["second"] is None - - -def test_a_pdf_beside_a_pdf_source_is_passed_over_and_the_record_says_why( - pdf, tmp_path -): - (tmp_path / "sheet-2.pdf").write_bytes(b"%PDF-1.4 the next sheet") - - result = pipeline.run( - pdf, - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - mathpix=FakeMathpix(markdown=SOURCE.read_text()), - backend=FakeBackend(SPEC), - ) - (over_set,) = [stage for stage in result.stages if stage.name == "set"] - - assert over_set.message == ( - "sheet-2.pdf passed over: converting it takes an OCR call, and the " - "spec loop makes no call but the model's" - ) - (line,) = (tmp_path / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - assert record["second"]["name"] == "sheet-2.pdf" - assert record["second"]["passed_over"].startswith("converting it takes") - (one,) = record["iterations"] - assert one["second"] is None - - -def test_a_saved_spec_the_checks_fault_is_the_try_the_rewrite_improves_on( - sheets, tmp_path -): - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - backend = FakeBackend(SPEC) - - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - saved, written = json.loads(line)["iterations"] - - # The saved spec is try 0: it is what the call was asked to improve on, and - # it cost no call of its own. - assert (saved["try"], saved["unassigned"], saved["chosen"]) == (0, 6, False) - assert saved["input_tokens"] == saved["output_tokens"] == 0 - assert (written["try"], written["unassigned"], written["chosen"]) == (1, 0, True) - assert written["output_tokens"] > 0 - - -def test_a_spec_that_covers_this_sheet_alone_does_not_stop_the_loop(sheets, tmp_path): - # The first spec covers sheet.md completely and leaves sheet-2.md's stem in - # no field. The spec is saved for the whole set, so that is not a spec to - # stop at: the second call is made, and covers both. - backend = FakeBackend(FIRST_SHEET_SPEC, SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=2, - backend=backend, - ) - over_set = [stage for stage in result.stages if stage.name == "set"] - - assert len(backend.calls) == 2 - assert over_set[0].message.startswith("sheet-2.md: ") - # Nothing of the second sheet is covered: its stem is in no field, and its - # parts and solutions have no question to belong to. - assert "b3, b4, b5, b7, b8 unassigned" in over_set[0].message - assert (sheets / SPEC_NAME).read_text() == SPEC - assert result.zip_path is not None and result.zip_path.exists() - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - iterations = json.loads(line)["iterations"] - assert [one["unassigned"] for one in iterations] == [0, 0] - assert [one["second"] for one in iterations] == [5, 0] - assert [one["chosen"] for one in iterations] == [False, True] - - -def test_a_sheet_with_no_other_document_beside_it_is_judged_on_its_own( - faulty, tmp_path -): - backend = FakeBackend(FAULTY_SPEC, FIXES) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - - assert not [stage for stage in result.stages if stage.name == "set"] - (line,) = (faulty / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - (one,) = record["iterations"] - assert one["second"] is None and one["chosen"] is True - # Null here is the folder holding no other document, which is what a reader - # of the record can tell it from a document the run passed over by. - assert record["second"] is None - - -def test_the_rounds_fix_what_the_checks_found_and_the_run_builds(faulty, tmp_path): - backend = FakeBackend(FAULTY_SPEC, FIXES) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - fixed = next(stage for stage in result.stages if stage.name == "fix") - - # One call to write the spec, one round to answer what it left over. - assert len(backend.calls) == 2 - assert [stage.name for stage in result.stages] == [ - "ocr", - "freeze", - "spec", - "coverage", - "validate", - "fix", - "validate", - "review", - "build", - ] - assert fixed.message.startswith( - "round 1: 5 commands (split block b7, question add b7a, part add q2, " - "question solution q2, field replace q2.solution), " - ) - assert result.stages[-3].message == "nothing to report" - assert result.zip_path is not None and result.zip_path.exists() - - -def test_each_round_answers_what_the_one_before_it_left(faulty, tmp_path): - # The same fixes, spread over two rounds: the second is given the report the - # first left behind, and the source with the block the first split in it. - backend = FakeBackend(FAULTY_SPEC, FIXES[:2], FIXES[2:]) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - second = backend.calls[2][1] - - assert [one.number for one in result.rounds] == [1, 2] - assert [one.left for one in result.rounds] == [2, 0] - # The half of b7 still in no field, under the id the first round's split - # gave it, which is not an id the first round was shown. - assert "b7b (lines 14-14) is in no field" in second - assert "b7b" in second.split("in2lambda validate reports")[0] - assert result.zip_path is not None and result.zip_path.exists() - - -def test_every_fix_is_in_the_drafts_log_with_the_layer_it_wrote(faulty, tmp_path): - pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=FakeBackend(FAULTY_SPEC, FIXES), - ) - log = package.command_log(drafted(faulty, "faulty.md")) - fields = json.loads(drafted(faulty, "faulty.md").read_text())["fields"] - - # The spec, and then every fix after it, all recorded by in2lambda as it - # applied them: replaying this log rebuilds the draft with no model in it. - assert [entry["command"] for entry in log] == [ - "spec run", - "split block", - "question add", - "part add", - "question solution", - "field replace", - ] - assert {entry["by"] for entry in log} == {package.BY} - # Each of those quoted a range of the source, so the fields they wrote are - # layer 3: what they say is what the source says, and can be shown against it. - quoted = ["q2.text", "q2.p1.text", "q2.solution"] - assert [fields[key]["layer"] for key in quoted] == [3, 3, 3] - assert not any(fields[key]["edited"] for key in quoted[:2]) - # The replacement is the layer 4 work: it leaves the field quoting the lines - # it came from and marks it as no longer saying what they say, which is how - # the brace the OCR dropped ends up repaired in what is built. - assert fields["q2.solution"]["edited"] is True - assert r"\mathbf{B}$" in fields["q2.solution"]["value"] - # And the fields the spec wrote, which nothing touched, are still layer 1. - assert fields["q1.text"]["layer"] == 1 and not fields["q1.text"]["edited"] - - -def test_a_saved_log_is_run_after_the_spec_and_builds_with_no_call(faulty, tmp_path): - pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=FakeBackend(FAULTY_SPEC, FIXES), - ) - saved = tmp_path / "faulty.md.commands.json" - saved.write_text(json.dumps(package.fix_log(drafted(faulty, "faulty.md")))) - backend = FakeBackend(reason="a replay makes no call") - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "again", - settings=Settings(), - commands=saved, - rounds=0, - backend=backend, - ) - - # The commands run between the spec and the checks, so the draft the checks - # see is the one the first run's rounds repaired. - names = [one.name for one in result.stages] - assert names[names.index("coverage") + 1] == "replay" - assert names[names.index("replay") + 1] == "validate" - assert result.stages[names.index("replay")].message.startswith("5 commands from") - assert backend.calls == [] - assert result.rounds == [] - assert result.zip_path is not None and result.zip_path.exists() - - -def test_the_record_says_what_each_round_cost(faulty, tmp_path): - pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=FakeBackend(FAULTY_SPEC, FIXES), - ) - - (line,) = (faulty / RECORD_NAME).read_text().splitlines() - (round_one,) = json.loads(line)["rounds"] - - assert round_one["round"] == 1 - assert round_one["input_tokens"] > 0 and round_one["output_tokens"] > 0 - assert round_one["seconds"] > 0 - assert round_one["commands"] == [name for name, _ in FIXES] - # Nothing left for a second round, which is why there was not one. - assert round_one["left"] == 0 - - -def test_a_command_in2lambda_refuses_is_answered_rather_than_ending_the_run( - faulty, tmp_path -): - backend = FakeBackend(FAULTY_SPEC, [("mark_ignore", {"block": "b99"})] + FIXES) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - refused = result.rounds[0].commands[0] - - assert "was refused" in refused.result and "no block b99" in refused.result - # The refusal is what the model was told, and it went on to fix the draft. - assert result.zip_path is not None and result.zip_path.exists() - assert "b99" not in str(package.command_log(drafted(faulty, "faulty.md"))) - - -def test_a_part_whose_solution_is_not_on_the_sheet_is_reported_not_written( - unsolved, tmp_path -): - # The round answers the two findings the source can answer and leaves the - # one it cannot: q1's third part has no solution anywhere in the document, - # and there is nothing to quote for it. - backend = FakeBackend(FAULTY_SPEC, UNSOLVED_FIXES) - - result = pipeline.run( - unsolved / "faulty-unsolved.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - checked = [stage for stage in result.stages if stage.name == "validate"][-1] - fields = json.loads(drafted(unsolved, "faulty-unsolved.md").read_text())["fields"] - - assert [stage.name for stage in result.stages].count("fix") == 1 - # A part nothing answers is a warning, not an error: it is said and the set - # is written anyway, since half the sheets there are keep their solutions - # somewhere else or have none. - assert "q1.p3" in checked.message and "has no solution" in checked.message - assert checked.message.endswith("— warnings, building") - # Nothing was typed into the gap, and nothing in the log could have been. - assert "q1.p3.solution" not in fields - assert not any( - "literal" in entry["args"] - for entry in package.command_log(drafted(unsolved, "faulty-unsolved.md")) - ) - assert result.clean is True - assert result.zip_path is not None and result.zip_path.exists() - # What the build went past, for the table a sweep writes to carry. - assert "q1.p3" in result.reason and "has no solution" in result.reason - - -def test_a_solution_the_model_types_out_is_refused_and_the_finding_stays( - unsolved, tmp_path -): - invented = "The ball rises, slows and falls back along the same line. " * 4 - backend = FakeBackend( - FAULTY_SPEC, - [("question_solution", {"question": "q1", "literal": invented})] - + UNSOLVED_FIXES, - ) - - result = pipeline.run( - unsolved / "faulty-unsolved.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - typed = result.rounds[0].commands[0] - - assert "was refused" in typed.result and "may be typed" in typed.result - # The refusal never reached in2lambda, so the draft was built by the - # quotations alone and the gap it could not answer is reported rather than - # filled: the set is written with the warning, not with an invented answer. - assert not any( - "literal" in entry["args"] - for entry in package.command_log(drafted(unsolved, "faulty-unsolved.md")) - ) - checked = [stage for stage in result.stages if stage.name == "validate"][-1] - assert checked.message.endswith("— warnings, building") - assert "q1.p3" in result.reason and "has no solution" in result.reason - - -def test_a_questions_only_sheet_builds_with_its_warnings_in_the_reason( - questions_only, tmp_path -): - # A sheet with no solutions on it at all, which is half the sets there are: - # every part is a warning, nothing is an error, and the set is written. - result = pipeline.run( - questions_only / "questions-only.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=0, - ) - checked = [stage for stage in result.stages if stage.name == "validate"][-1] - - # No round was spent on them, and the saved spec was not written again: - # there is nothing here a second spec would cover any better. - assert (result.rounds, result.reused, result.clean) == ([], True, True) - assert result.zip_path is not None and result.zip_path.exists() - assert checked.message.count("has no solution") == 4 - assert checked.message.endswith("— warnings, building") - assert result.reason.count("has no solution") == 4 - - -def test_the_warnings_in2lambda_says_as_it_builds_reach_no_one_twice( - questions_only, tmp_path, capsys -): - # in2lambda warns about each of the four unanswered parts as it builds, and - # Python would print each one to stderr with the line of in2lambda that - # raised it. The validate stage line lists the same four, so the run prints - # its stage lines and nothing else. - with warnings.catch_warnings(record=True) as escaped: - warnings.simplefilter("always") - result = pipeline.run( - questions_only / "questions-only.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=0, - ) - - assert escaped == [] - printed = capsys.readouterr().err - assert "UserWarning" not in printed and "beartype" not in printed - built = [stage for stage in result.stages if stage.name == "build"] - assert not any(stage.message.startswith("warning: ") for stage in built) - assert built[-1].message == str(result.zip_path) - - -def test_a_warning_the_validate_line_does_not_list_gets_a_stage_line( - tmp_path, monkeypatch -): - # Everything in2lambda warns inside a build is handed back, including a - # warning no finding of the report accounts for. One of those is a line of - # its own, before the zip. - monkeypatch.setattr( - package, - "build", - lambda draft, out_dir: package.Built( - out_dir / "set.zip", ["q1.p1 has no solution", "something new"] - ), - ) - result = pipeline.RunResult() - - pipeline._build( - tmp_path / "sheet.draft.json", - tmp_path / "out", - result, - ["q1.p1 has no solution"], - ) - - assert [(stage.name, stage.message) for stage in result.stages] == [ - ("build", "warning: something new"), - ("build", str(tmp_path / "out" / "set.zip")), - ] - - -@pytest.mark.skipif( - shutil.which("node") is None or bool(missing_tools()), - reason="the set checks need Node for KaTeX and pandoc and xelatex to compile", -) -def test_a_problem_the_set_checks_find_reaches_the_next_round(faulty, tmp_path): - # The first round quotes the solution the spec missed and stops there, - # leaving the brace the OCR dropped out of it. That is in2lambda's own - # validation of the set rather than one of the draft's own checks, and it - # reaches the next round as a finding like any other. - backend = FakeBackend(FAULTY_SPEC, FIXES[:-1], FIXES[-1:]) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=2, - tries=1, - backend=backend, - ) - _, second_round = backend.calls[2] - - named = [ - line - for line in second_round.splitlines() - if line.startswith("- error problem q2.solution ") - ] - assert named and any(r"\mathbf{B" in line for line in named) - assert len(result.rounds) == 2 - assert result.zip_path is not None and result.zip_path.exists() - - -def test_a_round_that_answers_nothing_ends_the_run_with_what_it_left( - faulty, tmp_path -): - # A model that answers without running a command: the report the round was - # given is the report it left, so there is nothing for another round to do. - backend = FakeBackend(FAULTY_SPEC, []) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - tries=1, - backend=backend, - ) - last = result.stages[-1] - - assert [stage.name for stage in result.stages].count("fix") == 1 - assert [one.left for one in result.rounds] == [2] - assert last.name == "validate" - assert "is in no field and not marked ignore" in last.message - assert last.message.endswith("— left by round 1, no zip") - assert result.zip_path is None - assert not (tmp_path / "out").exists() - - -def test_a_field_the_round_tried_to_write_ends_the_run_naming_it(faulty, tmp_path): - # The round splits b7, so there is a finding no round was given before and - # the run would otherwise go on. Its other command answers a finding by - # writing a field, which the rounds have no command for: the run ends there - # naming the field, whatever the limit allows. - backend = FakeBackend( - FAULTY_SPEC, - [ - ("split_block", {"block": "b7", "at": 14}), - ("field_replace", {"field": "q1.text", "old": "", "new": "A ball."}), - ], - ) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=3, - tries=1, - backend=backend, - ) - last = result.stages[-1] - - assert [stage.name for stage in result.stages].count("fix") == 1 - assert last.name == "validate" - assert "q1.text" in last.message - assert "cannot be repaired by the loop" in last.message - assert last.message.endswith("left by round 1, no zip") - assert result.zip_path is None - # And the field is as the spec wrote it: a refused command writes nothing. - assert "A ball is thrown" in package.field_value( - drafted(faulty, "faulty.md"), "q1.text" - ) - - -def test_a_run_still_making_progress_stops_at_the_limit_with_no_zip(faulty, tmp_path): - # The split leaves b7b, which no round was given before: there is more for a - # round to do, and it is the limit rather than the report that stops the run. - backend = FakeBackend(FAULTY_SPEC, FIXES[:2]) - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=1, - tries=1, - backend=backend, - ) - last = result.stages[-1] - - assert [stage.name for stage in result.stages].count("fix") == 1 - assert "b7b" in last.message - assert last.message.endswith("— round limit 1 reached, no zip") - assert result.zip_path is None - # The reason is one finding, not the joined line the stage printed, and not - # the limit that stopped the rounds: it is what the draft is still faulted for. - assert result.reason == package.validate(result.draft).errors[0] - assert result.reason in last.message and "round limit" not in result.reason - assert not (tmp_path / "out").exists() - - -def test_the_rounds_running_out_prints_its_stages_and_exits_one( - faulty, tmp_path, monkeypatch, capsys -): - backend = FakeBackend(FAULTY_SPEC, FIXES[:2]) - monkeypatch.setattr(pipeline, "choose_backend", lambda settings: backend) - - code = main( - [ - *RUN, - str(faulty / "faulty.md"), - "--rounds", - "1", - "--tries", - "1", - "--out", - str(tmp_path / "out"), - ] - ) - printed = capsys.readouterr().out - - assert code == 1 - assert [line.split()[0] for line in printed.splitlines()].count("fix") == 1 - assert "round limit 1 reached, no zip" in printed - assert not (tmp_path / "out").exists() - - -def test_no_rounds_leaves_a_saved_spec_alone_and_stops(sheets, tmp_path): - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - backend = FakeBackend(SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - rounds=0, - backend=backend, - ) - - assert backend.calls == [] - assert result.zip_path is None - assert (sheets / SPEC_NAME).read_text() == PARTLESS_SPEC - - -def test_a_saved_spec_needs_no_backend(sheets, tmp_path): - (sheets / SPEC_NAME).write_text(SPEC) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(reason="no login"), - ) - - assert result.zip_path.exists() - assert result.usage.output_tokens == 0 - - -def test_writing_a_spec_without_a_backend_says_what_to_do(sheets, tmp_path): - with pytest.raises(ModelUnavailable, match="run claude login"): - pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(reason="run claude login"), - ) - - -def test_the_stages_run_in_order(sheets, tmp_path): - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(SPEC), - ) - - assert [stage.name for stage in result.stages] == [ - "ocr", - "freeze", - "spec", - "coverage", - "validate", - # What the spec covered of the set's other sheet, which is the last - # thing the choice between two specs is made on. - "set", - "review", - "build", - ] - - -def reviewed(sheets, tmp_path, mode="sample", **given): - """A run stopped for review, with its cache and out directories under tmp.""" - (sheets / SPEC_NAME).write_text(SPEC) - return pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - review=mode, - cache_dir=tmp_path / "cache", - rng=random.Random(0), - backend=FakeBackend(), - **given, - ) - - -@pytest.mark.skipif( - bool(missing_tools()), reason="rendering the questions needs pandoc and xelatex" -) -def test_a_review_stops_the_run_with_the_questions_listed_and_no_zip( - sheets, tmp_path -): - result = reviewed(sheets, tmp_path) - stages = {stage.name: stage.message for stage in result.stages} - pages = tmp_path / "out" / "render" - - assert [stage.name for stage in result.stages][-3:] == [ - "validate", - "render", - "review", - ] - # Each question by its key, the PDF it was rendered to, and the lines of - # the frozen source it was built from. - assert stages["render"] == f"2 questions to {pages}" - assert ( - f"q1 pending: {pages / 'question_000_Question_1.pdf'}, " - f"{sheets / 'sheet.md'} lines 5-5, 7-7" - ) in stages["review"] - assert (pages / "question_000_Question_1.pdf").is_file() - assert f"q2 pending: {pages / 'question_001_Question_2.pdf'}, " in stages["review"] - assert "in2lambda-agent review approve Q" in stages["review"] - # Nothing built, and no run recorded: the run is not over. - assert result.zip_path is None - assert not (tmp_path / "out" / "set.zip").exists() - assert not (sheets / RECORD_NAME).exists() - assert json.loads((tmp_path / "cache" / "review.json").read_text())["mode"] == ( - "sample" - ) - - -def test_a_review_of_a_question_a_literal_wrote_lists_the_lines_it_has( - faulty, tmp_path -): - # The faulty sheet again, except that the round found the solution's line - # too mangled to quote: it marked that block as belonging nowhere and typed - # the repair out instead. So q2.solution is layer 4 with no range of the - # source behind it, which is the field the listing has to read past. - TYPED_FIXES = [ - ("split_block", {"block": "b7", "at": 14}), - ("question_add", {"text": "b7a"}), - ("part_add", {"question": "q2", "text": "b7b"}), - ("mark_ignore", {"block": "b11"}), - ( - "question_solution", - { - "question": "q2", - "literal": r"Write $\mathbf{B}$ in components and differentiate.", - }, - ), - ] - - result = pipeline.run( - faulty / "faulty.md", - out_dir=tmp_path / "out", - settings=Settings(), - review="sample", - cache_dir=tmp_path / "cache", - rng=random.Random(0), - tries=1, - backend=FakeBackend(FAULTY_SPEC, TYPED_FIXES), - ) - stages = {stage.name: stage.message for stage in result.stages} - written = json.loads(drafted(faulty, "faulty.md").read_text())["fields"] - - assert (written["q2.solution"]["layer"], written["q2.solution"]["edited"]) == ( - 4, - True, - ) - assert written["q2.solution"]["ranges"] == [] - # The run reaches the reviewer rather than the field with no ranges in it - # stopping the listing, and q2 is named by the lines its other fields do - # have — the typed one adds none. - assert package.questions(drafted(faulty, "faulty.md"))["q2"].ranges == [ - [13, 13], - [14, 14], - ] - assert "q2 pending: " in stages["review"] - assert "lines 13-13, 14-14" in stages["review"] - assert result.zip_path is None - - -def test_a_sample_shows_a_few_questions_and_per_question_shows_them_all( - sheets, tmp_path -): - sample = reviewed(sheets, tmp_path, sample=1) - every = reviewed(sheets, tmp_path, mode="per-question") - - assert len(sample.review.questions) == 1 - assert [one.key for one in every.review.questions] == ["q1", "q2"] - - -def test_a_review_carries_on_when_the_pages_cannot_be_compiled( - sheets, tmp_path, monkeypatch -): - # A machine without xelatex, which is where most of the corpus is read: the - # render stage says what in2lambda said, and the reviewer gets the same - # listing named by lines instead of pages. - def missing(draft, directory): - raise ConversionToolsMissing("Rendering questions needs xelatex.") - - monkeypatch.setattr(in2lambda.draft.export, "render", missing) - - result = reviewed(sheets, tmp_path) - stages = {stage.name: stage.message for stage in result.stages} - - assert stages["render"] == "Rendering questions needs xelatex." - assert f"q1 pending: not rendered, {sheets / 'sheet.md'} lines 5-5, 7-7" in ( - stages["review"] - ) - assert result.zip_path is None - assert json.loads((tmp_path / "cache" / "review.json").read_text())["mode"] == ( - "sample" - ) - - -def test_the_render_line_counts_one_page_as_one_question(tmp_path, monkeypatch): - # A sheet with one question, or a set the compiler gave up on all but one - # of: either way the line reads as English rather than `1 questions`. - monkeypatch.setattr( - package, - "render", - lambda draft, out: {"q1": out / "question_000_Question_1.pdf"}, - ) - - rendered, message = pipeline._render( - tmp_path / "sheet.draft.json", tmp_path / "out" - ) - - assert list(rendered) == ["q1"] - assert message == f"1 question to {tmp_path / 'out' / 'render'}" - - -def test_approving_every_question_builds_the_set_and_records_the_review( - sheets, tmp_path -): - waiting = reviewed(sheets, tmp_path) - - for question in list(waiting.review.questions): - result = pipeline.resume( - tmp_path / "cache", - verdict="approve", - key=question.key, - settings=Settings(), - ) - - assert result.zip_path is not None and result.zip_path.exists() - # The review is answered, so the record of it goes, and the run's line is - # written with what the reviewer said in it. - assert not (tmp_path / "cache" / "review.json").exists() - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - recorded = json.loads(line)["review"] - assert recorded["mode"] == "sample" - assert [one["status"] for one in recorded["questions"]] == ["approved"] * 2 - assert recorded["rejections"] == [] and recorded["edits"] == [] - - -def test_the_line_an_approval_writes_names_the_other_document_of_the_set( - sheets, tmp_path -): - # The spec loop ran before the review, so the line the last approval writes - # says which document of the set the specs were run over, as the line a run - # without a review writes does. - waiting = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - review="sample", - cache_dir=tmp_path / "cache", - rng=random.Random(0), - backend=FakeBackend(SPEC), - ) - - for question in list(waiting.review.questions): - pipeline.resume( - tmp_path / "cache", - verdict="approve", - key=question.key, - settings=Settings(), - ) - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - record = json.loads(line) - assert record["second"] == {"name": "sheet-2.md", "passed_over": None} - assert [one["second"] for one in record["iterations"]] == [0] - - -def test_a_review_is_answered_from_wherever_the_reviewer_is( - sheets, tmp_path, monkeypatch -): - (sheets / SPEC_NAME).write_text(SPEC) - monkeypatch.chdir(sheets) - waiting = pipeline.run( - Path("sheet.md"), - out_dir=tmp_path / "out", - settings=Settings(), - review="per-question", - cache_dir=tmp_path / "cache", - backend=FakeBackend(), - ) - # The review commands are another process, run from wherever the reviewer - # happens to be, so the record's paths cannot mean the run's directory. - monkeypatch.chdir(tmp_path) - - for question in list(waiting.review.questions): - result = pipeline.resume( - tmp_path / "cache", - verdict="approve", - key=question.key, - settings=Settings(), - ) - - assert result.zip_path is not None and result.zip_path.exists() - assert (sheets / RECORD_NAME).is_file() - - -def test_a_question_that_is_not_under_review_is_refused(sheets, tmp_path): - reviewed(sheets, tmp_path) - - with pytest.raises(ReviewError, match="q9 is not under review"): - pipeline.resume( - tmp_path / "cache", verdict="approve", key="q9", settings=Settings() - ) - - -def test_a_rejection_goes_back_to_the_fix_loop_with_the_note(sheets, tmp_path): - waiting = reviewed(sheets, tmp_path) - backend = FakeBackend( - [("field_replace", {"field": "q2.p2.text", "old": "least", "new": "smallest"})] - ) - - result = pipeline.resume( - tmp_path / "cache", - verdict="reject", - key="q2", - note="part (b) asks for the smallest coefficient", - settings=Settings(), - backend=backend, - ) - stages = [stage.name for stage in result.stages] - - # Exactly one round, with the note in what the model was asked, and the - # checks run again after it. - assert len(backend.calls) == 1 - assert "The reviewer rejected q2: part (b) asks" in backend.calls[0][1] - assert stages.count("fix") == 1 - assert [one.number for one in result.rounds] == [1] - assert result.stages[stages.index("fix") + 1].message == "nothing to report" - # And q2 is back in front of the reviewer, with the note saying why. - assert result.zip_path is None - saved = pipeline.Review.load(tmp_path / "cache" / "review.json") - assert saved.question("q2").status == "pending" - assert saved.rejections == [ - {"key": "q2", "note": "part (b) asks for the smallest coefficient"} - ] - assert "smallest coefficient" in saved.listing() - assert waiting.review.question("q2").lines == saved.question("q2").lines - - -def test_a_rejection_with_no_rounds_left_says_so_rather_than_doing_nothing( - sheets, tmp_path -): - reviewed(sheets, tmp_path, rounds=0) - logged = package.command_log(drafted(sheets, "sheet.md")) - - # No backend, and none to be had: a run with no rounds in it never asks for - # one, so a machine with no key can still record what the reviewer said. - result = pipeline.resume( - tmp_path / "cache", - verdict="reject", - key="q2", - note="part (b) is wrong", - settings=Settings(), - ) - stages = {stage.name: stage.message for stage in result.stages} - - assert "no rounds left" in stages["fix"] - # Nothing was run, so the draft is as it was and q2 comes back unchanged — - # but the note is in the record, so it says the reviewer objected and why. - assert package.command_log(drafted(sheets, "sheet.md")) == logged - saved = pipeline.Review.load(tmp_path / "cache" / "review.json") - assert saved.question("q2").status == "pending" - assert saved.rejections == [{"key": "q2", "note": "part (b) is wrong"}] - assert "part (b) is wrong" in stages["review"] - - -def test_a_reviewers_edit_is_logged_as_theirs_and_leaves_the_question_waiting( - sheets, tmp_path -): - reviewed(sheets, tmp_path) - - result = pipeline.resume( - tmp_path / "cache", - verdict="edit", - field="q1.text", - old="ball", - new="stone", - by="ada", - settings=Settings(), - ) - fields = json.loads(drafted(sheets, "sheet.md").read_text())["fields"] - - assert package.command_log(drafted(sheets, "sheet.md"))[-1] == { - "command": "field replace", - "args": {"field": "q1.text", "old": "ball", "new": "stone"}, - "by": "ada", - } - assert fields["q1.text"]["edited"] is True and "stone" in fields["q1.text"]["value"] - assert result.zip_path is None - saved = pipeline.Review.load(tmp_path / "cache" / "review.json") - assert saved.edits == [{"field": "q1.text", "by": "ada"}] - assert saved.question("q1").status == "pending" - - -# An edit that leaves a field holding nothing, which is the shortest way for a -# reviewer to put a draft the checks fault in front of the next command. -EMPTIES = { - "field": "q1.text", - "old": r"A ball is thrown straight up at $20\,\mathrm{m/s}$.", - "new": " ", -} - - -def test_an_edit_that_faults_the_draft_says_so_in_the_listing(sheets, tmp_path): - reviewed(sheets, tmp_path) - - result = pipeline.resume( - tmp_path / "cache", verdict="edit", **EMPTIES, by="ada", settings=Settings() - ) - - assert "q1.text (lines 5-5) is empty." in result.stages[-1].message - assert "the checks fault the draft" in result.stages[-1].message - saved = pipeline.Review.load(tmp_path / "cache" / "review.json") - assert saved.errors == ["q1.text (lines 5-5) is empty."] - - -def test_a_rejection_the_rounds_cannot_answer_leaves_the_fault_in_the_listing( - sheets, tmp_path -): - reviewed(sheets, tmp_path, rounds=1) - # A round that makes things worse rather than better: the checks were quiet - # when the reviewer was asked, and are not when they answer. The round - # repairs wording, since a round may not replace the whole of a field as the - # reviewer's own edit does, and the repair drops a closing $. - backend = FakeBackend( - [("field_replace", {"field": "q1.text", "old": r"m/s}$.", "new": "m/s}."})] - ) - - result = pipeline.resume( - tmp_path / "cache", - verdict="reject", - key="q2", - note="part (b) answers the wrong question", - settings=Settings(), - backend=backend, - ) - - assert "the checks fault the draft" in result.stages[-1].message - saved = pipeline.Review.load(tmp_path / "cache" / "review.json") - assert "q1.text (lines 5-5): unclosed inline $ ... $" in saved.errors - assert saved.question("q2").status == "pending" - - -def test_approving_a_draft_the_checks_fault_refuses_to_build_it(sheets, tmp_path): - reviewed(sheets, tmp_path) - pipeline.resume( - tmp_path / "cache", verdict="edit", **EMPTIES, by="ada", settings=Settings() - ) - - for key in ("q1", "q2"): - result = pipeline.resume( - tmp_path / "cache", verdict="approve", key=key, settings=Settings() - ) - checked = [stage for stage in result.stages if stage.name == "validate"][-1] - - # Every question approved, and still no zip: the design spec builds only - # after validate returns clean, and the last line says what it found. - assert checked.message == "q1.text (lines 5-5) is empty." - assert result.zip_path is None - assert not (tmp_path / "out" / "set.zip").exists() - assert "build" not in [stage.name for stage in result.stages] - # And the run is not over: the review is there to answer again, and no run - # record claims a set was made. - assert (tmp_path / "cache" / "review.json").exists() - assert not (sheets / RECORD_NAME).exists() - - -def test_a_refused_build_is_a_stage_line_and_the_review_stays( - sheets, tmp_path, monkeypatch -): - reviewed(sheets, tmp_path) - # The checks pass and in2lambda still will not write the set out. - monkeypatch.setattr( - package, - "build", - lambda draft, out_dir: (_ for _ in ()).throw( - package.BuildRefused("figures/ball.png is not beside the draft") - ), - ) - - for key in ("q1", "q2"): - result = pipeline.resume( - tmp_path / "cache", verdict="approve", key=key, settings=Settings() - ) - build = [stage for stage in result.stages if stage.name == "build"][-1] - - assert build.message == "refused: figures/ball.png is not beside the draft" - assert result.zip_path is None - assert (tmp_path / "cache" / "review.json").exists() - - -def test_what_a_rejection_cost_is_in_the_run_record(sheets, tmp_path): - reviewed(sheets, tmp_path) - pipeline.resume( - tmp_path / "cache", - verdict="reject", - key="q1", - note="the greatest height is (a), not the stem", - settings=Settings(), - backend=FakeBackend([]), - ) - for key in ("q1", "q2"): - pipeline.resume( - tmp_path / "cache", verdict="approve", key=key, settings=Settings() - ) - - (line,) = (sheets / RECORD_NAME).read_text().splitlines() - recorded = json.loads(line) - - # The round the rejection caused is one of the run's rounds, and what it - # cost is in the run's total. - assert [one["round"] for one in recorded["rounds"]] == [1] - assert recorded["output_tokens"] > 0 - assert recorded["review"]["rejections"][0]["key"] == "q1" - - -def test_a_review_mode_none_builds_at_once(sheets, tmp_path): - result = reviewed(sheets, tmp_path, mode="none") - review = next(stage for stage in result.stages if stage.name == "review") - - assert review.message == "not asked for (mode none)" - assert result.review is None - assert result.zip_path.exists() - assert not (tmp_path / "cache" / "review.json").exists() - - -def test_the_set_is_written_where_the_run_was_told_to(sheets, tmp_path, monkeypatch): - working = tmp_path / "working" - working.mkdir() - monkeypatch.chdir(working) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(SPEC), - ) - - # The zip the build stage names is the one on disk, in the given directory. - assert result.zip_path == tmp_path / "out" / "set.zip" - assert result.zip_path.exists() - # The cache is the one thing the run writes under the working directory, - # since `--cache` defaults to a folder there, and it holds the copy of the - # set's other sheet each spec was run over. - assert [one.name for one in working.iterdir()] == [".in2lambda-agent"] - assert (working / ".in2lambda-agent" / "second" / "sheet-2.md").is_file() - - -def test_a_relative_out_dir_is_resolved_against_the_working_directory( - sheets, tmp_path, monkeypatch -): - monkeypatch.chdir(tmp_path) - - result = pipeline.run( - sheets / "sheet.md", - out_dir=Path("out"), - settings=Settings(), - backend=FakeBackend(SPEC), - ) - - assert result.zip_path == tmp_path / "out" / "set.zip" - assert result.zip_path.exists() - - -def test_a_markdown_source_needs_no_ocr(sheets, tmp_path): - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(SPEC), - ) - ocr = next(stage for stage in result.stages if stage.name == "ocr") - - assert ocr.message == "not needed for sheet.md" - - -def test_a_pdf_is_converted_and_the_run_carries_on_from_the_markdown(pdf, tmp_path): - client = FakeMathpix(markdown=SOURCE.read_text()) - - result = pipeline.run( - pdf, - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - mathpix=client, - backend=FakeBackend(SPEC), - ) - stages = {stage.name: stage.message for stage in result.stages} - - assert "fresh pass" in stages["ocr"] and "source.md" in stages["ocr"] - # The freeze read the OCR's markdown, not the PDF; the spec is kept with - # the PDF, which is the document set, rather than in the cache entry. - assert "cache" in stages["freeze"] - assert stages["spec"].startswith(f"wrote {tmp_path / SPEC_NAME}") - assert result.zip_path.exists() - assert len(client.calls) == 1 - - -def test_a_second_run_over_the_same_pdf_uses_the_cache(pdf, tmp_path): - client = FakeMathpix(markdown=SOURCE.read_text()) - backend = FakeBackend(SPEC) - for _ in range(2): - result = pipeline.run( - pdf, - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - mathpix=client, - backend=backend, - ) - ocr = next(stage for stage in result.stages if stage.name == "ocr") - - assert ocr.message.startswith("cached ") - assert len(client.calls) == 1 - assert len(backend.calls) == 1 - - -def test_a_cached_pdf_runs_with_no_mathpix_credentials(pdf, tmp_path, monkeypatch): - monkeypatch.delenv("MATHPIX_APP_ID", raising=False) - monkeypatch.delenv("MATHPIX_API_KEY", raising=False) - ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix(SOURCE.read_text())) - - # No client at all: the cache is what the run reads, and building one from - # empty settings would raise before it got there. - result = pipeline.run( - pdf, - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - backend=FakeBackend(SPEC), - ) - ocr = next(stage for stage in result.stages if stage.name == "ocr") - - assert ocr.message.startswith("cached ") - assert result.zip_path.exists() - - -def test_a_pdf_without_credentials_exits_one_naming_the_variables( - pdf, tmp_path, monkeypatch, capsys -): - monkeypatch.chdir(tmp_path) - monkeypatch.delenv("MATHPIX_APP_ID", raising=False) - monkeypatch.delenv("MATHPIX_API_KEY", raising=False) - - code = main([*RUN, str(pdf), "--out", str(tmp_path / "out")]) - printed = capsys.readouterr() - - assert code == 1 - assert "MATHPIX_APP_ID and MATHPIX_API_KEY" in printed.err - assert printed.out == "" - - -def test_the_command_exits_zero_and_prints_a_line_per_stage(sheets, tmp_path, capsys): - (sheets / SPEC_NAME).write_text(SPEC) - - code = main([*RUN, str(sheets / "sheet.md"), "--out", str(tmp_path / "out")]) - printed = capsys.readouterr().out.splitlines() - - assert code == 0 - assert [line.split()[0] for line in printed] == [ - "ocr", - "freeze", - "spec", - "coverage", - "validate", - "review", - "build", - ] - assert (tmp_path / "out" / "set.zip").exists() - - -def test_a_review_run_and_its_approvals_exit_zero_and_build(sheets, tmp_path, capsys): - (sheets / SPEC_NAME).write_text(SPEC) - where = ["--cache", str(tmp_path / "cache")] - out = ["--out", str(tmp_path / "out")] - - stopped = main([*RUN, str(sheets / "sheet.md"), "--review", "sample", *where, *out]) - printed = capsys.readouterr().out - - # Waiting for a reviewer is not a failure, and nothing is built yet. - assert stopped == 0 - assert "q1 pending" in printed and "q2 pending" in printed - assert not (tmp_path / "out" / "set.zip").exists() - - assert main(["review", "approve", "q1", *where]) == 0 - assert main(["review", "approve", "q2", *where]) == 0 - - assert (tmp_path / "out" / "set.zip").exists() - assert "build" in capsys.readouterr().out - - -def test_approving_a_draft_the_checks_fault_exits_one_saying_what_they_found( - sheets, tmp_path, capsys -): - (sheets / SPEC_NAME).write_text(SPEC) - where = ["--cache", str(tmp_path / "cache")] - out = ["--out", str(tmp_path / "out")] - - main([*RUN, str(sheets / "sheet.md"), "--review", "sample", *where, *out]) - main(["review", "edit", EMPTIES["field"], EMPTIES["old"], EMPTIES["new"], *where]) - capsys.readouterr() - - assert main(["review", "approve", "q1", *where]) == 0 - # Nothing left to answer and nothing built, which is a failure like any - # other build that did not happen. - assert main(["review", "approve", "q2", *where]) == 1 - assert "q1.text (lines 5-5) is empty." in capsys.readouterr().out - assert not (tmp_path / "out" / "set.zip").exists() - - -def test_a_review_command_with_no_review_waiting_exits_one(tmp_path, capsys): - code = main(["review", "approve", "q1", "--cache", str(tmp_path / "cache")]) - printed = capsys.readouterr() - - assert code == 1 - assert "No review is waiting" in printed.err - assert printed.out == "" - - -def test_the_default_out_is_the_working_directorys_out(sheets, tmp_path, monkeypatch): - (sheets / SPEC_NAME).write_text(SPEC) - monkeypatch.chdir(tmp_path) - - assert main([*RUN, str(sheets / "sheet.md")]) == 0 - assert (tmp_path / "out" / "set.zip").exists() - - -def test_a_run_the_checks_fault_prints_its_stages_and_exits_one( - sheets, tmp_path, capsys -): - (sheets / SPEC_NAME).write_text(PARTLESS_SPEC) - - code = main( - [*RUN, str(sheets / "sheet.md"), "--rounds", "0", "--out", str(tmp_path)] - ) - printed = capsys.readouterr() - - assert code == 1 - assert "is in no field and not marked ignore" in printed.out - assert not (tmp_path / "set.zip").exists() - - -def test_a_run_with_no_backend_exits_one_naming_what_to_do( - sheets, tmp_path, monkeypatch, capsys -): - monkeypatch.setattr( - pipeline, "choose_backend", lambda settings: FakeBackend(reason="run claude login") - ) - - code = main([*RUN, str(sheets / "sheet.md"), "--out", str(tmp_path / "out")]) - printed = capsys.readouterr() - - assert code == 1 - assert "run claude login" in printed.err - assert printed.out == "" - - -def test_a_build_in2lambda_refuses_ends_in_one_stage_line_with_no_zip( - figures, tmp_path, monkeypatch -): - out_dir = tmp_path / "out" - # The checks pass and in2lambda still will not write the set out. Stubbed - # because the refusals it has today — a missing image among them — are - # findings of the report now, which is a faulted draft and not this. - monkeypatch.setattr( - package, - "build", - lambda draft, out: (_ for _ in ()).throw( - package.BuildRefused("figures/ball.png is not beside the draft") - ), - ) - - result = pipeline.run( - figures / "figure.md", out_dir=out_dir, settings=Settings() - ) - build = result.stages[-1] - - assert build.name == "build" - assert build.message.startswith("refused: ") - assert "figures/ball.png" in build.message - assert result.zip_path is None - # The checks came clean and the export refused: no zip, but nothing faulted, - # which is what a run of many documents has to tell apart. - assert result.clean is True - # And why, without the stage line's prefix, for a table to be read on its own. - assert result.reason == build.message.removeprefix("refused: ") - assert "figures/ball.png" in result.reason - assert not list(out_dir.glob("*.zip")) - # The run still ends the way any other does, with its record beside the spec. - assert (figures / RECORD_NAME).is_file() - - -def test_a_refused_build_prints_its_stages_and_exits_one( - figures, tmp_path, capsys, monkeypatch -): - monkeypatch.setattr( - package, - "build", - lambda draft, out: (_ for _ in ()).throw( - package.BuildRefused("figures/ball.png is not beside the draft") - ), - ) - - code = main([*RUN, str(figures / "figure.md"), "--out", str(tmp_path / "out")]) - printed = capsys.readouterr() - - assert code == 1 - assert [line.split()[0] for line in printed.out.splitlines()] == [ - "ocr", - "freeze", - "spec", - "coverage", - "validate", - "review", - "build", - ] - assert "refused:" in printed.out and "figures/ball.png" in printed.out - assert printed.err == "" - assert not (tmp_path / "out" / "set.zip").exists() - - -def test_a_source_beside_its_figures_builds_with_the_images_in_media( - figures, tmp_path -): - result = pipeline.run( - figures / "figure.md", out_dir=tmp_path / "out", settings=Settings() - ) - - assert result.stages[-1].message == str(result.zip_path) - assert "media/ball.png" in zipfile.ZipFile(result.zip_path).namelist() - - -def test_a_sheet_and_its_solutions_are_frozen_into_one_draft(paired, tmp_path): - result = pipeline.run( - paired / "paired.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(PAIRED_SPEC), - ) - frozen = json.loads(result.draft.read_text()) - - assert result.draft == drafted(paired, "paired.md") - assert [one["source"] for one in frozen["sources"]] == [ - "paired.md", - "paired_solutions.md", - ] - # Every solution was quoted out of the second source, and every question's - # own text out of the first. - assert { - key: field.get("source", 1) - for key, field in frozen["fields"].items() - if not key.endswith(".ignore") - } == { - "q1.text": 1, - "q1.p1.text": 1, - "q1.p2.text": 1, - "q2.text": 1, - "q2.p1.text": 1, - "q2.p2.text": 1, - "q1.p1.solution": 2, - "q1.p2.solution": 2, - "q2.p1.solution": 2, - "q2.p2.solution": 2, - } - assert result.zip_path is not None and result.zip_path.exists() - - -def test_the_zip_quotes_the_part_solutions_from_the_solutions_file(paired, tmp_path): - result = pipeline.run( - paired / "paired.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(PAIRED_SPEC), - ) - - question = json.loads( - zipfile.ZipFile(result.zip_path).read("question_000_Question_1.json") - ) - assert [part["workedSolution"]["content"] for part in question["parts"]] == [ - "$\\omega = v / r$", - "$E = \\tfrac{3}{4} m v^2$", - ] - - -def test_a_marker_in_the_solutions_file_writes_no_question(marked, tmp_path): - result = pipeline.run( - marked / "marked.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(PAIRED_SPEC), - ) - fields = json.loads(result.draft.read_text())["fields"] - - # `Q1.` and `Q2.` match the `question` selector in the second source. The - # draft holds the two questions the first source states, and neither marker - # is a field. - assert sorted(key for key in fields if key.endswith(".text")) == [ - "q1.p1.text", - "q1.p2.text", - "q1.text", - "q2.p1.text", - "q2.p2.text", - "q2.text", - ] - # Each marker is ignored, so its text reaches no question of the set. - assert fields["2/b3.ignore"]["value"] is True - assert fields["2/b5.ignore"]["value"] is True - # Each marker sends the solutions below it to its own question, so question - # 2's solutions answer question 2's parts. Question 1's part (b) is - # unanswered, which is a warning the build runs after. - assert { - key: field["source"] for key, field in fields.items() if ".solution" in key - } == { - "q1.p1.solution": 2, - "q2.p1.solution": 2, - "q2.p2.solution": 2, - } - assert result.zip_path is not None and result.zip_path.exists() - - -def test_the_spec_prompt_shows_both_sources(paired, tmp_path): - backend = FakeBackend(PAIRED_SPEC) - - pipeline.run( - paired / "paired.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - (_, prompt) = backend.calls[0] - - assert "whose blocks are `2/b1` onwards" in prompt - assert "Source 1: paired.md" in prompt - assert "Source 2: paired_solutions.md" in prompt - assert "2/b3 5 1(a)" in prompt - - -def test_naming_the_solutions_file_runs_the_questions_file(paired, tmp_path): - result = pipeline.run( - paired / "paired_solutions.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(PAIRED_SPEC), - ) - freeze = next(stage for stage in result.stages if stage.name == "freeze") - - assert result.draft == drafted(paired, "paired.md") - assert freeze.message == ( - f"{drafted(paired, 'paired.md')}, with paired_solutions.md as source 2" - ) - # The record is the questions file's, so the two names are one run. - (line,) = (paired / RECORD_NAME).read_text().splitlines() - assert json.loads(line)["source"] == str(paired / "paired.md") - assert result.zip_path.exists() - - -def test_solutions_with_no_questions_beside_them_are_converted_alone(tmp_path): - folder = tmp_path / "lone" - folder.mkdir() - shutil.copy(FIXTURES / "lone_solutions.md", folder / "lone_solutions.md") - backend = FakeBackend(LONE_SPEC) - - result = pipeline.run( - folder / "lone_solutions.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=backend, - ) - stages = {stage.name: stage.message for stage in result.stages} - ((_, prompt),) = backend.calls - - assert stages["pair"] == ( - "no questions file named lone.md beside lone_solutions.md; " - "converting the solutions alone" - ) - # One source, so the freeze line names the draft and nothing else. - assert stages["freeze"] == str(drafted(folder, "lone_solutions.md")) - assert "This document holds solutions and no questions" in prompt - assert result.draft == drafted(folder, "lone_solutions.md") - # The run is the solutions file's, so the record names it. - (line,) = (folder / RECORD_NAME).read_text().splitlines() - assert json.loads(line)["source"] == str(folder / "lone_solutions.md") - # The marker above each solution is a question, and the paragraph under it - # is that question's solution. - zip_file = zipfile.ZipFile(result.zip_path) - assert [ - json.loads(zip_file.read(name))["parts"][0]["workedSolution"]["content"] - for name in ("question_000_Question_1.json", "question_001_Question_2.json") - ] == ["$\\omega = v / r$", "$T = 2\\pi\\sqrt{m/k}$"] - - -def test_a_pair_of_pdfs_is_converted_and_frozen_into_one_draft(tmp_path): - for name in ("paired.pdf", "paired_solutions.pdf"): - (tmp_path / name).write_bytes(f"%PDF-1.4 {name}".encode()) - client = PairedMathpix() - - result = pipeline.run( - tmp_path / "paired.pdf", - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - mathpix=client, - backend=FakeBackend(PAIRED_SPEC), - ) - frozen = json.loads(result.draft.read_text()) - - assert [one.name for one in client.calls] == [ - "paired.pdf", - "paired_solutions.pdf", - ] - # Each PDF has a cache entry of its own, and a draft holds the documents of - # one directory, so the solutions markdown is copied beside the questions. - assert [one["source"] for one in frozen["sources"]] == [ - "source.md", - "paired_solutions.md", - ] - # The images came with it, and under a name of their own: both conversions - # call their figure media/plot.png, and the set holds each of them. - assert result.zip_path.exists() - assert [ - one - for one in zipfile.ZipFile(result.zip_path).namelist() - if one.startswith("media/") - ] == ["media/plot.png", "media/question_000_Question_1_0001.png"] - - -def test_a_pdf_with_a_figure_builds_with_the_image_in_media(pdf, tmp_path): - # As the OCR leaves a converted PDF: the image under the cache entry's - # media/, and the markdown referring to it by that folder and its name. - markdown = (FIXTURES / "figure.md").read_text().replace("figures/ball", "media/plot") - - result = pipeline.run( - pdf, - out_dir=tmp_path / "out", - settings=Settings(), - cache_dir=tmp_path / "cache", - mathpix=FakeMathpix(markdown=markdown), - backend=FakeBackend(SPEC), - ) - - assert result.zip_path is not None - assert "media/plot.png" in zipfile.ZipFile(result.zip_path).namelist() - - -def test_on_stage_is_called_with_each_stage_of_a_run(sheets, tmp_path): - (sheets / SPEC_NAME).write_text(SPEC) - watched = [] - - result = pipeline.run( - sheets / "sheet.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(), - # The name and the message, rather than the stage itself: the run's own - # list holds those objects, so comparing the two lists of them would - # hold whatever was appended and prove nothing. - on_stage=lambda stage: watched.append((stage.name, stage.message)), - ) - - assert watched == [(stage.name, stage.message) for stage in result.stages] - assert watched[0][0] == "ocr" - - -def test_on_stage_is_called_with_the_stages_of_a_rejection(sheets, tmp_path): - reviewed(sheets, tmp_path) - watched = [] - - result = pipeline.resume( - tmp_path / "cache", - verdict="reject", - key="q2", - note="part (b) asks for the smallest coefficient", - settings=Settings(), - backend=FakeBackend( - [("field_replace", {"field": "q2.p2.text", "old": "least", "new": "small"})] - ), - on_stage=lambda stage: watched.append((stage.name, stage.message)), - ) - - assert watched == [(stage.name, stage.message) for stage in result.stages] - assert "fix" in [name for name, _ in watched] diff --git a/tests/test_review.py b/tests/test_review.py deleted file mode 100644 index 7fc36d2..0000000 --- a/tests/test_review.py +++ /dev/null @@ -1,196 +0,0 @@ -"""The record a run leaves for the reviewer, and which questions it shows them.""" - -import random -import re -from pathlib import Path - -import pytest - -from in2lambda_agent.fix import RoundResult -from in2lambda_agent.model import ToolCall, Usage -from in2lambda_agent.package import Coverage, QuestionInfo -from in2lambda_agent.review import Question, Review, ReviewError, choose -from in2lambda_agent.spec import Second, SpecTry - - -def infos(*layers): - """A draft's questions, numbered from 1, each at the layer given.""" - return { - f"q{number}": QuestionInfo(f"q{number}", layer, [[number, number]]) - for number, layer in enumerate(layers, start=1) - } - - -def review(**changed): - """A review of two questions, as a run in sample mode writes one.""" - state = dict( - mode="sample", - count=3, - source="sheets/sheet.md", - spec="sheets/in2lambda-spec.yaml", - out_dir="out", - limit=3, - draft="sheets/sheet.draft.json", - frozen="sheets/sheet.md", - reused=False, - coverage=Coverage(layout="PartsSepSol", blocks=14, fields={1: 10}, ignored=4), - questions=[ - Question("q1", pdf="out/render/q1.pdf", lines=[[5, 5], [7, 9]]), - Question("q2", lines=[[13, 13]]), - ], - ) - return Review(**{**state, **changed}) - - -def test_per_question_shows_every_question(): - assert choose(infos(1, 1, 3), "per-question", 3, random.Random(0)) == [ - "q1", - "q2", - "q3", - ] - - -def test_a_sample_shows_what_something_past_the_spec_wrote_first(): - chosen = choose(infos(1, 3, 1, 4, 1), "sample", 3, random.Random(0)) - - # The two the rounds or an edit touched, then one of the three the spec - # wrote by itself, to make the count up. - assert chosen[:2] == ["q2", "q4"] - assert len(chosen) == 3 and chosen[2] in {"q1", "q3", "q5"} - - -def test_a_sample_stops_at_the_count(): - assert choose(infos(3, 4, 3, 4), "sample", 2, random.Random(0)) == ["q1", "q2"] - - -def test_a_sample_of_a_set_smaller_than_the_count_is_all_of_it(): - assert sorted(choose(infos(1, 1), "sample", 5, random.Random(0))) == ["q1", "q2"] - - -def test_a_sample_is_the_same_sample_twice_from_the_same_seed(): - first = choose(infos(1, 1, 1, 1, 1), "sample", 2, random.Random(7)) - second = choose(infos(1, 1, 1, 1, 1), "sample", 2, random.Random(7)) - - assert first == second - - -def test_the_record_goes_to_json_and_comes_back(tmp_path): - saved = review( - usage=Usage(input_tokens=120, output_tokens=40, seconds=1.5), - tries=[ - SpecTry(0, Usage(), unassigned=2, errors=2, dropped=1), - SpecTry(1, Usage(input_tokens=120, output_tokens=40), second=0, chosen=True), - ], - rounds=[ - RoundResult(1, [ToolCall("part_add", {"question": "q2"}, "wrote")], Usage(), 0) - ], - ) - saved.questions[1].status = "rejected" - saved.questions[1].note = "the solution belongs to (b)" - - saved.save(tmp_path / "review.json") - read = Review.load(tmp_path / "review.json") - - assert read == saved - # The layers keep their numbers, which is how every other reader has them. - assert read.coverage.fields == {1: 10} - assert read.rounds[0].commands[0].name == "part_add" - # The iterations too, so that the record the last approval writes says what - # each spec the run wrote covered and cost. - assert [one.number for one in read.tries] == [0, 1] - assert read.tries[1].usage.input_tokens == 120 and read.tries[1].chosen is True - - -def test_the_other_document_of_the_set_goes_to_json_and_comes_back(tmp_path): - # A review with no other document of the set records None, which the record - # the last approval writes reads as a folder of one sheet. - assert Review.load(_written(review(), tmp_path)).second is None - - over = review(second=Second("sheet-2.md", path=Path("cache/second/sheet-2.md"))) - passed = review(second=Second("sheet-2.md", passed_over="an OCR call")) - - # The copy each spec was run over is not kept: the approval writes the - # record and runs no spec. - assert Review.load(_written(over, tmp_path)).second == Second("sheet-2.md") - assert Review.load(_written(passed, tmp_path)).second == Second( - "sheet-2.md", passed_over="an OCR call" - ) - - -def _written(waiting, tmp_path): - """The file a review was saved to, for a test that reads it back.""" - waiting.save(tmp_path / "review.json") - return tmp_path / "review.json" - - -def test_no_review_waiting_says_what_writes_one(tmp_path): - with pytest.raises(ReviewError, match="--review sample"): - Review.load(tmp_path / "review.json") - - -def test_a_record_cut_short_names_the_file_rather_than_breaking(tmp_path): - # A run killed while it was writing the record, which the next review - # command reads: the reviewer is told what to delete, not given a traceback. - (tmp_path / "review.json").write_text('{"mode": "sample", "questions": [') - - with pytest.raises(ReviewError, match=re.escape(str(tmp_path / "review.json"))): - Review.load(tmp_path / "review.json") - - -def test_a_record_an_older_agent_wrote_names_the_file_too(tmp_path): - (tmp_path / "review.json").write_text('{"mode": "sample", "count": 3}') - - with pytest.raises(ReviewError, match="not a review this run can read"): - Review.load(tmp_path / "review.json") - - -def test_a_question_that_is_not_under_review_names_the_ones_that_are(): - with pytest.raises(ReviewError, match="This review has q1, q2"): - review().question("q9") - - -def test_the_listing_names_the_pdf_the_source_and_the_lines(): - listed = review().listing().splitlines() - - assert listed[0] == ( - " q1 pending: out/render/q1.pdf, sheets/sheet.md lines 5-5, 7-9" - ) - assert listed[1] == " q2 pending: not rendered, sheets/sheet.md lines 13-13" - - -def test_the_listing_carries_a_rejection_note_back(): - one = review() - one.questions[0].status = "rejected" - one.questions[0].note = "part (b) is missing" - - assert one.listing().splitlines()[0].endswith("— part (b) is missing") - - -def test_a_review_is_done_only_once_every_question_is_approved(): - one = review() - one.questions[0].status = "approved" - - assert one.done is False - - one.questions[1].status = "approved" - - assert one.done is True - - -def test_the_record_line_carries_the_verdicts_and_the_notes(): - one = review() - one.questions[0].status = "approved" - one.questions[1].status = "rejected" - one.questions[1].note = "wrong solution" - one.rejections.append({"key": "q2", "note": "wrong solution"}) - one.edits.append({"field": "q1.text", "by": "ada"}) - - assert one.to_json() == { - "mode": "sample", - "questions": [ - {"key": "q1", "status": "approved", "note": None}, - {"key": "q2", "status": "rejected", "note": "wrong solution"}, - ], - "rejections": [{"key": "q2", "note": "wrong solution"}], - "edits": [{"field": "q1.text", "by": "ada"}], - } diff --git a/tests/test_spec.py b/tests/test_spec.py deleted file mode 100644 index 070bedd..0000000 --- a/tests/test_spec.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Layer 1: where a set's spec lives, and the one call that writes it.""" - -import json -import shutil -from pathlib import Path - -import pytest -from conftest import FakeBackend - -from in2lambda_agent.model import Usage -from in2lambda_agent.package import Coverage, Finding, Report -from in2lambda_agent.spec import ( - SPEC_NAME, - BadSpec, - Previous, - Second, - SpecTry, - iterate_spec, - record_run, - spec_path, - write_spec, -) - -FIXTURES = Path(__file__).parent / "fixtures" -SPEC = (FIXTURES / "sheet-spec.yaml").read_text() -IGNORES_THE_FIGURE = (FIXTURES / "figure-paragraph-spec.yaml").read_text() - - -def test_the_sets_spec_is_beside_the_source(tmp_path): - assert spec_path(tmp_path / "sheets" / "sheet.md") == ( - tmp_path / "sheets" / SPEC_NAME - ) - - -def test_every_sheet_in_one_folder_shares_a_spec(tmp_path): - first = spec_path(tmp_path / "sheet.md") - second = spec_path(tmp_path / "sheet-2.md") - - assert first == second - - -def test_a_named_spec_overrides_the_sets_own(tmp_path, monkeypatch): - monkeypatch.chdir(tmp_path) - - assert spec_path(Path("sheets/sheet.md"), Path("mine.yaml")) == ( - tmp_path / "mine.yaml" - ) - - -def test_the_prompt_carries_the_numbered_source_as_it_was_shown(): - backend = FakeBackend(SPEC) - shown = "b1 1 # Tutorial Sheet 1\n 2\nb2 3 A ball is thrown." - - text, reply = write_spec(shown, backend) - - (_, prompt), = backend.calls - assert shown in prompt - assert text == SPEC - assert reply.usage.output_tokens > 0 - - -def test_a_file_of_solutions_alone_is_said_to_hold_no_questions(): - backend = FakeBackend(SPEC, SPEC) - - write_spec("b1 1 ## Solutions", backend, solutions_only=True) - write_spec("b1 1 ## Question 1", backend) - - lone, paired = (prompt for _, prompt in backend.calls) - assert "This document holds solutions and no questions" in lone - assert "is the question here" in lone - assert "holds solutions and no questions" not in paired - - -def test_a_revision_carries_the_last_spec_and_what_running_it_covered(): - backend = FakeBackend(SPEC) - previous = Previous( - text="question: Para\nlayout: PartsSepSol\n", - coverage=Coverage( - layout="PartsSepSol", blocks=14, fields={1: 9}, ignored=4, - unassigned=["b4"], - ), - report=Report(clean=False, errors=["b4 (lines 7-7) is in no field."]), - second=Coverage(layout="PartsSepSol", blocks=9, unassigned=["b5", "b6"]), - second_name="sheet-2.md", - ) - - write_spec("b1 1 # Sheet", backend, previous) - - (_, prompt), = backend.calls - assert "question: Para\nlayout: PartsSepSol\n" in prompt - assert "b4 unassigned" in prompt - assert "b4 (lines 7-7) is in no field." in prompt - assert "over sheet-2.md, another document of this set, left b5, b6" in prompt - - -def test_a_revision_shows_the_images_the_last_spec_ignored(tmp_path): - backend = FakeBackend(SPEC) - previous = Previous( - text="question: Para\nlayout: PartsSepSol\n", - coverage=Coverage( - layout="PartsSepSol", - blocks=9, - fields={1: 5}, - ignored=4, - dropped=[ - Finding( - check="coverage", - level="error", - field="b4", - ranges=[[7, 8]], - message="b4 (lines 7-8) holds an image and is marked ignore.", - ) - ], - ), - report=Report(clean=True, errors=[]), - ) - - write_spec("b1 1 # Sheet", backend, previous) - - ((_, prompt),) = backend.calls - # Under the one heading as the checks' own errors: the next call answers an - # ignored figure the way it answers a block left in no field. - assert "The checks then found:\n\nb4 (lines 7-8) holds an image" in prompt - assert "fewer blocks unassigned, fewer images ignored" in prompt - - -def test_a_spec_that_ignores_a_figure_is_written_again_and_the_drop_reported(tmp_path): - # Every try marks the figure's paragraph ignored, so no try scores zero and - # the loop spends both its calls before keeping the first. - folder = tmp_path / "figure-paragraph" - (folder / "figures").mkdir(parents=True) - shutil.copy(FIXTURES / "figure-paragraph.md", folder / "figure-paragraph.md") - shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") - backend = FakeBackend(IGNORES_THE_FIGURE, IGNORES_THE_FIGURE) - stages: list[tuple[str, str]] = [] - - _, coverage, report, tries = iterate_spec( - folder / "figure-paragraph.md", - folder / SPEC_NAME, - backend, - tries=2, - on_stage=lambda name, message: stages.append((name, message)), - ) - - assert len(backend.calls) == 2 - # The checks find nothing in either draft, so the dropped image is the whole - # of the score. - assert [(one.unassigned, one.errors, one.dropped) for one in tries] == [ - (0, 0, 1), - (0, 0, 1), - ] - assert [one.score for one in tries] == [1, 1] - # The revision names the image the first try dropped. - assert "b4 (lines 7-8) holds an image and is marked ignore." in backend.calls[1][1] - # The run goes on past the drop, and every coverage line names it. - assert report.clean is True - assert str(coverage).endswith("; 1 image dropped: b4 (lines 7-8)") - assert [message for name, message in stages if name == "coverage"] == [ - str(coverage) - ] * 3 - - -def test_a_revision_of_a_spec_that_covered_the_set_says_so(): - backend = FakeBackend(SPEC) - previous = Previous( - text="question: Para\nlayout: PartsSepSol\n", - second=Coverage(layout="PartsSepSol", blocks=9), - second_name="sheet-2.md", - ) - - write_spec("b1 1 # Sheet", backend, previous) - - (_, prompt), = backend.calls - assert "left no blocks in no field" in prompt - - -def test_a_spec_in_a_code_fence_is_unwrapped(): - backend = FakeBackend(f"```yaml\n{SPEC}```\n") - - text, _ = write_spec("b1 1 # Sheet", backend) - - assert text == SPEC - - -def test_a_reply_that_is_not_a_mapping_is_refused(): - backend = FakeBackend("I am afraid I cannot write a spec for this.\n") - - with pytest.raises(BadSpec, match="a mapping of selectors"): - write_spec("b1 1 # Sheet", backend) - - -def test_a_reply_that_is_not_yaml_is_refused(): - backend = FakeBackend("question: Para\n layout: 'unclosed\n") - - with pytest.raises(BadSpec, match="not YAML"): - write_spec("b1 1 # Sheet", backend) - - -def test_a_spec_naming_no_layout_is_refused_by_name(): - backend = FakeBackend("question: Para\n") - - with pytest.raises(BadSpec, match="names no layout"): - write_spec("b1 1 # Sheet", backend) - - -def test_a_spec_naming_something_that_is_not_a_layout_is_refused_by_name(): - backend = FakeBackend("question: Para\nlayout: PartsThenSols\n") - - with pytest.raises(BadSpec, match="'PartsThenSols' is not a layout"): - write_spec("b1 1 # Sheet", backend) - - -def test_each_run_appends_one_line_saying_what_the_spec_covered(tmp_path): - record = tmp_path / "runs.jsonl" - coverage = Coverage( - layout="PartsSepSol", - blocks=14, - fields={1: 10}, - ignored=4, - unassigned=["b13"], - ) - - record_run( - record, - Path("sheet.md"), - reused=False, - coverage=coverage, - usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), - ) - record_run( - record, - Path("sheet-2.md"), - reused=True, - coverage=coverage, - usage=Usage(), - ) - - first, second = (json.loads(line) for line in record.read_text().splitlines()) - assert first["source"] == "sheet.md" - assert first["reused"] is False - assert first["layout"] == "PartsSepSol" - assert (first["blocks"], first["fields"], first["ignored"]) == (14, {"1": 10}, 4) - assert first["unassigned"] == ["b13"] - assert (first["input_tokens"], first["output_tokens"]) == (900, 80) - assert second["reused"] is True and second["output_tokens"] == 0 - # A run that reused the set's spec wrote none, so it iterated over nothing. - assert first["iterations"] == [] and second["iterations"] == [] - - -def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path): - record = tmp_path / "runs.jsonl" - coverage = Coverage(layout="PartsSepSol", blocks=14, fields={1: 10}) - - record_run( - record, - Path("sheet.md"), - reused=False, - coverage=coverage, - usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), - tries=[ - SpecTry(number=0, unassigned=2, errors=2, dropped=1), - SpecTry( - number=1, - usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), - unassigned=0, - errors=0, - second=1, - ), - SpecTry( - number=2, - usage=Usage(input_tokens=950, output_tokens=70, seconds=2.0), - chosen=True, - ), - ], - ) - - (line,) = record.read_text().splitlines() - saved, first, second = json.loads(line)["iterations"] - - # The saved spec the rewrite started from, which cost no call of its own. - assert saved == { - "try": 0, - "input_tokens": 0, - "output_tokens": 0, - "seconds": 0.0, - "unassigned": 2, - "errors": 2, - "dropped": 1, - "second": None, - "chosen": False, - } - assert (first["input_tokens"], first["output_tokens"]) == (900, 80) - assert (first["second"], first["chosen"]) == (1, False) - assert (second["try"], second["seconds"], second["chosen"]) == (2, 2.0, True) - - -def test_a_try_is_scored_on_its_dropped_images_as_well(): - one = SpecTry(number=1, unassigned=1, errors=2, dropped=3, second=4) - - assert one.score == 10 - assert SpecTry(number=1).score == 0 - - -def test_the_record_names_the_other_document_of_the_set(tmp_path): - record = tmp_path / "runs.jsonl" - coverage = Coverage(layout="PartsSepSol", blocks=14, fields={1: 10}) - - for second in ( - Second("sheet-2.md", path=tmp_path / "second" / "sheet-2.md"), - Second("sheet-2.pdf", passed_over="converting it takes an OCR call"), - None, - ): - record_run( - record, - Path("sheet.md"), - reused=False, - coverage=coverage, - usage=Usage(), - second=second, - ) - - over, passed, alone = [json.loads(line) for line in record.read_text().splitlines()] - - assert over["second"] == {"name": "sheet-2.md", "passed_over": None} - assert passed["second"] == { - "name": "sheet-2.pdf", - "passed_over": "converting it takes an OCR call", - } - # Null is the folder holding no other document, and nothing else. - assert alone["second"] is None diff --git a/tests/test_targets.py b/tests/test_targets.py index 5260623..f8c2800 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -327,6 +327,77 @@ def test_a_scanned_target_converts_with_no_filter(tmp_path, monkeypatch): assert not (tmp_path / "filters" / "ME2" / targets.FILTER_NAME).exists() +def test_a_replay_refuses_a_target_whose_filter_or_reply_is_not_saved( + tmp_path, monkeypatch +): + # A replay reads what is committed. Writing the missing file is a call that + # reads the document, which is the one thing a replay does not do, so the + # target is an error naming the file and the command that writes it. + calls = fake_convert(monkeypatch) + make_target(tmp_path / "corpus", "ME2") + (target,) = targets.find(tmp_path / "corpus") + filters = tmp_path / "filters" + ran = dict( + filters=filters, out_dir=tmp_path / "out", cache_dir=tmp_path / "cache", + backend=FakeBackend("-- filter"), replay=True, + ) + + no_reply = targets.run_one(target, **ran) + (filters / "ME2").mkdir(parents=True) + (filters / "ME2" / targets.REPLY_NAME).write_text(json.dumps(REPLY)) + no_filter = targets.run_one(target, **ran) + + assert targets.REPLY_NAME in no_reply.error + assert f"--filters {filters}" in no_reply.error + assert targets.FILTER_NAME in no_filter.error + assert calls == [] + + +def test_a_replay_of_a_saved_target_reports_its_differences_and_calls_nothing( + tmp_path, monkeypatch +): + calls = fake_convert(monkeypatch) + make_target(tmp_path / "corpus", "ME2") + (target,) = targets.find(tmp_path / "corpus") + filters = tmp_path / "filters" + (filters / "ME2").mkdir(parents=True) + (filters / "ME2" / targets.FILTER_NAME).write_text("-- filter") + (filters / "ME2" / targets.REPLY_NAME).write_text(json.dumps(REPLY)) + backend = FakeBackend() + + result = targets.run_one( + target, filters=filters, out_dir=tmp_path / "out", + cache_dir=tmp_path / "cache", backend=backend, replay=True, + ) + + assert result.error is None + assert result.new == result.differences and result.new + assert calls[0]["route_a"] == REPLY + assert backend.calls == [] + + +def test_a_replay_of_a_scanned_target_needs_only_the_reply(tmp_path, monkeypatch): + # There is no filter for a PDF target to save, so the reply is all a replay + # of one reads. + calls = fake_convert(monkeypatch) + make_target( + tmp_path / "corpus", "ME2", + questions="sheet.pdf", solutions="sheet_solutions.pdf", + ) + (target,) = targets.find(tmp_path / "corpus") + filters = tmp_path / "filters" + (filters / "ME2").mkdir(parents=True) + (filters / "ME2" / targets.REPLY_NAME).write_text(json.dumps(REPLY)) + + result = targets.run_one( + target, filters=filters, out_dir=tmp_path / "out", + cache_dir=tmp_path / "cache", backend=FakeBackend(), replay=True, + ) + + assert result.error is None + assert calls[0]["lua"] is None + + def test_a_target_whose_export_cannot_be_read_is_an_error_and_the_next_one_runs( tmp_path, monkeypatch ): diff --git a/tests/test_ui.py b/tests/test_ui.py index 4e0dcd6..b20a4d6 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -203,7 +203,7 @@ def test_a_file_the_process_may_not_read_is_one_line(client, root, monkeypatch): def refuse(path): raise PermissionError(f"[Errno 13] Permission denied: '{path}'") - monkeypatch.setattr(server.corpus, "is_document", refuse) + monkeypatch.setattr(server.pair, "is_document", refuse) answer = client.get(f"/api/sources?path={root / 'figures'}")