From 79bb3b0a059f291fcdc27f3b740a515aa052dd6e Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 02:04:52 +0100 Subject: [PATCH 1/2] implement: Replay each document's draft log in the corpus sweep (t27) --- README.md | 16 ++++++--- docs/how-it-works.md | 26 ++++++++++++--- in2lambda_agent/cli.py | 6 ++-- in2lambda_agent/corpus.py | 47 +++++++++++++++++++++----- in2lambda_agent/package.py | 42 +++++++++++++++++++++++ in2lambda_agent/pipeline.py | 12 +++++++ tests/test_corpus.py | 66 +++++++++++++++++++++++++++++++++++++ tests/test_docs.py | 2 +- tests/test_gate.py | 20 ++++++++++- tests/test_package.py | 51 ++++++++++++++++++++++++++++ tests/test_pipeline.py | 31 +++++++++++++++++ 11 files changed, 298 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 1f3a14c..100d714 100644 --- a/README.md +++ b/README.md @@ -267,11 +267,17 @@ with the reason `solutions without questions`. The sweep never writes to the corpus. It copies each set's folder into `--work` (default `./.in2lambda-agent/corpus`), empties that copy first, and runs the documents -there. It keeps the sets' specs in `--specs` (default `./corpus-specs`) in a tree -mirroring the corpus: set `A/B` keeps its spec at -`corpus-specs/A/B/in2lambda-spec.yaml`. So the copies are throwaway and the specs are -worth keeping — `--replay` reruns the saved specs and nothing else, making no model -call, which turns a document set into a deterministic test. +there. It keeps two kinds of file in `--specs` (default `./corpus-specs`), in a tree +mirroring the corpus: the set's spec, and each document's log of the commands its +fixing rounds ran. Set `A/B` keeps its spec at `corpus-specs/A/B/in2lambda-spec.yaml` +and the log of `A/B/sheet.tex` at `corpus-specs/A/B/sheet.tex.commands.json`. A log +holds the blocks, field keys and line ranges each command named, and none of the +document's text. + +So the copies are throwaway and the specs and the logs are worth keeping. `--replay` +runs the set's spec, then the document's log, and nothing from the model, which turns +a document set into a deterministic test: a document a fixing round repaired replays +to the set the sweep built. `--results` (default `./results.csv`) holds one row per document, sorted by path, with these columns: diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 234c956..2d88a40 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -26,9 +26,10 @@ review not asked for (mode none) build /home/me/sheets/out/set.zip ``` -There are nine stage names: `ocr`, `freeze`, `spec`, `coverage`, `validate`, `fix`, -`render`, `review` and `build`. A run prints `validate` once per check and `fix` once -per fixing round, so those two names repeat. +There are ten stage names: `ocr`, `freeze`, `spec`, `coverage`, `replay`, `validate`, +`fix`, `render`, `review` and `build`. A run prints `validate` once per check and `fix` +once per fixing round, so those two names repeat. A run prints `replay` only when it +was given a saved log to run, which is what a corpus replay is given. | Stage | in2lambda function | What the stage writes | | --- | --- | --- | @@ -36,6 +37,7 @@ per fixing round, so those two names repeat. | `freeze` | `in2lambda.source.add` | `SOURCE.draft.json`, beside the frozen source | | `spec` | `in2lambda.source.show`, for the model's prompt | `in2lambda-spec.yaml`, beside `SOURCE` or at `--spec` | | `coverage` | `in2lambda.draft.execute` with `in2lambda.draft.spec_command` | the layer 1 fields of the draft | +| `replay` | `in2lambda.draft.execute` once per saved command | the fields and the log of the draft | | `validate` | `in2lambda.draft.report.validate` | the report inside the draft | | `fix` | `in2lambda.source.show`, then `in2lambda.draft.execute` once per command | the fields and the log of the draft | | `render` | `in2lambda.draft.export.render`, which the agent does not call yet | `OUT/render/q1.pdf`, one PDF per question, once it does | @@ -105,6 +107,22 @@ stage prints `no fields` where the spec wrote none. `4 ignored` is the number of blocks the spec's `ignore` selector matched. The unassigned blocks are listed by id, and the stage prints `none unassigned` where every block reached a field. +### `replay` + +The stage runs the commands an earlier run's fixing rounds ran, read from the file the +run was given, in the order they were saved. It calls no model: the commands name the +blocks, field keys and line ranges each one wrote, so the draft the checks then see is +the draft the earlier run's rounds left. The message has one form: + +``` +5 commands from /home/me/corpus-specs/sheets/sheet.md.commands.json +``` + +in2lambda refusing one of the commands ends the run, and the refusal names which +command it was: `command 3 of 5, question add: b7b is in a field already`. The corpus +sweep writes these files and `corpus --replay` reads them; a `run` is given one through +`pipeline.run`. + ### `validate` `in2lambda.draft.report.validate` checks the draft and writes its report into the @@ -321,7 +339,7 @@ these 21, in this order: | --- | --- | | `source` | the document, relative to the corpus root | | `set` | the folder the document is in | -| `outcome` | `built`, `build refused`, `faulted`, `skipped`, `no spec`, `no model`, `spec rejected`, `bad spec`, or `error: ` | +| `outcome` | `built`, `build refused`, `faulted`, `skipped`, `no spec`, `replay refused` where in2lambda would not run one of the saved commands, `no model`, `spec rejected`, `bad spec`, or `error: ` | | `reason` | the build's refusal, the first error the checks still found, the warnings a build proceeded past, or what an exception said | | `spec` | `wrote`, `reused`, or `rewritten` where the spec rewrite ran | | `layout` | the coverage's layout | diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index f83bd6b..e203405 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -180,7 +180,8 @@ def build_parser() -> argparse.ArgumentParser: sweep.add_argument( "--replay", action="store_true", - help="Run the saved specs and nothing else, making no model call.", + help="Run the saved specs and the documents' saved logs, making no " + "model call.", ) sweep.add_argument( "--rounds", @@ -205,7 +206,8 @@ def build_parser() -> argparse.ArgumentParser: "--specs", type=Path, default=corpus.DEFAULT_SPEC_DIR, - help="The tree the sets' specs are kept in, mirroring the corpus.", + help="The tree the sets' specs are kept in, mirroring the corpus, with " + "each document's log of its fixing rounds beside its set's spec.", ) sweep.add_argument( "--cache", diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 881036b..e0eef87 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -12,12 +12,15 @@ Nothing here writes into the corpus. A run leaves a draft beside its source, and a spec and a record beside that, so each set's folder is copied into a work directory and run there, and the specs are kept in a tree of their own -mirroring the corpus. The copy is thrown away and made again every run; the spec -is what survives, and is what makes a document replayable — a saved spec plus -its source rebuilt with no model call at all. +mirroring the corpus. The work directory is deleted and written again every +sweep. A sweep keeps two files per set in the spec tree: the set's spec, and +each document's log of the commands its fixing rounds ran. A replay runs the +spec, then the document's log, and makes no model call, so a document a round +repaired replays to the set the sweep built. """ import csv +import json import shutil import time from dataclasses import asdict, dataclass, fields @@ -44,6 +47,11 @@ """The mirror tree the sets' specs are kept in, which the work directory being wiped does not touch.""" +COMMANDS_SUFFIX = ".commands.json" +"""What a document's saved log is named with, beside its set's spec: the +commands the fixing rounds ran, each with the blocks, field keys and line ranges +it named and none of the document's text.""" + ROOT_SET = "_root" """What the corpus root's own documents are staged under. They are a set like any other, but the set's folder under the work directory would be the work @@ -62,7 +70,9 @@ class Row: the checks still fault and no zip, `skipped` for a file that is not a document and for a solutions document with no questions document beside it, `no spec` for a replay with nothing saved to replay, - `no model`, `spec rejected`, `bad spec`, or `error: `. + `replay refused` where in2lambda would not run one of the saved + commands, `no model`, `spec rejected`, `bad spec`, or + `error: `. reason: What the run had to say for itself, in the words of whatever said it: the refusal, the first error the checks were still finding, or what the exception said. On a `built` row it holds the warnings @@ -266,10 +276,12 @@ def run_one( name: What to call it in the table, relative to the corpus root. set_name: Its set, relative to the corpus root. spec: The set's spec, in the mirror tree: read if it is there, written - if it is not. + if it is not. The document's log is written beside it, named after + the document. settings: The environment the run has available. rounds: The round limit, ignored in a replay, which can run none. - replay: Run the saved spec and nothing else, making no model call. + replay: Run the saved spec and the document's saved log, making no + model call. cache: Where the OCR of each PDF is kept. backend: The backend to write a spec with, chosen from the settings if absent. @@ -278,6 +290,9 @@ def run_one( The document's row. """ existed = Path(spec).is_file() + # The log is one document's and the spec is the set's, so the log is named + # after the document, beside the spec. + commands = Path(spec).parent / f"{source.name}{COMMANDS_SUFFIX}" row = Row(source=name, set=set_name) started = time.monotonic() try: @@ -286,6 +301,9 @@ def run_one( out_dir=source.parent / "out", settings=settings, spec=spec, + # The log of an earlier sweep's fixing rounds, which is what a + # replay has in place of the rounds. A sweep writes it below. + commands=commands if replay else None, review=row.review, # A replay has no model to run a round with, so it stops at the # report: the row then says what the saved spec left rather than @@ -297,6 +315,9 @@ def run_one( except ModelUnavailable as error: row.outcome = "no model" if existed else "no spec" row.reason = _one_line(str(error)) + except package.CommandRefused as error: + row.outcome = "replay refused" + row.reason = _one_line(str(error)) except SpecRejected as error: row.outcome = "spec rejected" row.reason = _one_line(str(error)) @@ -326,6 +347,14 @@ def run_one( row.blocks = result.coverage.blocks row.unassigned = len(result.coverage.unassigned) if result.draft is not None: + if not replay: + # What a later replay runs in place of the fixing rounds. A run + # that took no round writes an empty list, so that a replay + # reading no file there knows the document was never swept. + commands.write_text( + json.dumps(package.fix_log(result.draft), indent=1), + encoding="utf-8", + ) # The counts are keyed by the column names they fill, and `fields` # is their total, since a round writes fields the spec run's own # count knows nothing about. @@ -381,8 +410,10 @@ def sweep( suffixes: The file suffixes that are documents. results: Where to write the table. work: Where each set's folder is copied to be run. - specs: The tree the sets' specs are kept in, mirroring the corpus. - replay: Run the saved specs and nothing else, making no model call. + specs: The tree the sets' specs are kept in, mirroring the corpus, with + each document's log of its fixing rounds beside its set's spec. + replay: Run the saved specs and logs and nothing else, making no model + call. rounds: The round limit each run is given. cache: Where the OCR of each PDF is kept, so that a sweep pointed at a cache another run filled converts nothing. diff --git a/in2lambda_agent/package.py b/in2lambda_agent/package.py index 3fcd69f..ed0c4c0 100644 --- a/in2lambda_agent/package.py +++ b/in2lambda_agent/package.py @@ -316,6 +316,48 @@ def command_log(draft: Path) -> list[dict[str, Any]]: return _frozen(draft)["log"] +def fix_log(draft: Path) -> list[dict[str, Any]]: + """The commands the fixing rounds ran, without the spec run before them. + + A replay runs the set's spec itself and then these, so the `spec run` entry + is left out: it names the spec file relative to the draft, and the replay's + draft is in the work directory rather than where the sweep's draft was. + + Args: + draft: The draft file. + + Returns: + One entry per command, each `{"command", "args", "by"}`, in the order + they ran. The entries hold block ids, field keys and line ranges, and no + field's text. + """ + return [one for one in command_log(draft) if one["command"] != "spec run"] + + +def replay(draft: Path, commands: list[dict[str, Any]]) -> int: + """Runs a saved list of commands over a draft, in the order they were saved. + + Args: + draft: The draft file, with the set's spec already run over it. + commands: The entries `fix_log` wrote. + + Returns: + How many commands ran. + + Raises: + CommandRefused: in2lambda would not run one of them. The message names + which command it was, and the commands before it have been applied. + """ + for index, entry in enumerate(commands, start=1): + try: + command(draft, entry["command"], entry["args"], entry.get("by", BY)) + except CommandRefused as error: + raise CommandRefused( + f"command {index} of {len(commands)}, {entry['command']}: {error}" + ) from None + return len(commands) + + def questions(draft: Path) -> dict[str, QuestionInfo]: """The questions a draft holds, in the order they are numbered. diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index d57f05f..b02e65e 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -25,6 +25,7 @@ reviewer was shown has been approved. """ +import json import random import re import shutil @@ -114,6 +115,7 @@ def run( out_dir: Path, settings: Settings, spec: Optional[Path] = None, + commands: Optional[Path] = None, review: str = "none", rounds: int = 3, sample: int = 3, @@ -132,6 +134,10 @@ def run( out_dir: Where in2lambda writes the set's JSON folder and zip. settings: The environment the run has available. spec: The set's spec file, when it is not the one beside the source. + commands: A file holding the commands an earlier run's fixing rounds + ran, as `package.fix_log` wrote them. They are run after the spec, + which is how a replay rebuilds a draft the rounds repaired without + calling a model. A path that is not a file runs the spec alone. review: One of REVIEW_MODES. rounds: The round limit, N in the design spec: how many model calls may answer what the checks found before the run stops without a zip. @@ -160,6 +166,7 @@ def run( ModelUnavailable: If a spec must be written and no backend can run. BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. + CommandRefused: If in2lambda will not run one of the saved commands. SourceError: If in2lambda cannot freeze or check the source. SolutionsWithoutQuestions: If `source` is a solutions file and no questions file is beside it. @@ -266,6 +273,11 @@ def run( raise result.add_stage("coverage", str(result.coverage)) + if commands is not None and Path(commands).is_file(): + saved_commands = json.loads(Path(commands).read_text(encoding="utf-8")) + ran = package.replay(draft, saved_commands) + result.add_stage("replay", f"{ran} commands from {commands}") + report = package.validate(draft) if report.clean: result.add_stage("validate", _said(report)) diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 2124397..1cb7630 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -136,6 +136,72 @@ def test_a_saved_spec_and_its_source_replay_with_no_model_call(root, tmp_path): ] +def test_a_sweep_keeps_each_documents_log_beside_the_sets_spec(tmp_path): + root = tmp_path / "corpus" + make_set(root, "faulty", ["faulty.md"]) + + sweep(root, tmp_path, backend=FakeBackend(FAULTY_SPEC, FIXES)) + saved = tmp_path / "specs" / "faulty" / f"faulty.md{corpus.COMMANDS_SUFFIX}" + + # The round's commands, and not the spec run before them: a replay runs the + # set's spec itself. + assert [one["command"] for one in json.loads(saved.read_text())] == [ + "split block", + "question add", + "part add", + "question solution", + "field replace", + ] + # Each command names blocks, field keys and ranges. The document's own text + # stays in the corpus. + assert "A ball is thrown straight up" not in saved.read_text() + + +def test_a_document_that_took_no_round_keeps_an_empty_log(root, tmp_path): + sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC)) + saved = tmp_path / "specs" / "sheets" / f"sheet.md{corpus.COMMANDS_SUFFIX}" + + # Written all the same: a replay that finds no file there reads it as a + # document no sweep has run. + assert json.loads(saved.read_text()) == [] + + +def test_a_replay_runs_the_saved_log_and_builds_what_the_rounds_repaired( + tmp_path, monkeypatch +): + root = tmp_path / "corpus" + make_set(root, "faulty", ["faulty.md"]) + (swept,) = sweep(root, tmp_path, backend=FakeBackend(FAULTY_SPEC, FIXES)) + monkeypatch.setattr( + pipeline, + "choose_backend", + lambda settings: pytest.fail("a replay chose a backend"), + ) + + (row,) = sweep(root, tmp_path, replay=True, backend=None) + + # The sweep took a round to build this document, and the replay builds it + # with the same fields and no model call at all. + assert (swept.outcome, row.outcome) == ("built", "built") + assert (row.rounds, row.input_tokens, row.output_tokens) == (0, 0, 0) + assert (row.layer3, row.edited) == (swept.layer3, swept.edited) == (3, 1) + + +def test_a_log_in2lambda_refuses_names_the_command_in_the_rows_reason(tmp_path): + root = tmp_path / "corpus" + make_set(root, "faulty", ["faulty.md"]) + sweep(root, tmp_path, backend=FakeBackend(FAULTY_SPEC, FIXES)) + saved = tmp_path / "specs" / "faulty" / f"faulty.md{corpus.COMMANDS_SUFFIX}" + written = json.loads(saved.read_text()) + written[0]["args"]["block"] = "b99" + saved.write_text(json.dumps(written)) + + (row,) = sweep(root, tmp_path, replay=True, backend=None) + + assert row.outcome == "replay refused" + assert "command 1 of 5, split block" in row.reason + + def test_a_replay_with_nothing_saved_says_so_and_still_makes_no_call(root, tmp_path): backend = FakeBackend(reason="a replay makes no call") diff --git a/tests/test_docs.py b/tests/test_docs.py index 35b43e0..cc980fd 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -57,6 +57,6 @@ def test_readme_names_every_option(): def test_how_it_works_names_every_stage(): names = _stage_names() - assert len(names) == 9 + assert len(names) == 10 missing = [one for one in sorted(names) if f"`{one}`" not in HOW_IT_WORKS] assert not missing diff --git a/tests/test_gate.py b/tests/test_gate.py index 6dd7921..55e2807 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -1,12 +1,13 @@ """The merge gate: a replay over a corpus, checked against a recorded baseline.""" +import json from pathlib import Path import pytest from test_corpus import make_set from test_pipeline import SPEC, TEX_SPEC -from in2lambda_agent import gate +from in2lambda_agent import corpus, gate from in2lambda_agent.gate import Baseline, Folder from in2lambda_agent.settings import Settings from in2lambda_agent.spec import SPEC_NAME @@ -74,6 +75,23 @@ def test_a_recorded_baseline_passes_the_run_that_recorded_it(baseline, tmp_path) assert [one.built for one in report.folders.values()] == [2, 2] +def test_the_gate_replays_the_logs_kept_beside_the_specs(baseline, tmp_path): + # A log in2lambda refuses, so that the outcome says whether the gate ran it. + saved = tmp_path / "specs" / "tex" / f"tex-sheet.tex{corpus.COMMANDS_SUFFIX}" + saved.write_text( + json.dumps([{"command": "mark ignore", "args": {"block": "b99"}, "by": "ada"}]) + ) + + run(baseline, tmp_path, record=True) + + # The gate reads the specs from its own copy of the tree, and the logs are + # copied with them. + copied = tmp_path / "gate" / "specs" / "tex" / saved.name + assert copied.is_file() + assert baseline.folders["tex"].documents["tex/tex-sheet.tex"] == "replay refused" + assert baseline.folders["tex"].documents["tex/tex-sheet-2.tex"] == "built" + + def test_a_folder_that_builds_fewer_than_recorded_fails(baseline, tmp_path): run(baseline, tmp_path, record=True) baseline.folders["tex"].built += 1 diff --git a/tests/test_package.py b/tests/test_package.py index 64ed67f..87a2ab6 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -77,6 +77,57 @@ def test_a_reviewers_edit_is_logged_under_their_name(draft): assert "stone" in fields["q1.text"]["value"] +def test_the_fix_log_is_the_commands_after_the_spec_run(draft): + package.command( + draft, "field replace", {"field": "q1.text", "old": "ball", "new": "stone"} + ) + + assert package.command_log(draft)[0]["command"] == "spec run" + assert [one["command"] for one in package.fix_log(draft)] == ["field replace"] + + +def test_a_saved_log_replays_in_the_order_it_was_saved(draft, tmp_path): + saved = [ + { + "command": "field replace", + "args": {"field": "q1.text", "old": "ball", "new": "stone"}, + "by": package.BY, + }, + { + "command": "field replace", + "args": {"field": "q1.text", "old": "stone", "new": "brick"}, + "by": package.BY, + }, + ] + again = package.source_add(package.frozen_source(draft)) + package.spec_run(again, FIXTURES / "sheet-spec.yaml") + + assert package.replay(again, saved) == 2 + assert "brick" in package.field_value(again, "q1.text") + + +def test_a_refused_command_names_its_place_in_the_log(draft): + saved = [ + { + "command": "field replace", + "args": {"field": "q1.text", "old": "ball", "new": "stone"}, + "by": package.BY, + }, + {"command": "mark ignore", "args": {"block": "b99"}, "by": package.BY}, + { + "command": "field replace", + "args": {"field": "q1.text", "old": "stone", "new": "brick"}, + "by": package.BY, + }, + ] + + with pytest.raises(package.CommandRefused, match="command 2 of 3, mark ignore"): + package.replay(draft, saved) + + # The command before the refusal was applied, and the one after it was not. + assert "stone" in package.field_value(draft, "q1.text") + + def test_a_part_with_no_solution_is_a_warning_the_report_is_still_clean_for( tmp_path, ): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d07639c..8b7a9f6 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -540,6 +540,37 @@ def test_every_fix_is_in_the_drafts_log_with_the_layer_it_wrote(faulty, tmp_path assert fields["q1.text"]["layer"] == 1 and not fields["q1.text"]["edited"] +def test_a_saved_log_is_run_after_the_spec_and_builds_with_no_call(faulty, tmp_path): + pipeline.run( + faulty / "faulty.md", + out_dir=tmp_path / "out", + settings=Settings(), + backend=FakeBackend(FAULTY_SPEC, FIXES), + ) + saved = tmp_path / "faulty.md.commands.json" + saved.write_text(json.dumps(package.fix_log(drafted(faulty, "faulty.md")))) + backend = FakeBackend(reason="a replay makes no call") + + result = pipeline.run( + faulty / "faulty.md", + out_dir=tmp_path / "again", + settings=Settings(), + commands=saved, + rounds=0, + backend=backend, + ) + + # The commands run between the spec and the checks, so the draft the checks + # see is the one the first run's rounds repaired. + names = [one.name for one in result.stages] + assert names[names.index("coverage") + 1] == "replay" + assert names[names.index("replay") + 1] == "validate" + assert result.stages[names.index("replay")].message.startswith("5 commands from") + assert backend.calls == [] + assert result.rounds == [] + assert result.zip_path is not None and result.zip_path.exists() + + def test_the_record_says_what_each_round_cost(faulty, tmp_path): pipeline.run( faulty / "faulty.md", From aeb961695294a69eb184ae2c228ef21e868a140d Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 04:08:23 +0100 Subject: [PATCH 2/2] implement: Replay each document's draft log in the corpus sweep (t27) --- README.md | 14 ++++++++------ in2lambda_agent/corpus.py | 14 ++++++++------ in2lambda_agent/package.py | 6 ++++-- tests/test_corpus.py | 10 ++++++---- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 100d714..408a5a1 100644 --- a/README.md +++ b/README.md @@ -267,12 +267,14 @@ with the reason `solutions without questions`. The sweep never writes to the corpus. It copies each set's folder into `--work` (default `./.in2lambda-agent/corpus`), empties that copy first, and runs the documents -there. It keeps two kinds of file in `--specs` (default `./corpus-specs`), in a tree -mirroring the corpus: the set's spec, and each document's log of the commands its -fixing rounds ran. Set `A/B` keeps its spec at `corpus-specs/A/B/in2lambda-spec.yaml` -and the log of `A/B/sheet.tex` at `corpus-specs/A/B/sheet.tex.commands.json`. A log -holds the blocks, field keys and line ranges each command named, and none of the -document's text. +there. It keeps three kinds of file in `--specs` (default `./corpus-specs`), in a tree +mirroring the corpus: the set's spec, each document's log of the commands its fixing +rounds ran, and the `in2lambda-agent-runs.jsonl` every run appends a line to. Set `A/B` +keeps its spec at `corpus-specs/A/B/in2lambda-spec.yaml` and the log of `A/B/sheet.tex` +at `corpus-specs/A/B/sheet.tex.commands.json`. A log entry holds the block ids, field +keys and line ranges its command named, and the wording a `field replace` or a typed +field spells out. It does not hold the draft's fields, which hold every field's +captured text. So the copies are throwaway and the specs and the logs are worth keeping. `--replay` runs the set's spec, then the document's log, and nothing from the model, which turns diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index e0eef87..53db5e6 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -13,10 +13,10 @@ and a spec and a record beside that, so each set's folder is copied into a work directory and run there, and the specs are kept in a tree of their own mirroring the corpus. The work directory is deleted and written again every -sweep. A sweep keeps two files per set in the spec tree: the set's spec, and -each document's log of the commands its fixing rounds ran. A replay runs the -spec, then the document's log, and makes no model call, so a document a round -repaired replays to the set the sweep built. +sweep. A sweep keeps in the spec tree the set's spec, each document's log of +the commands its fixing rounds ran, and the record every run appends a line to. +A replay runs the spec, then the document's log, and makes no model call, so a +document a round repaired replays to the set the sweep built. """ import csv @@ -49,8 +49,10 @@ COMMANDS_SUFFIX = ".commands.json" """What a document's saved log is named with, beside its set's spec: the -commands the fixing rounds ran, each with the blocks, field keys and line ranges -it named and none of the document's text.""" +commands the fixing rounds ran, each with the block ids, field keys and line +ranges it named, and the wording a `field replace` or a typed field spells out. +The draft's `fields`, which hold every field's captured text, stay in the work +directory.""" ROOT_SET = "_root" """What the corpus root's own documents are staged under. They are a set like diff --git a/in2lambda_agent/package.py b/in2lambda_agent/package.py index ed0c4c0..b3299a3 100644 --- a/in2lambda_agent/package.py +++ b/in2lambda_agent/package.py @@ -328,8 +328,10 @@ def fix_log(draft: Path) -> list[dict[str, Any]]: Returns: One entry per command, each `{"command", "args", "by"}`, in the order - they ran. The entries hold block ids, field keys and line ranges, and no - field's text. + they ran. An entry's arguments are the block ids, field keys and line + ranges the command named, together with the wording a `field replace` + or a typed field spells out. The draft's `fields`, which hold every + field's captured text, are not here. """ return [one for one in command_log(draft) if one["command"] != "spec run"] diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 1cb7630..fed41ca 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -9,7 +9,7 @@ from conftest import FakeBackend from test_pipeline import FAULTY_SPEC, FIXES, PAIRED_SPEC, SPEC, TEX_SPEC -from in2lambda_agent import corpus, pipeline +from in2lambda_agent import corpus, package, pipeline from in2lambda_agent.settings import Settings from in2lambda_agent.spec import SPEC_NAME @@ -142,9 +142,11 @@ def test_a_sweep_keeps_each_documents_log_beside_the_sets_spec(tmp_path): sweep(root, tmp_path, backend=FakeBackend(FAULTY_SPEC, FIXES)) saved = tmp_path / "specs" / "faulty" / f"faulty.md{corpus.COMMANDS_SUFFIX}" + draft = package.draft_of(tmp_path / "work" / "faulty" / "faulty.md") # The round's commands, and not the spec run before them: a replay runs the # set's spec itself. + assert json.loads(saved.read_text()) == package.fix_log(draft) assert [one["command"] for one in json.loads(saved.read_text())] == [ "split block", "question add", @@ -152,9 +154,9 @@ def test_a_sweep_keeps_each_documents_log_beside_the_sets_spec(tmp_path): "question solution", "field replace", ] - # Each command names blocks, field keys and ranges. The document's own text - # stays in the corpus. - assert "A ball is thrown straight up" not in saved.read_text() + # The log and nothing beside it: the draft's `fields`, which hold every + # field's captured text, stay in the work directory. + assert "fields" not in saved.read_text() def test_a_document_that_took_no_round_keeps_an_empty_log(root, tmp_path):