From 8f966e1f639fb7c416eb7344710687d13ebbb6b9 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 10:07:26 +0100 Subject: [PATCH 1/2] implement: Give a refused spec a second try with in2lambda's refusal in the prompt (t34) --- docs/how-it-works.md | 19 +++++--- in2lambda_agent/review.py | 4 ++ in2lambda_agent/spec.py | 76 ++++++++++++++++++++++++++--- tests/test_corpus.py | 4 +- tests/test_pipeline.py | 34 +++++++++++++ tests/test_review.py | 10 ++-- tests/test_spec.py | 100 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 227 insertions(+), 20 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index dcbcf77..611de9d 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -102,7 +102,7 @@ run wrote. ### `spec` -The stage prints one of three messages: +The stage prints one of four messages: * `reused /home/me/sheets/in2lambda-spec.yaml` — the spec file exists, and the stage makes no model call. A run that reuses a spec the checks then fault prints this stage @@ -112,16 +112,23 @@ The stage prints one of three messages: `agent-sdk`. The token count is the call's input and output tokens added together, and the time is the wall time of the call to one decimal place. The try number counts from 1 to `--tries`. +* `in2lambda refused the spec: ', Table' is not something a selector holds. A selector + is a block type followed by any number of name=value or name~'regex' constraints. + See line 1 of the spec. (try 1 of 3)` — in2lambda cannot read the spec the model + wrote. The refusal is in2lambda's own message, and it names the line. * `kept try 2 of 3` — the loop wrote more than one spec, and this names the try saved for the set: the one that scored lowest. A try's score adds up the blocks it left unassigned, the errors the checks then found, the images it dropped and the blocks it left unassigned in the set's other document. The loop writes one spec where the first scores zero, and prints no `kept` line. -in2lambda refuses a spec it cannot run, and the run raises `SpecRejected`. A spec this -run wrote is deleted before that refusal reaches the user, and a spec this run wrote -over an older one is replaced by the older one, so the next run over the set reads a -spec in2lambda accepts. +A spec in2lambda refused filled no draft, so it scores above every spec that ran and +the loop never keeps it. The refusal goes to the next call, which writes another spec, +and the run goes on with the tries it has left. The run raises `SpecRejected` where +in2lambda refused every spec the loop wrote, carrying what in2lambda said about the +last of them. A spec this run wrote is deleted before that refusal reaches the user, +and a spec this run wrote over an older one is replaced by the older one, so the next +run over the set reads a spec in2lambda accepts. ### `coverage` @@ -305,7 +312,7 @@ Each backend limits a call differently: | Call | What it is given | What it may write | | --- | --- | --- | -| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds, the images that spec dropped and the blocks that spec left in no field in the set's other document | `in2lambda-spec.yaml`, and nothing else | +| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds, the images that spec dropped and the blocks that spec left in no field in the set's other document, or in2lambda's refusal of that spec where in2lambda would not run it | `in2lambda-spec.yaml`, and nothing else | | Spec rewrite | the same, with the saved spec and what running it covered as the first call's try 0 | `in2lambda-spec.yaml`, and nothing else | | Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the eight draft commands, and nothing else | diff --git a/in2lambda_agent/review.py b/in2lambda_agent/review.py index 989aff8..ae39800 100644 --- a/in2lambda_agent/review.py +++ b/in2lambda_agent/review.py @@ -269,6 +269,7 @@ def _try_json(one: SpecTry) -> dict[str, Any]: "errors": one.errors, "dropped": one.dropped, "second": one.second, + "rejected": one.rejected, "chosen": one.chosen, } @@ -282,6 +283,9 @@ def _try_from(saved: dict[str, Any]) -> SpecTry: errors=saved["errors"], dropped=saved["dropped"], second=saved["second"], + # A review a run left waiting before the agent recorded a refused spec + # holds no `rejected` key, and its tries all ran. + rejected=saved.get("rejected"), chosen=saved["chosen"], ) diff --git a/in2lambda_agent/spec.py b/in2lambda_agent/spec.py index e0045c8..5419840 100644 --- a/in2lambda_agent/spec.py +++ b/in2lambda_agent/spec.py @@ -20,6 +20,7 @@ """ import json +import sys from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Optional, Sequence @@ -66,6 +67,16 @@ level, `text` is everything the block says, `label` its first word. Put single quotes round any regex with a backslash in it. +Several selectors for one role go under the role as a YAML list, one to a line, +never on one line with commas between them: + + ignore: + - Header + - Table + +The one comma a selector holds is the one after its `after` anchor, and a comma +inside a pattern goes inside the pattern's own quotes. + Every block is tried against ignore, then question, then part, then solution, whatever order the keys are written in, and is whatever the first of them says it is. So the selectors must not overlap: if the solutions are paragraphs and @@ -151,6 +162,9 @@ class SpecTry: — the record's `second` says why. A spec in2lambda refuses over that document wrote no field there, so it left every block of it in no field. + rejected: What in2lambda said where it refused the spec over this + source, and None where it ran the spec. Such a try filled no draft, + so `unassigned`, `errors` and `dropped` are 0 and `second` is None. chosen: Whether this is the spec the run saved and went on with. """ @@ -160,12 +174,16 @@ class SpecTry: errors: int = 0 dropped: int = 0 second: Optional[int] = None + rejected: Optional[str] = None chosen: bool = False @property def score(self) -> int: """What the tries are ranked by, the lowest winning. + A spec in2lambda refused wrote no field at all, so it scores above + every spec that ran, whatever that spec left over. + A block in no field is an error of the report as well as a line of the coverage, so it counts twice. That is the same double for every try and does not change the order they come in. @@ -174,6 +192,8 @@ def score(self) -> int: nothing about an ignored block, so a spec that ignores a figure is scored like one that leaves a block unassigned and is written again. """ + if self.rejected is not None: + return sys.maxsize return self.unassigned + self.errors + self.dropped + (self.second or 0) @@ -222,6 +242,9 @@ class Previous: second: What running it made of another document of the set, or None where the folder holds no other document. second_name: That document's file name. + rejected: What in2lambda said where it refused the spec over this + source, and None where it ran the spec. The coverage and the report + are then None, because the spec filled no draft. """ text: str @@ -229,6 +252,7 @@ class Previous: report: Optional[Report] = None second: Optional[Coverage] = None second_name: str = "" + rejected: Optional[str] = None def spec_path(source: Path, spec: Optional[Path] = None) -> Path: @@ -303,6 +327,8 @@ def write_spec( def _revision(previous: Previous) -> str: """The last spec and what running it covered, as the next call is shown them.""" said = [f"\nYour last spec for this set was:\n\n{previous.text}"] + if previous.rejected is not None: + said.append(f"\nin2lambda refused the spec:\n\n{previous.rejected}\n") if previous.coverage is not None: said.append(f"\nRunning it over this source covered:\n\n{previous.coverage}\n") # The images the spec dropped go in beside the report's errors, under the @@ -319,11 +345,17 @@ def _revision(previous: Previous) -> str: f"\nRunning it over {previous.second_name}, another document of this " f"set, left {left} in no field.\n" ) - said.append( - "\nWrite a spec that leaves fewer blocks unassigned, fewer images " - "ignored and fewer errors behind, over this source and over the rest " - "of the set.\n" - ) + if previous.rejected is not None: + said.append( + "\nWrite a spec in2lambda will run, over this source and over the " + "rest of the set.\n" + ) + else: + said.append( + "\nWrite a spec that leaves fewer blocks unassigned, fewer images " + "ignored and fewer errors behind, over this source and over the " + "rest of the set.\n" + ) return "".join(said) @@ -347,6 +379,10 @@ def iterate_spec( blocks the spec left over in another document of the set. The loop stops at a spec that scores zero, since a further call has nothing to improve. + A spec in2lambda refuses over this source is a failed try: the refusal names + the line and the fault, so the next call is shown it and writes another + spec. Such a try filled no draft and is never the spec the loop keeps. + Args: frozen: The markdown, tex or docx file each spec is run over. saved: The set's spec file, which every try writes and the chosen spec @@ -382,7 +418,8 @@ def iterate_spec( Raises: ModelError: a call did not finish. BadSpec: what the model answered with is not a spec. - SpecRejected: in2lambda will not run a spec this loop wrote. + SpecRejected: in2lambda refused every spec this loop wrote, carrying + what it said about the last of them. SourceError: in2lambda cannot freeze or check this source. Every error leaving this function puts the spec the set had before the loop @@ -418,6 +455,9 @@ def iterate_spec( # ran over it: the record is to say what became of the other document under # the spec the loop kept, not under a later try it threw away. best: Optional[tuple[SpecTry, str, Optional[str]]] = None + # What in2lambda said about the last spec it refused, which the run raises + # where it refused every one of them. + refusal = "" more = [solutions] if solutions is not None else [] try: for number in range(1, tries + 1): @@ -437,7 +477,24 @@ def iterate_spec( f"wrote {saved} via {reply.backend}, {tokens} tokens, " f"{reply.usage.seconds:.1f}s (try {number} of {tries})", ) - coverage, report = _run(draft, saved, on_stage) + try: + coverage, report = _run(draft, saved, on_stage) + except package.SpecRejected as error: + # in2lambda cannot read this spec and filled no draft with it, + # so there is no coverage and no report to score the try on. + # The refusal names the line and the fault, so the next call is + # shown it and the loop goes on with the tries it has left. + refusal = str(error) + on_stage( + "spec", + f"in2lambda refused the spec: {refusal} " + f"(try {number} of {tries})", + ) + made.append( + SpecTry(number=number, usage=reply.usage, rejected=refusal) + ) + previous = Previous(text=text, rejected=refusal) + continue over_second, left_over = _over_second(second, saved, on_stage) one = SpecTry( number=number, @@ -459,6 +516,10 @@ def iterate_spec( second=over_second, second_name=second.name if over_second is not None else "", ) + if best is None: + # Every try was refused, so the run has no spec to go on with and + # ends on the last refusal, as it ended on the first one before. + raise package.SpecRejected(refusal) except Exception: if replaced is None: saved.unlink(missing_ok=True) @@ -603,6 +664,7 @@ def record_run( "errors": one.errors, "dropped": one.dropped, "second": one.second, + "rejected": one.rejected, "chosen": one.chosen, } for one in tries diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 78a64cd..2850ac6 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -332,7 +332,9 @@ def refuse(draft, spec): monkeypatch.setattr(pipeline.package, "spec_run", refuse) - rows = sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC, TEX_SPEC)) + # One try each: a run with tries left writes another spec after a refusal, + # and the row a refusal makes is what this test is about. + rows = sweep(root, tmp_path, tries=1, backend=FakeBackend(SPEC, TEX_SPEC, TEX_SPEC)) rejected = [row for row in rows if row.set == "tex"] assert [row.outcome for row in rejected] == ["spec rejected"] * 2 diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 22ac5f1..860331b 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -422,6 +422,8 @@ def test_a_named_spec_that_is_not_there_yet_is_written_there(sheets, tmp_path): def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path): + # One try, and in2lambda refuses the spec it wrote: the loop has no try + # left to write another, so the refusal ends the run. backend = FakeBackend("question: NotAnElement\nlayout: PartsSepSol\n") with pytest.raises(SpecRejected, match="not a pandoc element"): @@ -429,6 +431,7 @@ def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path sheets / "sheet.md", out_dir=tmp_path / "out", settings=Settings(), + tries=1, backend=backend, ) @@ -442,6 +445,7 @@ def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path sheets / "sheet.md", out_dir=tmp_path / "out", settings=Settings(), + tries=1, backend=again, ) @@ -449,6 +453,35 @@ def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path assert result.zip_path.exists() +def test_a_refused_spec_is_written_again_and_the_run_builds_the_set(sheets, tmp_path): + # Two selectors on one line with a comma between them, which in2lambda + # reads as an `after` clause and refuses. The loop has a try left, so the + # refusal goes to the second call rather than ending the run. + backend = FakeBackend( + "ignore: Header, Table\nquestion: Para\nlayout: PartsSepSol\n", SPEC + ) + + result = pipeline.run( + sheets / "sheet.md", + out_dir=tmp_path / "out", + settings=Settings(), + tries=2, + backend=backend, + ) + + assert len(backend.calls) == 2 + assert "in2lambda refused the spec:" in backend.calls[1][1] + assert result.zip_path is not None and result.zip_path.exists() + assert (sheets / SPEC_NAME).read_text() == SPEC + + (line,) = (sheets / RECORD_NAME).read_text().splitlines() + refused, kept = json.loads(line)["iterations"] + + assert "holds no `after` clause" in refused["rejected"] + assert refused["chosen"] is False + assert (kept["rejected"], kept["chosen"]) == (None, True) + + def test_a_saved_spec_that_drops_an_image_is_written_again_and_the_run_goes_on( figure_paragraph, tmp_path ): @@ -491,6 +524,7 @@ def test_a_rewrite_in2lambda_will_not_run_leaves_the_saved_spec_alone( sheets / "sheet.md", out_dir=tmp_path / "out", settings=Settings(), + tries=1, backend=backend, ) diff --git a/tests/test_review.py b/tests/test_review.py index 7fc36d2..8b120b7 100644 --- a/tests/test_review.py +++ b/tests/test_review.py @@ -79,7 +79,8 @@ def test_the_record_goes_to_json_and_comes_back(tmp_path): usage=Usage(input_tokens=120, output_tokens=40, seconds=1.5), tries=[ SpecTry(0, Usage(), unassigned=2, errors=2, dropped=1), - SpecTry(1, Usage(input_tokens=120, output_tokens=40), second=0, chosen=True), + SpecTry(1, Usage(), rejected="'Header, Table' holds no `after` clause"), + SpecTry(2, Usage(input_tokens=120, output_tokens=40), second=0, chosen=True), ], rounds=[ RoundResult(1, [ToolCall("part_add", {"question": "q2"}, "wrote")], Usage(), 0) @@ -97,8 +98,11 @@ def test_the_record_goes_to_json_and_comes_back(tmp_path): assert read.rounds[0].commands[0].name == "part_add" # The iterations too, so that the record the last approval writes says what # each spec the run wrote covered and cost. - assert [one.number for one in read.tries] == [0, 1] - assert read.tries[1].usage.input_tokens == 120 and read.tries[1].chosen is True + assert [one.number for one in read.tries] == [0, 1, 2] + assert read.tries[2].usage.input_tokens == 120 and read.tries[2].chosen is True + # A spec in2lambda refused is a try of the run like any other, and what + # in2lambda said about it comes back with the rest. + assert read.tries[1].rejected == "'Header, Table' holds no `after` clause" def test_the_other_document_of_the_set_goes_to_json_and_comes_back(tmp_path): diff --git a/tests/test_spec.py b/tests/test_spec.py index 070bedd..082f1bd 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -8,7 +8,7 @@ from conftest import FakeBackend from in2lambda_agent.model import Usage -from in2lambda_agent.package import Coverage, Finding, Report +from in2lambda_agent.package import Coverage, Finding, Report, SpecRejected from in2lambda_agent.spec import ( SPEC_NAME, BadSpec, @@ -25,6 +25,12 @@ SPEC = (FIXTURES / "sheet-spec.yaml").read_text() IGNORES_THE_FIGURE = (FIXTURES / "figure-paragraph-spec.yaml").read_text() +# Two selectors written on one line with a comma between them, which in2lambda +# reads as an `after` clause and refuses. It is the shape three documents of +# the sweep ended on. +REFUSED_SPEC = "ignore: Header, Table\nquestion: Para\nlayout: PartsSepSol\n" +ALSO_REFUSED_SPEC = "ignore: Header\nquestion: Para, Table\nlayout: PartsSepSol\n" + def test_the_sets_spec_is_beside_the_source(tmp_path): assert spec_path(tmp_path / "sheets" / "sheet.md") == ( @@ -160,6 +166,83 @@ def test_a_spec_that_ignores_a_figure_is_written_again_and_the_drop_reported(tmp ] * 3 +def test_a_revision_of_a_refused_spec_carries_what_in2lambda_said(): + backend = FakeBackend(SPEC) + previous = Previous( + text="ignore: Header, Table\nquestion: Para\nlayout: PartsSepSol\n", + rejected="line 1: 'Header, Table' is not a pandoc element.", + ) + + write_spec("b1 1 # Sheet", backend, previous) + + ((_, prompt),) = backend.calls + assert "ignore: Header, Table\nquestion: Para\nlayout: PartsSepSol\n" in prompt + assert ( + "in2lambda refused the spec:\n\nline 1: 'Header, Table' is not a pandoc " + "element.\n" in prompt + ) + # A refused spec covered nothing, so the call is asked for one that runs + # rather than for one that leaves fewer blocks over. + assert "Write a spec in2lambda will run" in prompt + assert "fewer blocks unassigned" not in prompt + + +def a_sheet(tmp_path): + """A folder holding one sheet, for a loop to write the set's spec over.""" + folder = tmp_path / "sheets" + folder.mkdir() + shutil.copy(FIXTURES / "sheet.md", folder / "sheet.md") + return folder + + +def test_a_spec_in2lambda_refuses_is_a_try_and_the_next_call_is_shown_it(tmp_path): + folder = a_sheet(tmp_path) + backend = FakeBackend(REFUSED_SPEC, SPEC) + stages: list[tuple[str, str]] = [] + + _, _, report, tries = iterate_spec( + folder / "sheet.md", + folder / SPEC_NAME, + backend, + tries=2, + on_stage=lambda name, message: stages.append((name, message)), + ) + + assert len(backend.calls) == 2 + # The refusal names the line and the fault, and the spec it refused is in + # the prompt above it. + assert "in2lambda refused the spec:\n\nA selector's comma" in backend.calls[1][1] + assert REFUSED_SPEC in backend.calls[1][1] + assert "holds no `after` clause" in tries[0].rejected + assert tries[0].score > tries[1].score + assert [one.chosen for one in tries] == [False, True] + # The run goes on with the second spec, which in2lambda ran. + assert (folder / SPEC_NAME).read_text() == SPEC + assert report.clean is True + assert [message for name, message in stages if name == "spec"][1].startswith( + "in2lambda refused the spec: A selector's comma" + ) + + +def test_a_run_whose_every_spec_is_refused_stops_on_the_last_refusal(tmp_path): + folder = a_sheet(tmp_path) + backend = FakeBackend(REFUSED_SPEC, ALSO_REFUSED_SPEC) + + with pytest.raises(SpecRejected, match="'Para, Table' holds no `after` clause"): + iterate_spec( + folder / "sheet.md", + folder / SPEC_NAME, + backend, + tries=2, + on_stage=lambda name, message: None, + ) + + assert len(backend.calls) == 2 + # The set had no spec before the loop, and neither refused spec is left for + # the next run to read. + assert not (folder / SPEC_NAME).exists() + + def test_a_revision_of_a_spec_that_covered_the_set_says_so(): backend = FakeBackend(SPEC) previous = Previous( @@ -259,6 +342,7 @@ def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path) usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), tries=[ SpecTry(number=0, unassigned=2, errors=2, dropped=1), + SpecTry(number=3, rejected="line 1: 'Header, Table' is not an element."), SpecTry( number=1, usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), @@ -275,7 +359,7 @@ def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path) ) (line,) = record.read_text().splitlines() - saved, first, second = json.loads(line)["iterations"] + saved, refused, first, second = json.loads(line)["iterations"] # The saved spec the rewrite started from, which cost no call of its own. assert saved == { @@ -287,10 +371,14 @@ def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path) "errors": 2, "dropped": 1, "second": None, + "rejected": None, "chosen": False, } + # A spec in2lambda refused is a try like any other, and the line says what + # in2lambda said about it. + assert refused["rejected"] == "line 1: 'Header, Table' is not an element." assert (first["input_tokens"], first["output_tokens"]) == (900, 80) - assert (first["second"], first["chosen"]) == (1, False) + assert (first["second"], first["chosen"], first["rejected"]) == (1, False, None) assert (second["try"], second["seconds"], second["chosen"]) == (2, 2.0, True) @@ -301,6 +389,12 @@ def test_a_try_is_scored_on_its_dropped_images_as_well(): assert SpecTry(number=1).score == 0 +def test_a_refused_try_scores_above_every_try_that_ran(): + refused = SpecTry(number=1, rejected="line 1: 'Header, Table' is not an element.") + + assert refused.score > SpecTry(number=2, unassigned=99, errors=99).score + + def test_the_record_names_the_other_document_of_the_set(tmp_path): record = tmp_path / "runs.jsonl" coverage = Coverage(layout="PartsSepSol", blocks=14, fields={1: 10}) From 31df362a15e01dd991c4cebfe55155c0a6d5014e Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 10:59:05 +0100 Subject: [PATCH 2/2] implement: Give a refused spec a second try with in2lambda's refusal in the prompt (t34) --- docs/how-it-works.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 611de9d..f27f2f6 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -112,10 +112,10 @@ The stage prints one of four messages: `agent-sdk`. The token count is the call's input and output tokens added together, and the time is the wall time of the call to one decimal place. The try number counts from 1 to `--tries`. -* `in2lambda refused the spec: ', Table' is not something a selector holds. A selector - is a block type followed by any number of name=value or name~'regex' constraints. - See line 1 of the spec. (try 1 of 3)` — in2lambda cannot read the spec the model - wrote. The refusal is in2lambda's own message, and it names the line. +* ``in2lambda refused the spec: A selector's comma separates its `after` clause from + the rest, and 'Header, Table' holds no `after` clause. See line 1 of the spec. (try 1 + of 3)`` — in2lambda cannot read the spec the model wrote. The refusal is in2lambda's + own message, and it names the line. * `kept try 2 of 3` — the loop wrote more than one spec, and this names the try saved for the set: the one that scored lowest. A try's score adds up the blocks it left unassigned, the errors the checks then found, the images it dropped and the blocks