Give a refused spec a second try with in2lambda's refusal in the prompt

docs/how-it-works.md

@@ -102,7 +102,7 @@ run wrote.
 
 ### `spec`
 
-The stage prints one of three messages:
+The stage prints one of four messages:
 
 * `reused /home/me/sheets/in2lambda-spec.yaml` — the spec file exists, and the stage
   makes no model call. A run that reuses a spec the checks then fault prints this stage
@@ -112,16 +112,23 @@ The stage prints one of three messages:
   `agent-sdk`. The token count is the call's input and output tokens added together,
   and the time is the wall time of the call to one decimal place. The try number counts
   from 1 to `--tries`.
+* ``in2lambda refused the spec: A selector's comma separates its `after` clause from
+  the rest, and 'Header, Table' holds no `after` clause. See line 1 of the spec. (try 1
+  of 3)`` — in2lambda cannot read the spec the model wrote. The refusal is in2lambda's
+  own message, and it names the line.
 * `kept try 2 of 3` — the loop wrote more than one spec, and this names the try saved
   for the set: the one that scored lowest. A try's score adds up the blocks it left
   unassigned, the errors the checks then found, the images it dropped and the blocks
   it left unassigned in the set's other document. The loop writes one spec where the
   first scores zero, and prints no `kept` line.
 
-in2lambda refuses a spec it cannot run, and the run raises `SpecRejected`. A spec this
-run wrote is deleted before that refusal reaches the user, and a spec this run wrote
-over an older one is replaced by the older one, so the next run over the set reads a
-spec in2lambda accepts.
+A spec in2lambda refused filled no draft, so it scores above every spec that ran and
+the loop never keeps it. The refusal goes to the next call, which writes another spec,
+and the run goes on with the tries it has left. The run raises `SpecRejected` where
+in2lambda refused every spec the loop wrote, carrying what in2lambda said about the
+last of them. A spec this run wrote is deleted before that refusal reaches the user,
+and a spec this run wrote over an older one is replaced by the older one, so the next
+run over the set reads a spec in2lambda accepts.
 
 ### `coverage`
 
@@ -305,7 +312,7 @@ Each backend limits a call differently:
 
 | Call | What it is given | What it may write |
 | --- | --- | --- |
-| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds, the images that spec dropped and the blocks that spec left in no field in the set's other document | `in2lambda-spec.yaml`, and nothing else |
+| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds, the images that spec dropped and the blocks that spec left in no field in the set's other document, or in2lambda's refusal of that spec where in2lambda would not run it | `in2lambda-spec.yaml`, and nothing else |
 | Spec rewrite | the same, with the saved spec and what running it covered as the first call's try 0 | `in2lambda-spec.yaml`, and nothing else |
 | Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the eight draft commands, and nothing else |
 

in2lambda_agent/review.py

@@ -269,6 +269,7 @@ def _try_json(one: SpecTry) -> dict[str, Any]:
         "errors": one.errors,
         "dropped": one.dropped,
         "second": one.second,
+        "rejected": one.rejected,
         "chosen": one.chosen,
     }
 
@@ -282,6 +283,9 @@ def _try_from(saved: dict[str, Any]) -> SpecTry:
         errors=saved["errors"],
         dropped=saved["dropped"],
         second=saved["second"],
+        # A review a run left waiting before the agent recorded a refused spec
+        # holds no `rejected` key, and its tries all ran.
+        rejected=saved.get("rejected"),
         chosen=saved["chosen"],
     )
 

in2lambda_agent/spec.py

@@ -20,6 +20,7 @@
 """
 
 import json
+import sys
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Callable, Optional, Sequence
@@ -66,6 +67,16 @@
 level, `text` is everything the block says, `label` its first word. Put single
 quotes round any regex with a backslash in it.
 
+Several selectors for one role go under the role as a YAML list, one to a line,
+never on one line with commas between them:
+
+  ignore:
+    - Header
+    - Table
+
+The one comma a selector holds is the one after its `after` anchor, and a comma
+inside a pattern goes inside the pattern's own quotes.
+
 Every block is tried against ignore, then question, then part, then solution,
 whatever order the keys are written in, and is whatever the first of them says
 it is. So the selectors must not overlap: if the solutions are paragraphs and
@@ -151,6 +162,9 @@ class SpecTry:
             — the record's `second` says why. A spec in2lambda refuses over
             that document wrote no field there, so it left every block of it in
             no field.
+        rejected: What in2lambda said where it refused the spec over this
+            source, and None where it ran the spec. Such a try filled no draft,
+            so `unassigned`, `errors` and `dropped` are 0 and `second` is None.
         chosen: Whether this is the spec the run saved and went on with.
     """
 
@@ -160,12 +174,16 @@ class SpecTry:
     errors: int = 0
     dropped: int = 0
     second: Optional[int] = None
+    rejected: Optional[str] = None
     chosen: bool = False
 
     @property
     def score(self) -> int:
         """What the tries are ranked by, the lowest winning.
 
+        A spec in2lambda refused wrote no field at all, so it scores above
+        every spec that ran, whatever that spec left over.
+
         A block in no field is an error of the report as well as a line of the
         coverage, so it counts twice. That is the same double for every try and
         does not change the order they come in.
@@ -174,6 +192,8 @@ def score(self) -> int:
         nothing about an ignored block, so a spec that ignores a figure is
         scored like one that leaves a block unassigned and is written again.
         """
+        if self.rejected is not None:
+            return sys.maxsize
         return self.unassigned + self.errors + self.dropped + (self.second or 0)
 
 
@@ -222,13 +242,17 @@ class Previous:
         second: What running it made of another document of the set, or None
             where the folder holds no other document.
         second_name: That document's file name.
+        rejected: What in2lambda said where it refused the spec over this
+            source, and None where it ran the spec. The coverage and the report
+            are then None, because the spec filled no draft.
     """
 
     text: str
     coverage: Optional[Coverage] = None
     report: Optional[Report] = None
     second: Optional[Coverage] = None
     second_name: str = ""
+    rejected: Optional[str] = None
 
 
 def spec_path(source: Path, spec: Optional[Path] = None) -> Path:
@@ -303,6 +327,8 @@ def write_spec(
 def _revision(previous: Previous) -> str:
     """The last spec and what running it covered, as the next call is shown them."""
     said = [f"\nYour last spec for this set was:\n\n{previous.text}"]
+    if previous.rejected is not None:
+        said.append(f"\nin2lambda refused the spec:\n\n{previous.rejected}\n")
     if previous.coverage is not None:
         said.append(f"\nRunning it over this source covered:\n\n{previous.coverage}\n")
     # The images the spec dropped go in beside the report's errors, under the
@@ -319,11 +345,17 @@ def _revision(previous: Previous) -> str:
             f"\nRunning it over {previous.second_name}, another document of this "
             f"set, left {left} in no field.\n"
         )
-    said.append(
-        "\nWrite a spec that leaves fewer blocks unassigned, fewer images "
-        "ignored and fewer errors behind, over this source and over the rest "
-        "of the set.\n"
-    )
+    if previous.rejected is not None:
+        said.append(
+            "\nWrite a spec in2lambda will run, over this source and over the "
+            "rest of the set.\n"
+        )
+    else:
+        said.append(
+            "\nWrite a spec that leaves fewer blocks unassigned, fewer images "
+            "ignored and fewer errors behind, over this source and over the "
+            "rest of the set.\n"
+        )
     return "".join(said)
 
 
@@ -347,6 +379,10 @@ def iterate_spec(
     blocks the spec left over in another document of the set. The loop stops at
     a spec that scores zero, since a further call has nothing to improve.
 
+    A spec in2lambda refuses over this source is a failed try: the refusal names
+    the line and the fault, so the next call is shown it and writes another
+    spec. Such a try filled no draft and is never the spec the loop keeps.
+
     Args:
         frozen: The markdown, tex or docx file each spec is run over.
         saved: The set's spec file, which every try writes and the chosen spec
@@ -382,7 +418,8 @@ def iterate_spec(
     Raises:
         ModelError: a call did not finish.
         BadSpec: what the model answered with is not a spec.
-        SpecRejected: in2lambda will not run a spec this loop wrote.
+        SpecRejected: in2lambda refused every spec this loop wrote, carrying
+            what it said about the last of them.
         SourceError: in2lambda cannot freeze or check this source.
 
     Every error leaving this function puts the spec the set had before the loop
@@ -418,6 +455,9 @@ def iterate_spec(
     # ran over it: the record is to say what became of the other document under
     # the spec the loop kept, not under a later try it threw away.
     best: Optional[tuple[SpecTry, str, Optional[str]]] = None
+    # What in2lambda said about the last spec it refused, which the run raises
+    # where it refused every one of them.
+    refusal = ""
     more = [solutions] if solutions is not None else []
     try:
         for number in range(1, tries + 1):
@@ -437,7 +477,24 @@ def iterate_spec(
                 f"wrote {saved} via {reply.backend}, {tokens} tokens, "
                 f"{reply.usage.seconds:.1f}s (try {number} of {tries})",
             )
-            coverage, report = _run(draft, saved, on_stage)
+            try:
+                coverage, report = _run(draft, saved, on_stage)
+            except package.SpecRejected as error:
+                # in2lambda cannot read this spec and filled no draft with it,
+                # so there is no coverage and no report to score the try on.
+                # The refusal names the line and the fault, so the next call is
+                # shown it and the loop goes on with the tries it has left.
+                refusal = str(error)
+                on_stage(
+                    "spec",
+                    f"in2lambda refused the spec: {refusal} "
+                    f"(try {number} of {tries})",
+                )
+                made.append(
+                    SpecTry(number=number, usage=reply.usage, rejected=refusal)
+                )
+                previous = Previous(text=text, rejected=refusal)
+                continue
             over_second, left_over = _over_second(second, saved, on_stage)
             one = SpecTry(
                 number=number,
@@ -459,6 +516,10 @@ def iterate_spec(
                 second=over_second,
                 second_name=second.name if over_second is not None else "",
             )
+        if best is None:
+            # Every try was refused, so the run has no spec to go on with and
+            # ends on the last refusal, as it ended on the first one before.
+            raise package.SpecRejected(refusal)
     except Exception:
         if replaced is None:
             saved.unlink(missing_ok=True)
@@ -603,6 +664,7 @@ def record_run(
                 "errors": one.errors,
                 "dropped": one.dropped,
                 "second": one.second,
+                "rejected": one.rejected,
                 "chosen": one.chosen,
             }
             for one in tries

tests/test_corpus.py

@@ -332,7 +332,9 @@ def refuse(draft, spec):
 
     monkeypatch.setattr(pipeline.package, "spec_run", refuse)
 
-    rows = sweep(root, tmp_path, backend=FakeBackend(SPEC, TEX_SPEC, TEX_SPEC))
+    # One try each: a run with tries left writes another spec after a refusal,
+    # and the row a refusal makes is what this test is about.
+    rows = sweep(root, tmp_path, tries=1, backend=FakeBackend(SPEC, TEX_SPEC, TEX_SPEC))
     rejected = [row for row in rows if row.set == "tex"]
 
     assert [row.outcome for row in rejected] == ["spec rejected"] * 2

tests/test_pipeline.py

@@ -422,13 +422,16 @@ def test_a_named_spec_that_is_not_there_yet_is_written_there(sheets, tmp_path):
 
 
 def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path):
+    # One try, and in2lambda refuses the spec it wrote: the loop has no try
+    # left to write another, so the refusal ends the run.
     backend = FakeBackend("question: NotAnElement\nlayout: PartsSepSol\n")
 
     with pytest.raises(SpecRejected, match="not a pandoc element"):
         pipeline.run(
             sheets / "sheet.md",
             out_dir=tmp_path / "out",
             settings=Settings(),
+            tries=1,
             backend=backend,
         )
 
@@ -442,13 +445,43 @@ def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path
         sheets / "sheet.md",
         out_dir=tmp_path / "out",
         settings=Settings(),
+        tries=1,
         backend=again,
     )
 
     assert len(again.calls) == 1
     assert result.zip_path.exists()
 
 
+def test_a_refused_spec_is_written_again_and_the_run_builds_the_set(sheets, tmp_path):
+    # Two selectors on one line with a comma between them, which in2lambda
+    # reads as an `after` clause and refuses. The loop has a try left, so the
+    # refusal goes to the second call rather than ending the run.
+    backend = FakeBackend(
+        "ignore: Header, Table\nquestion: Para\nlayout: PartsSepSol\n", SPEC
+    )
+
+    result = pipeline.run(
+        sheets / "sheet.md",
+        out_dir=tmp_path / "out",
+        settings=Settings(),
+        tries=2,
+        backend=backend,
+    )
+
+    assert len(backend.calls) == 2
+    assert "in2lambda refused the spec:" in backend.calls[1][1]
+    assert result.zip_path is not None and result.zip_path.exists()
+    assert (sheets / SPEC_NAME).read_text() == SPEC
+
+    (line,) = (sheets / RECORD_NAME).read_text().splitlines()
+    refused, kept = json.loads(line)["iterations"]
+
+    assert "holds no `after` clause" in refused["rejected"]
+    assert refused["chosen"] is False
+    assert (kept["rejected"], kept["chosen"]) == (None, True)
+
+
 def test_a_saved_spec_that_drops_an_image_is_written_again_and_the_run_goes_on(
     figure_paragraph, tmp_path
 ):
@@ -491,6 +524,7 @@ def test_a_rewrite_in2lambda_will_not_run_leaves_the_saved_spec_alone(
             sheets / "sheet.md",
             out_dir=tmp_path / "out",
             settings=Settings(),
+            tries=1,
             backend=backend,
         )
 

tests/test_review.py

@@ -79,7 +79,8 @@ def test_the_record_goes_to_json_and_comes_back(tmp_path):
         usage=Usage(input_tokens=120, output_tokens=40, seconds=1.5),
         tries=[
             SpecTry(0, Usage(), unassigned=2, errors=2, dropped=1),
-            SpecTry(1, Usage(input_tokens=120, output_tokens=40), second=0, chosen=True),
+            SpecTry(1, Usage(), rejected="'Header, Table' holds no `after` clause"),
+            SpecTry(2, Usage(input_tokens=120, output_tokens=40), second=0, chosen=True),
         ],
         rounds=[
             RoundResult(1, [ToolCall("part_add", {"question": "q2"}, "wrote")], Usage(), 0)
@@ -97,8 +98,11 @@ def test_the_record_goes_to_json_and_comes_back(tmp_path):
     assert read.rounds[0].commands[0].name == "part_add"
     # The iterations too, so that the record the last approval writes says what
     # each spec the run wrote covered and cost.
-    assert [one.number for one in read.tries] == [0, 1]
-    assert read.tries[1].usage.input_tokens == 120 and read.tries[1].chosen is True
+    assert [one.number for one in read.tries] == [0, 1, 2]
+    assert read.tries[2].usage.input_tokens == 120 and read.tries[2].chosen is True
+    # A spec in2lambda refused is a try of the run like any other, and what
+    # in2lambda said about it comes back with the rest.
+    assert read.tries[1].rejected == "'Header, Table' holds no `after` clause"
 
 
 def test_the_other_document_of_the_set_goes_to_json_and_comes_back(tmp_path):