From e6ef1bd430ec7e2ac85a9cdb37f282ee51a49e75 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 02:01:40 +0100 Subject: [PATCH 1/2] implement: Keep the agent-sdk backend to the call's tools and report an SDK error as the cause (t29) --- docs/how-it-works.md | 2 +- in2lambda_agent/cli.py | 10 +++--- in2lambda_agent/corpus.py | 10 ++++-- in2lambda_agent/model.py | 70 +++++++++++++++++++++++++++++++++--- in2lambda_agent/pipeline.py | 55 +++++++++++++++++++--------- in2lambda_agent/ui/server.py | 3 +- tests/conftest.py | 5 ++- tests/test_corpus.py | 27 ++++++++++++++ tests/test_model.py | 64 +++++++++++++++++++++++++++++++-- 9 files changed, 215 insertions(+), 31 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index ad22e5d..54109a5 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -320,7 +320,7 @@ these 21, in this order: | --- | --- | | `source` | the document, relative to the corpus root | | `set` | the folder the document is in | -| `outcome` | `built`, `build refused`, `faulted`, `skipped`, `no spec`, `no model`, `spec rejected`, `bad spec`, or `error: ` | +| `outcome` | `built`, `build refused`, `faulted`, `skipped`, `no spec`, `no model`, `spec failed` and `fix failed` where the model call did not finish, `spec rejected`, `bad spec`, or `error: ` | | `reason` | the build's refusal, the first error the checks still found, the warnings a build proceeded past, or what an exception said | | `spec` | `wrote`, `reused`, or `rewritten` where the spec rewrite ran | | `layout` | the coverage's layout | diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index 7664e72..c9be38a 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -8,7 +8,7 @@ from in2lambda_agent import compare, corpus, pipeline from in2lambda_agent.mathpix import MathpixClient, MathpixError -from in2lambda_agent.model import ModelUnavailable, choose_backend +from in2lambda_agent.model import ModelError, ModelUnavailable, choose_backend from in2lambda_agent.ocr import ocr_pdf from in2lambda_agent.package import CommandRefused, SpecRejected from in2lambda_agent.pair import SolutionsWithoutQuestions @@ -355,6 +355,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: except ( MathpixError, ModelUnavailable, + ModelError, BadSpec, SpecRejected, ReviewError, @@ -362,9 +363,10 @@ def main(argv: Optional[Sequence[str]] = None) -> int: SolutionsWithoutQuestions, ) as error: # Missing credentials among them: the message names the variables, or - # the login to run, or what a spec says that a spec cannot say, or the - # question a review command names that is not under review, or the - # questions file a solutions file was run without. + # the login to run, or what the provider said stopped a call, or what a + # spec says that a spec cannot say, or the question a review command + # names that is not under review, or the questions file a solutions file + # was run without. print(f"in2lambda-agent: {error}", file=sys.stderr) return 1 diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 494d19c..e28067e 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -25,7 +25,7 @@ from typing import Optional, Sequence from in2lambda_agent import package, pair, pipeline -from in2lambda_agent.model import Backend, ModelUnavailable +from in2lambda_agent.model import Backend, ModelError, ModelUnavailable from in2lambda_agent.package import SpecRejected from in2lambda_agent.settings import Settings from in2lambda_agent.spec import RECORD_NAME, SPEC_NAME, BadSpec @@ -62,7 +62,8 @@ class Row: the checks still fault and no zip, `skipped` for a file that is not a document and for a solutions document with no questions document beside it, `no spec` for a replay with nothing saved to replay, - `no model`, `spec rejected`, `bad spec`, or `error: `. + `no model`, `spec failed` and `fix failed` where a model call did + not finish, `spec rejected`, `bad spec`, or `error: `. reason: What the run had to say for itself, in the words of whatever said it: the refusal, the first error the checks were still finding, or what the exception said. On a `built` row it holds the warnings @@ -294,6 +295,11 @@ def run_one( except ModelUnavailable as error: row.outcome = "no model" if existed else "no spec" row.reason = _one_line(str(error)) + except ModelError as error: + # Which call did not finish, and what the provider said it stopped on. + # A row reading `error: ResultError` says neither. + row.outcome = f"{error.stage or 'model'} failed" + row.reason = _one_line(str(error)) except SpecRejected as error: row.outcome = "spec rejected" row.reason = _one_line(str(error)) diff --git a/in2lambda_agent/model.py b/in2lambda_agent/model.py index 534c325..2503ac5 100644 --- a/in2lambda_agent/model.py +++ b/in2lambda_agent/model.py @@ -14,6 +14,9 @@ Every `Reply` carries the tokens and the wall time for that call, which the design spec's test plan records per document. + +A call that does not finish raises `ModelError`, whose message is what the +provider said. A call that cannot be made at all raises `ModelUnavailable`. """ import asyncio @@ -46,6 +49,35 @@ REQUEST_TIMEOUT = 300.0 +# Claude Code's own tools, named for `disallowed_tools`. The agent gives the +# model the tools each call needs and no others: a spec call has none, and a +# call that could run Bash or Read on the paths its prompt names spends its +# turns reading the corpus. `tools=[]` alone does not switch them off — the SDK +# sends it as `--tools ""`, and the run that recorded `error_max_turns` on +# Worksheet_1.pdf passed it. `disallowed_tools` refuses each tool by name, and +# a name Claude Code does not have is ignored. +BUILTIN_TOOLS = ( + "Agent", + "Bash", + "BashOutput", + "Edit", + "ExitPlanMode", + "Glob", + "Grep", + "KillShell", + "LS", + "MultiEdit", + "NotebookEdit", + "Read", + "Skill", + "SlashCommand", + "Task", + "TodoWrite", + "WebFetch", + "WebSearch", + "Write", +) + @dataclass class Tool: @@ -99,6 +131,19 @@ class ModelUnavailable(RuntimeError): """A backend was called without the credential or the login it needs.""" +class ModelError(RuntimeError): + """A call was made and did not finish: the provider stopped it, or the + model asked for tools until the round limit and never answered. + + Attributes: + stage: Which of the agent's calls this was — `spec` or `fix` — set by + the pipeline and read by the corpus sweep, which names it in the + row's outcome. Empty where nothing set it. + """ + + stage: str = "" + + def _encoded(image: bytes) -> str: """One PNG page as the base64 every provider's image block carries.""" return base64.standard_b64encode(image).decode("ascii") @@ -137,6 +182,7 @@ def call( Raises: ModelUnavailable: If `unavailable` would give a reason. + ModelError: If the call did not finish. """ @@ -182,6 +228,7 @@ async def _call( ) -> Reply: from claude_agent_sdk import ( ClaudeAgentOptions, + ClaudeSDKError, ResultMessage, create_sdk_mcp_server, query, @@ -202,7 +249,10 @@ async def handler(arguments: dict[str, Any]) -> dict[str, Any]: options = ClaudeAgentOptions( system_prompt=system, mcp_servers={"agent": server}, + # The call's own tools, and no others: `allowed_tools` is empty for + # the spec call, which has none. allowed_tools=[f"mcp__agent__{one.name}" for one in tools], + disallowed_tools=list(BUILTIN_TOOLS), # No built-in tools, and no settings file: nothing the machine # happens to have configured reaches the call. Both need the empty # list, which the SDK documents as "disable all built-in tools" and @@ -210,6 +260,9 @@ async def handler(arguments: dict[str, Any]) -> dict[str, Any]: # which loads the CLI's own set. tools=[], setting_sources=[], + # No permission prompt: a call has no terminal to answer one at, + # and the tools it may run are the two lists above. + permission_mode="bypassPermissions", max_turns=MAX_TOOL_ROUNDS, ) @@ -249,17 +302,26 @@ async def one_message(): result = None stream = query(prompt=asked, options=options) + failed = None try: async for message in stream: if isinstance(message, ResultMessage) and result is None: result = message + except ClaudeSDKError as error: + # The SDK raises rather than yielding a result for a run the CLI + # ended on an error, so the two branches below never see one. Its + # message says what stopped the run; raising it here rather than + # inside the `async for` keeps the `finally` below. + failed = error finally: await stream.aclose() + if failed is not None: + raise ModelError(str(failed)) if result is None: - raise RuntimeError("the agent-sdk backend returned no result") + raise ModelError("the agent-sdk backend returned no result") if result.is_error: - raise RuntimeError( + raise ModelError( f"the agent-sdk backend stopped on {result.subtype}: " f"{result.result}" ) @@ -389,7 +451,7 @@ def call( ) messages.append({"role": "user", "content": results}) - raise RuntimeError( + raise ModelError( f"the {self.name} backend asked for tools for " f"{MAX_TOOL_ROUNDS} rounds without answering" ) @@ -535,7 +597,7 @@ def _loop( } ) - raise RuntimeError( + raise ModelError( f"the {self.name} backend asked for tools for " f"{MAX_TOOL_ROUNDS} rounds without answering" ) diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index fe721d1..5e9c5a7 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -35,7 +35,13 @@ from in2lambda_agent import package, pair from in2lambda_agent.fix import RoundResult, fix_round, summary, unrepaired from in2lambda_agent.mathpix import MathpixClient -from in2lambda_agent.model import Backend, ModelUnavailable, Usage, choose_backend +from in2lambda_agent.model import ( + Backend, + ModelError, + ModelUnavailable, + Usage, + choose_backend, +) from in2lambda_agent.ocr import MEDIA_NAME, ocr_pdf from in2lambda_agent.review import RECORD, Question, Review, choose from in2lambda_agent.settings import Settings @@ -157,6 +163,8 @@ def run( MathpixError: If a PDF cannot be converted, MissingCredentials among them when the run has no Mathpix credentials. ModelUnavailable: If a spec must be written and no backend can run. + ModelError: If a call did not finish, with `stage` naming which — the + spec call or a fixing round. BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. SourceError: If in2lambda cannot freeze or check the source. @@ -231,12 +239,18 @@ def run( backend = backend or choose_backend(settings) if (reason := backend.unavailable()) is not None: raise ModelUnavailable(reason) - text, reply = write_spec( - package.source_show(draft), - backend, - report if report.errors else None, - sources=2 if frozen_solutions is not None else 1, - ) + try: + text, reply = write_spec( + package.source_show(draft), + backend, + report if report.errors else None, + sources=2 if frozen_solutions is not None else 1, + ) + except ModelError as error: + # Which call did not finish, for a caller that names it: a spec + # call and a fixing round both go to the same backend. + error.stage = "spec" + raise if saved.is_file(): replaced = saved.read_text(encoding="utf-8") saved.write_text(text, encoding="utf-8") @@ -279,7 +293,11 @@ def run( # Layers 3 and 4, a round at a time. Reached only with a spec this run # wrote, so the backend is the one that wrote it. - report = _fix_rounds(draft, report, backend, rounds, result) + try: + report = _fix_rounds(draft, report, backend, rounds, result) + except ModelError as error: + error.stage = "fix" + raise # What the corpus harness reads off the result rather than off the # record: set here so that a run that stops for a review carries them # too, since that return is above the record this run never writes. @@ -386,6 +404,7 @@ def resume( Raises: ReviewError: no review is waiting, or none of its questions is `key`. ModelUnavailable: a rejection has no backend to answer its note with. + ModelError: a rejection's fixing round did not finish. CommandRefused: in2lambda would not make the reviewer's edit. """ cache_dir = Path(cache_dir).resolve() @@ -463,14 +482,18 @@ def resume( raise ModelUnavailable(reason) # The note is a finding of its own: the checks are quiet, and it is # what the round is for. Rounds after it answer what they leave. - report = _fix_rounds( - draft, - package.validate(draft), - backend, - waiting.limit, - result, - instruction=f"The reviewer rejected {key}: {note}", - ) + try: + report = _fix_rounds( + draft, + package.validate(draft), + backend, + waiting.limit, + result, + instruction=f"The reviewer rejected {key}: {note}", + ) + except ModelError as error: + error.stage = "fix" + raise relisted = [key] else: package.command( diff --git a/in2lambda_agent/ui/server.py b/in2lambda_agent/ui/server.py index 6fafa34..8f150b3 100644 --- a/in2lambda_agent/ui/server.py +++ b/in2lambda_agent/ui/server.py @@ -37,7 +37,7 @@ from in2lambda_agent import corpus, pipeline, spec from in2lambda_agent.mathpix import MathpixError -from in2lambda_agent.model import ModelUnavailable +from in2lambda_agent.model import ModelError, ModelUnavailable from in2lambda_agent.package import CommandRefused, SpecRejected from in2lambda_agent.review import RECORD, ReviewError from in2lambda_agent.settings import Settings, load_settings @@ -62,6 +62,7 @@ FAILURES = ( MathpixError, ModelUnavailable, + ModelError, BadSpec, SpecRejected, ReviewError, diff --git a/tests/conftest.py b/tests/conftest.py index 1b5a6ad..e0e7ced 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,8 @@ class FakeBackend: A reply is the text to answer with, or a list of `(tool name, arguments)` for a call that uses its tools: the named tools are run in the order given, against whatever they were built over, exactly as a real backend's loop runs - them. That is what scripts a fixing round without a model in it. + them. That is what scripts a fixing round without a model in it. A reply + that is an exception is raised, which scripts a call that does not finish. """ name = "fake" @@ -36,6 +37,8 @@ def call(self, system, prompt, tools=(), images=()): self.calls.append((system, prompt)) self.images.append(list(images)) reply = self.replies.pop(0) + if isinstance(reply, Exception): + raise reply made = [] if isinstance(reply, list): by_name = {one.name: one for one in tools} diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 38d2f46..bdc08e1 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -10,6 +10,7 @@ from test_pipeline import FAULTY_SPEC, FIXES, PAIRED_SPEC, SPEC, TEX_SPEC from in2lambda_agent import corpus, pipeline +from in2lambda_agent.model import ModelError from in2lambda_agent.settings import Settings from in2lambda_agent.spec import SPEC_NAME @@ -363,6 +364,32 @@ def test_a_document_that_raises_is_a_row_and_not_the_end_of_the_sweep( assert [row.source for row in rows[1:]] == ["sheets/sheet-2.md", "sheets/sheet.md"] +def test_a_spec_call_that_did_not_finish_names_the_call_and_the_cause(tmp_path): + root = tmp_path / "corpus" + make_set(root, "broken", ["sheet.md"]) + make_set(root, "sheets", ["sheet.md"]) + stopped = ModelError( + "Claude Code returned an error result: Reached maximum number of turns (8)" + ) + + rows = sweep(root, tmp_path, backend=FakeBackend(stopped, SPEC)) + + assert (rows[0].outcome, rows[0].reason) == ("spec failed", str(stopped)) + # And the set after it still runs. + assert (rows[1].source, rows[1].outcome) == ("sheets/sheet.md", "built") + + +def test_a_fixing_round_that_did_not_finish_says_so_and_not_spec_failed(tmp_path): + root = tmp_path / "corpus" + make_set(root, "faulty", ["faulty.md"]) + stopped = ModelError("the agent-sdk backend stopped on error_during_execution: None") + + # The spec call answers, so what did not finish is the round the checks ask for. + (row,) = sweep(root, tmp_path, backend=FakeBackend(FAULTY_SPEC, stopped)) + + assert (row.outcome, row.reason) == ("fix failed", str(stopped)) + + def test_a_staged_set_leaves_behind_what_no_run_reads(root, tmp_path): (root / "sheets" / "scan.pdf").write_bytes(b"%PDF-1.4 most of what a corpus weighs") (root / "sheets" / "sources.zip").write_bytes(b"PK the rest of what it weighs") diff --git a/tests/test_model.py b/tests/test_model.py index c2ec2c3..cb8c611 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -13,10 +13,12 @@ import pytest from in2lambda_agent.model import ( + BUILTIN_TOOLS, MAX_TOOL_ROUNDS, OPENROUTER_URL, AgentSDKBackend, AnthropicBackend, + ModelError, ModelUnavailable, OpenRouterBackend, Tool, @@ -137,7 +139,7 @@ def test_an_agent_sdk_error_is_raised_after_the_generator_ends(monkeypatch): ) monkeypatch.setattr("claude_agent_sdk.query", query) - with pytest.raises(RuntimeError, match="error_max_turns"): + with pytest.raises(ModelError, match="error_max_turns"): AgentSDKBackend().call("system", "prompt", [ADD]) assert (ran["finished"], ran["closed"]) == (True, True) @@ -148,12 +150,70 @@ def test_an_agent_sdk_run_without_a_result_says_so(monkeypatch): query, ran = fake_query() monkeypatch.setattr("claude_agent_sdk.query", query) - with pytest.raises(RuntimeError, match="no result"): + with pytest.raises(ModelError, match="no result"): AgentSDKBackend().call("system", "prompt", [ADD]) assert (ran["finished"], ran["closed"]) == (True, True) +def sdk_options(monkeypatch, tools): + """The options `AgentSDKBackend` builds for a call with the given tools.""" + monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/claude") + query, _ = fake_query(result_message()) + seen = {} + + async def recording(*, prompt, options, transport=None): + seen["options"] = options + async for message in query(prompt=prompt, options=options): + yield message + + monkeypatch.setattr("claude_agent_sdk.query", recording) + AgentSDKBackend().call("system", "prompt", tools) + return seen["options"] + + +def test_the_agent_sdk_call_has_only_its_own_tools(monkeypatch): + options = sdk_options(monkeypatch, [ADD]) + + assert options.allowed_tools == ["mcp__agent__add"] + # Not `--tools ""` alone: the run that recorded error_max_turns passed that + # and Claude Code read its files anyway. + assert set(options.disallowed_tools) == set(BUILTIN_TOOLS) + assert {"Bash", "Read"} <= set(options.disallowed_tools) + assert options.permission_mode == "bypassPermissions" + assert (options.tools, options.setting_sources) == ([], []) + + +def test_the_agent_sdk_spec_call_is_allowed_no_tool_at_all(monkeypatch): + assert sdk_options(monkeypatch, []).allowed_tools == [] + + +def test_an_sdk_error_names_the_cause(monkeypatch): + from claude_agent_sdk import ResultError + + monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/claude") + stopped = ResultError( + "Claude Code returned an error result: " + "Reached maximum number of turns (8)", + {"subtype": "error_max_turns"}, + ) + ran = {"closed": False} + + async def query(*, prompt, options, transport=None): + try: + yield result_message() + raise stopped + finally: + ran["closed"] = True + + monkeypatch.setattr("claude_agent_sdk.query", query) + + with pytest.raises(ModelError, match="maximum number of turns"): + AgentSDKBackend().call("system", "prompt") + + assert ran["closed"] + + class FakeAnthropic: """Returns the given responses in turn, repeating the last one for ever. From b25d0a7ab75fc4e6caf8ace2215d83c2959bd71d Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 02:15:32 +0100 Subject: [PATCH 2/2] implement: Keep the agent-sdk backend to the call's tools and report an SDK error as the cause (t29) --- tests/test_model.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_model.py b/tests/test_model.py index cb8c611..984702c 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -199,14 +199,21 @@ def test_an_sdk_error_names_the_cause(monkeypatch): ) ran = {"closed": False} - async def query(*, prompt, options, transport=None): - try: - yield result_message() + class Stream: + """A query that raises rather than yielding, and closes only when it is + asked to: an async generator would run its own cleanup on the way out, + and the flag would say `closed` whether `_call` closed it or not.""" + + def __aiter__(self): + return self + + async def __anext__(self): raise stopped - finally: + + async def aclose(self): ran["closed"] = True - monkeypatch.setattr("claude_agent_sdk.query", query) + monkeypatch.setattr("claude_agent_sdk.query", lambda **called: Stream()) with pytest.raises(ModelError, match="maximum number of turns"): AgentSDKBackend().call("system", "prompt")