implement: Convert a solutions file on its own when no questions file is beside it (t30)

Author: peterbjohnson

docs/how-it-works.md

@@ -384,7 +384,7 @@ these 21, in this order:
 | --- | --- |
 | `source` | the document, relative to the corpus root |
 | `set` | the folder the document is in |
-| `outcome` | `built`, `build refused`, `faulted`, `skipped`, `no spec`, `no model`, `spec rejected`, `bad spec`, or `error: <exception>` |
+| `outcome` | `built`, `build refused`, `faulted`, `skipped`, `no spec`, `no model`, `spec failed` and `fix failed` where the model call did not finish, `spec rejected`, `bad spec`, or `error: <exception>` |
 | `reason` | the build's refusal, the first error the checks still found, the warnings a build proceeded past, or what an exception said |
 | `spec` | `wrote`, `reused`, or `rewritten` where the spec rewrite ran |
 | `layout` | the coverage's layout |

in2lambda_agent/cli.py

@@ -9,7 +9,7 @@
 
 from in2lambda_agent import compare, corpus, gate, pipeline
 from in2lambda_agent.mathpix import MathpixClient, MathpixError
-from in2lambda_agent.model import ModelUnavailable, choose_backend
+from in2lambda_agent.model import ModelError, ModelUnavailable, choose_backend
 from in2lambda_agent.ocr import ocr_pdf
 from in2lambda_agent.package import CommandRefused, SpecRejected
 from in2lambda_agent.review import ReviewError
@@ -443,14 +443,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     except (
         MathpixError,
         ModelUnavailable,
+        ModelError,
         BadSpec,
         SpecRejected,
         ReviewError,
         CommandRefused,
     ) as error:
         # Missing credentials among them: the message names the variables, or
-        # the login to run, or what a spec says that a spec cannot say, or the
-        # question a review command names that is not under review.
+        # the login to run, or what the provider said stopped a call, or what a
+        # spec says that a spec cannot say, or the question a review command
+        # names that is not under review.
         print(f"in2lambda-agent: {error}", file=sys.stderr)
         return 1
 

in2lambda_agent/corpus.py

@@ -25,7 +25,7 @@
 from typing import Optional, Sequence
 
 from in2lambda_agent import package, pair, pipeline
-from in2lambda_agent.model import Backend, ModelUnavailable
+from in2lambda_agent.model import Backend, ModelError, ModelUnavailable
 from in2lambda_agent.package import SpecRejected, is_document
 from in2lambda_agent.settings import Settings
 from in2lambda_agent.spec import RECORD_NAME, SPEC_NAME, BadSpec
@@ -62,7 +62,8 @@ class Row:
             the checks still fault and no zip, `skipped` for a file that is not
             a document and for a solutions document with no questions document
             beside it, `no spec` for a replay with nothing saved to replay,
-            `no model`, `spec rejected`, `bad spec`, or `error: <exception>`.
+            `no model`, `spec failed` and `fix failed` where a model call did
+            not finish, `spec rejected`, `bad spec`, or `error: <exception>`.
         reason: What the run had to say for itself, in the words of whatever
             said it: the refusal, the first error the checks were still finding,
             or what the exception said. On a `built` row it holds the warnings
@@ -287,6 +288,11 @@ def run_one(
     except ModelUnavailable as error:
         row.outcome = "no model" if existed else "no spec"
         row.reason = _one_line(str(error))
+    except ModelError as error:
+        # Which call did not finish, and what the provider said it stopped on.
+        # A row reading `error: ResultError` says neither.
+        row.outcome = f"{error.stage or 'model'} failed"
+        row.reason = _one_line(str(error))
     except SpecRejected as error:
         row.outcome = "spec rejected"
         row.reason = _one_line(str(error))

in2lambda_agent/model.py

@@ -14,6 +14,9 @@
 
 Every `Reply` carries the tokens and the wall time for that call, which the
 design spec's test plan records per document.
+
+A call that does not finish raises `ModelError`, whose message is what the
+provider said. A call that cannot be made at all raises `ModelUnavailable`.
 """
 
 import asyncio
@@ -46,6 +49,35 @@
 
 REQUEST_TIMEOUT = 300.0
 
+# Claude Code's own tools, named for `disallowed_tools`. The agent gives the
+# model the tools each call needs and no others: a spec call has none, and a
+# call that could run Bash or Read on the paths its prompt names spends its
+# turns reading the corpus. `tools=[]` alone does not switch them off — the SDK
+# sends it as `--tools ""`, and the run that recorded `error_max_turns` on
+# Worksheet_1.pdf passed it. `disallowed_tools` refuses each tool by name, and
+# a name Claude Code does not have is ignored.
+BUILTIN_TOOLS = (
+    "Agent",
+    "Bash",
+    "BashOutput",
+    "Edit",
+    "ExitPlanMode",
+    "Glob",
+    "Grep",
+    "KillShell",
+    "LS",
+    "MultiEdit",
+    "NotebookEdit",
+    "Read",
+    "Skill",
+    "SlashCommand",
+    "Task",
+    "TodoWrite",
+    "WebFetch",
+    "WebSearch",
+    "Write",
+)
+
 
 @dataclass
 class Tool:
@@ -99,6 +131,19 @@ class ModelUnavailable(RuntimeError):
     """A backend was called without the credential or the login it needs."""
 
 
+class ModelError(RuntimeError):
+    """A call was made and did not finish: the provider stopped it, or the
+    model asked for tools until the round limit and never answered.
+
+    Attributes:
+        stage: Which of the agent's calls this was — `spec` or `fix` — set by
+            the pipeline and read by the corpus sweep, which names it in the
+            row's outcome. Empty where nothing set it.
+    """
+
+    stage: str = ""
+
+
 def _encoded(image: bytes) -> str:
     """One PNG page as the base64 every provider's image block carries."""
     return base64.standard_b64encode(image).decode("ascii")
@@ -137,6 +182,7 @@ def call(
 
         Raises:
             ModelUnavailable: If `unavailable` would give a reason.
+            ModelError: If the call did not finish.
         """
 
 
@@ -182,6 +228,7 @@ async def _call(
     ) -> Reply:
         from claude_agent_sdk import (
             ClaudeAgentOptions,
+            ClaudeSDKError,
             ResultMessage,
             create_sdk_mcp_server,
             query,
@@ -202,14 +249,20 @@ async def handler(arguments: dict[str, Any]) -> dict[str, Any]:
         options = ClaudeAgentOptions(
             system_prompt=system,
             mcp_servers={"agent": server},
+            # The call's own tools, and no others: `allowed_tools` is empty for
+            # the spec call, which has none.
             allowed_tools=[f"mcp__agent__{one.name}" for one in tools],
+            disallowed_tools=list(BUILTIN_TOOLS),
             # No built-in tools, and no settings file: nothing the machine
             # happens to have configured reaches the call. Both need the empty
             # list, which the SDK documents as "disable all built-in tools" and
             # "disable filesystem settings"; the default for each is `None`,
             # which loads the CLI's own set.
             tools=[],
             setting_sources=[],
+            # No permission prompt: a call has no terminal to answer one at,
+            # and the tools it may run are the two lists above.
+            permission_mode="bypassPermissions",
             max_turns=MAX_TOOL_ROUNDS,
         )
 
@@ -249,17 +302,26 @@ async def one_message():
 
         result = None
         stream = query(prompt=asked, options=options)
+        failed = None
         try:
             async for message in stream:
                 if isinstance(message, ResultMessage) and result is None:
                     result = message
+        except ClaudeSDKError as error:
+            # The SDK raises rather than yielding a result for a run the CLI
+            # ended on an error, so the two branches below never see one. Its
+            # message says what stopped the run; raising it here rather than
+            # inside the `async for` keeps the `finally` below.
+            failed = error
         finally:
             await stream.aclose()
 
+        if failed is not None:
+            raise ModelError(str(failed))
         if result is None:
-            raise RuntimeError("the agent-sdk backend returned no result")
+            raise ModelError("the agent-sdk backend returned no result")
         if result.is_error:
-            raise RuntimeError(
+            raise ModelError(
                 f"the agent-sdk backend stopped on {result.subtype}: "
                 f"{result.result}"
             )
@@ -389,7 +451,7 @@ def call(
                 )
             messages.append({"role": "user", "content": results})
 
-        raise RuntimeError(
+        raise ModelError(
             f"the {self.name} backend asked for tools for "
             f"{MAX_TOOL_ROUNDS} rounds without answering"
         )
@@ -535,7 +597,7 @@ def _loop(
                     }
                 )
 
-        raise RuntimeError(
+        raise ModelError(
             f"the {self.name} backend asked for tools for "
             f"{MAX_TOOL_ROUNDS} rounds without answering"
         )

in2lambda_agent/pipeline.py

@@ -36,7 +36,13 @@
 from in2lambda_agent import package, pair
 from in2lambda_agent.fix import RoundResult, fix_round, summary, unrepaired
 from in2lambda_agent.mathpix import MathpixClient
-from in2lambda_agent.model import Backend, ModelUnavailable, Usage, choose_backend
+from in2lambda_agent.model import (
+    Backend,
+    ModelError,
+    ModelUnavailable,
+    Usage,
+    choose_backend,
+)
 from in2lambda_agent.ocr import MEDIA_NAME, cached, ocr_pdf
 from in2lambda_agent.review import RECORD, Question, Review, choose
 from in2lambda_agent.settings import Settings
@@ -176,6 +182,8 @@ def run(
             them when a conversion is needed and the run has no Mathpix
             credentials. A PDF already in the cache needs none.
         ModelUnavailable: If a spec must be written and no backend can run.
+        ModelError: If a call did not finish, with `stage` naming which — the
+            spec call or a fixing round.
         BadSpec: If what the model answers with is not a spec.
         SpecRejected: If in2lambda will not run the spec.
         SourceError: If in2lambda cannot freeze or check the source.
@@ -274,18 +282,24 @@ def run(
         if (reason := backend.unavailable()) is not None:
             raise ModelUnavailable(reason)
         result.second = _second(source, cache_dir, solutions)
-        draft, coverage, report, result.tries = iterate_spec(
-            frozen,
-            saved,
-            backend,
-            tries=tries,
-            on_stage=result.add_stage,
-            second=result.second,
-            previous=previous,
-            solutions=frozen_solutions,
-            solutions_name=solutions.name if solutions is not None else "",
-            solutions_only=alone is not None,
-        )
+        try:
+            draft, coverage, report, result.tries = iterate_spec(
+                frozen,
+                saved,
+                backend,
+                tries=tries,
+                on_stage=result.add_stage,
+                second=result.second,
+                previous=previous,
+                solutions=frozen_solutions,
+                solutions_name=solutions.name if solutions is not None else "",
+                solutions_only=alone is not None,
+            )
+        except ModelError as error:
+            # Which call did not finish, for a caller that names it: a spec call
+            # and a fixing round both go to the same backend.
+            error.stage = "spec"
+            raise
         result.draft = draft
         result.coverage = coverage
         for one in result.tries:
@@ -295,7 +309,11 @@ def run(
 
     # Layers 3 and 4, a round at a time. Reached only with a spec this run
     # wrote, so the backend is the one that wrote it.
-    report = _fix_rounds(draft, report, backend, rounds, result)
+    try:
+        report = _fix_rounds(draft, report, backend, rounds, result)
+    except ModelError as error:
+        error.stage = "fix"
+        raise
     # What the corpus harness reads off the result rather than off the
     # record: set here so that a run that stops for a review carries them
     # too, since that return is above the record this run never writes.
@@ -406,6 +424,7 @@ def resume(
     Raises:
         ReviewError: no review is waiting, or none of its questions is `key`.
         ModelUnavailable: a rejection has no backend to answer its note with.
+        ModelError: a rejection's fixing round did not finish.
         CommandRefused: in2lambda would not make the reviewer's edit.
     """
     cache_dir = Path(cache_dir).resolve()
@@ -488,14 +507,18 @@ def resume(
                 raise ModelUnavailable(reason)
             # The note is a finding of its own: the checks are quiet, and it is
             # what the round is for. Rounds after it answer what they leave.
-            report = _fix_rounds(
-                draft,
-                package.validate(draft),
-                backend,
-                waiting.limit,
-                result,
-                instruction=f"The reviewer rejected {key}: {note}",
-            )
+            try:
+                report = _fix_rounds(
+                    draft,
+                    package.validate(draft),
+                    backend,
+                    waiting.limit,
+                    result,
+                    instruction=f"The reviewer rejected {key}: {note}",
+                )
+            except ModelError as error:
+                error.stage = "fix"
+                raise
         relisted = [key]
     else:
         package.command(

in2lambda_agent/spec.py

@@ -365,6 +365,7 @@ def iterate_spec(
         prints went to `on_stage` as the loop made it.
 
     Raises:
+        ModelError: a call did not finish.
         BadSpec: what the model answered with is not a spec.
         SpecRejected: in2lambda will not run a spec this loop wrote.
         SourceError: in2lambda cannot freeze or check this source.

in2lambda_agent/ui/server.py

@@ -37,7 +37,7 @@
 
 from in2lambda_agent import corpus, pipeline, spec
 from in2lambda_agent.mathpix import MathpixError
-from in2lambda_agent.model import ModelUnavailable
+from in2lambda_agent.model import ModelError, ModelUnavailable
 from in2lambda_agent.package import CommandRefused, SpecRejected
 from in2lambda_agent.review import RECORD, ReviewError
 from in2lambda_agent.settings import Settings, load_settings
@@ -62,6 +62,7 @@
 FAILURES = (
     MathpixError,
     ModelUnavailable,
+    ModelError,
     BadSpec,
     SpecRejected,
     ReviewError,

tests/conftest.py

@@ -18,7 +18,8 @@ class FakeBackend:
     A reply is the text to answer with, or a list of `(tool name, arguments)`
     for a call that uses its tools: the named tools are run in the order given,
     against whatever they were built over, exactly as a real backend's loop runs
-    them. That is what scripts a fixing round without a model in it.
+    them. That is what scripts a fixing round without a model in it. A reply
+    that is an exception is raised, which scripts a call that does not finish.
     """
 
     name = "fake"
@@ -36,6 +37,8 @@ def call(self, system, prompt, tools=(), images=()):
         self.calls.append((system, prompt))
         self.images.append(list(images))
         reply = self.replies.pop(0)
+        if isinstance(reply, Exception):
+            raise reply
         made = []
         if isinstance(reply, list):
             by_name = {one.name: one for one in tools}