diff --git a/examples/.test_scripts/run-examples.py b/examples/.test_scripts/run-examples.py index 2b12b5d6..bfd07af6 100755 --- a/examples/.test_scripts/run-examples.py +++ b/examples/.test_scripts/run-examples.py @@ -6,6 +6,8 @@ uv run examples/.test_scripts/run-examples.py --image # also image samples uv run examples/.test_scripts/run-examples.py --video # also video samples uv run examples/.test_scripts/run-examples.py --e2e # also e2e test scripts + uv run examples/.test_scripts/run-examples.py --external + # also external samples: need logged-in `claude`, `codex` and `fx` CLIs uv run examples/.test_scripts/run-examples.py --all # everything uv run examples/.test_scripts/run-examples.py --parallel # in parallel uv run examples/.test_scripts/run-examples.py models/stream.py @@ -74,6 +76,22 @@ class Sample: Sample("media/speech_generation.py"), ] +# These drive real coding-agent CLIs, so --model and --protocol do not apply +# to the external harness. The CLIs must be installed and logged in. +EXTERNAL_SAMPLES: list[Sample] = [ + Sample("external/basic.py", timeout=300.0), + Sample("external/streaming.py", timeout=300.0), + Sample("external/agent_tools.py", timeout=300.0), + Sample("external/direct_control.py", timeout=300.0), + Sample("external/steering.py", timeout=300.0), + Sample("external/resume.py", timeout=300.0), + Sample("external/combine.py", timeout=300.0), + Sample("external/as_tool.py", timeout=300.0), + Sample("external/custom_loop.py", timeout=300.0), + Sample("external/until_done.py", timeout=600.0), + Sample("external/handoff.py", timeout=600.0), +] + BROKEN_SAMPLES: list[Sample] = [] # E2E tests pick non-default ports so they don't collide with a running @@ -104,6 +122,7 @@ class Sample: *IMAGE_SAMPLES, *VIDEO_SAMPLES, *AUDIO_SAMPLES, + *EXTERNAL_SAMPLES, *BROKEN_SAMPLES, *E2E_TESTS, ] @@ -248,6 +267,11 @@ def main() -> None: parser.add_argument( "--audio", action="store_true", help="include audio samples" ) + parser.add_argument( + "--external", + action="store_true", + help="include external harness samples (real claude/codex CLIs)", + ) parser.add_argument( "--broken", action="store_true", help="include broken samples" ) @@ -290,6 +314,7 @@ def main() -> None: or args.image or args.video or args.audio + or args.external or args.broken or args.e2e ) @@ -310,6 +335,8 @@ def main() -> None: samples.extend(VIDEO_SAMPLES) if not args.examples and (args.audio or args.all): samples.extend(AUDIO_SAMPLES) + if not args.examples and (args.external or args.all): + samples.extend(EXTERNAL_SAMPLES) if not args.examples and (args.broken or args.all): samples.extend(BROKEN_SAMPLES) if not args.examples and (args.e2e or args.all): diff --git a/examples/README.md b/examples/README.md index c1b7ea47..3b704f12 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,6 +7,9 @@ Examples are grouped by the main API surface they demonstrate. - `media/` contains image, video, speech, transcription, multimodal, embedding, and reranking examples. - `models/` contains model streaming, structured output, and provider examples. +- `external/` contains examples that use coding-agent CLIs (Claude Code, Codex, + fx, pi) where a model goes (experimental; run with + `run-examples.py --external`). Provider-specific model examples live under `models//`, such as `models/gateway/`, `models/openai/`, and `models/anthropic/`. diff --git a/examples/apps/durable_agent_temporal/backend/app/worker.py b/examples/apps/durable_agent_temporal/backend/app/worker.py index 58aac72f..fe10f33b 100644 --- a/examples/apps/durable_agent_temporal/backend/app/worker.py +++ b/examples/apps/durable_agent_temporal/backend/app/worker.py @@ -109,6 +109,8 @@ class DurableAgent(ai.Agent): async def loop(self, context: ai.Context) -> AsyncGenerator[ai.events.AgentEvent]: tools_data = [tool.model_dump(mode="json") for tool in context.tools] + # A durable step serializes the model: it must be an ai.Model. + assert isinstance(context.model, ai.Model) while context.keep_running(): result = await temporalio.workflow.execute_activity( llm_activity, diff --git a/examples/apps/durable_agent_workflows/backend/app/worker.py b/examples/apps/durable_agent_workflows/backend/app/worker.py index 0b88d193..23eed150 100644 --- a/examples/apps/durable_agent_workflows/backend/app/worker.py +++ b/examples/apps/durable_agent_workflows/backend/app/worker.py @@ -92,6 +92,8 @@ class DurableAgent(ai.Agent): async def loop(self, context: ai.Context) -> AsyncGenerator[ai.events.AgentEvent]: tools_data = [tool.model_dump(mode="json") for tool in context.tools] + # A durable step serializes the model: it must be an ai.Model. + assert isinstance(context.model, ai.Model) while context.keep_running(): result = await llm_step( context.model.model_dump(mode="json"), diff --git a/examples/external/__init__.py b/examples/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/external/agent_tools.py b/examples/external/agent_tools.py new file mode 100644 index 00000000..30e4fa35 --- /dev/null +++ b/examples/external/agent_tools.py @@ -0,0 +1,33 @@ +"""An ai.Agent gives its Python tools to a CLI, through the MCP bridge.""" + +import asyncio + +import ai + + +@ai.tool +async def secret_word(topic: str) -> str: + """Return the secret word for a topic.""" + return {"birds": "HERON"}.get(topic, "UNKNOWN") + + +async def main() -> None: + agent = ai.Agent(tools=[secret_word]) + messages = [ + ai.user_message( + "Use the secret_word tool for the topic 'birds'. " + "Reply with the word only." + ) + ] + async with ai.external.Fx() as harness, agent.run(harness, messages) as run: + async for event in run: + match event: + case ai.events.ToolEnd(): # an ai tool: the agent runs it + print(f"[ai tool] {event.tool_call.tool_name}") + case ai.events.TextDelta(): + print(event.chunk, end="", flush=True) + print() + print([message.role for message in run.messages]) + + +asyncio.run(main()) diff --git a/examples/external/as_tool.py b/examples/external/as_tool.py new file mode 100644 index 00000000..245a00c7 --- /dev/null +++ b/examples/external/as_tool.py @@ -0,0 +1,61 @@ +"""A coding-agent CLI as a tool: the external harness is a sub-agent.""" + +import asyncio +import tempfile + +import ai + +model = ai.get_model("anthropic/claude-sonnet-4.6") + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + # One block for the whole run: each call of the tool continues the + # same conversation, on one live CLI. + async with ai.external.ClaudeCode(cwd=tmp) as harness: + # The external harness's events stream up as partial tool results, + # and its last text becomes the tool result. To close the parent run + # stops the external harness's turn too. + @ai.tool + async def code(task: str) -> ai.SubAgentTool: + """Do a coding task in the project; returns what was done.""" + messages = [ + *harness.session.messages, + ai.user_message(task), + ] + async with ai.stream(harness, messages) as stream: + async for event in stream: + yield event + + orchestrator = ai.Agent(tools=[code]) + + messages = [ + ai.system_message( + "You plan; the code tool writes the code. Summarize the " + "result for the user in one sentence." + ), + ai.user_message( + "Create hello.py that prints 'hello', then run it." + ), + ] + + async with orchestrator.run(model, messages) as stream: + async for event in stream: + match event: + case ai.events.PartialToolCallResult( + value=ai.events.BuiltinToolEnd(tool_call=call) + ): + # The external harness at work, inside the tool + # call. + print(f" [external harness] {call.tool_name}") + case ai.events.TextDelta(): + print(event.chunk, end="", flush=True) + print() + + # The session has the external harness's own conversation. + count = len(harness.session.messages) + print(f"external harness messages: {count}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/basic.py b/examples/external/basic.py new file mode 100644 index 00000000..20b93395 --- /dev/null +++ b/examples/external/basic.py @@ -0,0 +1,30 @@ +"""Simplest external harness: one prompt to a CLI, where a model goes.""" + +import asyncio + +import ai + + +async def main() -> None: + messages = [ai.user_message("Reply with exactly: READY")] + + async with ( + ai.external.ClaudeCode() as harness, + ai.stream(harness, messages) as stream, + ): + async for event in stream: + match event: + case ai.events.TextDelta(): + print(event.chunk, end="", flush=True) + case ai.events.StreamEnd(): + print(f"\nfinish: {event.finish_reason}") + + if stream.usage is not None: + print( + f"tokens: {stream.usage.input_tokens} in / " + f"{stream.usage.output_tokens} out" + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/combine.py b/examples/external/combine.py new file mode 100644 index 00000000..407341b5 --- /dev/null +++ b/examples/external/combine.py @@ -0,0 +1,47 @@ +"""Run two external harnesses at the same time and read them as one stream.""" + +import asyncio +import tempfile +from collections.abc import AsyncGenerator + +import ai + + +async def tagged( + label: str, cwd: str, prompt: str +) -> AsyncGenerator[tuple[str, ai.events.Event]]: + async with ( + ai.external.ClaudeCode(cwd=cwd) as harness, + ai.stream(harness, [ai.user_message(prompt)]) as stream, + ): + async for event in stream: + yield label, event + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + # One external harness is one conversation: two at a time are two + # external harnesses, one in each stream. + async with ai.util.merge( + tagged( + "poem", + tmp, + "Write a two-line poem about rivers into poem.txt.", + ), + tagged( + "limerick", + tmp, + "Write a limerick about mountains into limerick.txt.", + ), + ) as events: + async for label, event in events: + match event: + case ai.events.BuiltinToolEnd(): + print(f"[{label}] {event.tool_call.tool_name}") + case ai.events.StreamEnd(): + print(f"[{label}] done ({event.finish_reason})") + print(f"[{label}] {event.message.text.strip()}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/custom_loop.py b/examples/external/custom_loop.py new file mode 100644 index 00000000..80dbd183 --- /dev/null +++ b/examples/external/custom_loop.py @@ -0,0 +1,99 @@ +"""Custom loop: an external harness turn as one step of your own agent loop.""" + +import asyncio +import pathlib +import tempfile +from collections.abc import AsyncGenerator + +import ai + +REASONING = ( + ai.events.ReasoningStart, + ai.events.ReasoningDelta, + ai.events.ReasoningEnd, +) + + +class CodeThenReview(ai.Agent): + def __init__(self, cwd: str) -> None: + super().__init__() + self.cwd = cwd + + async def loop( + self, context: ai.Context + ) -> AsyncGenerator[ai.events.AgentEvent]: + """The external harness does the task, then the model reviews it.""" + # Step 1: the external harness does the task. Each run is a new + # conversation: a new external harness. + async with ( + ai.external.ClaudeCode(cwd=self.cwd) as harness, + ai.stream(harness, [context.messages[-1]]) as work, + ): + async for event in work: + if not isinstance(event, REASONING): + yield event + + # Step 2: evidence from the directory, not the external harness's word + # for it. Only the external harness's answer joins the agent's context, + # not its tool calls: they name tools that this model does not have. + source = pathlib.Path(self.cwd, "hello.py").read_text() + process = await asyncio.create_subprocess_exec( + "python3", + "hello.py", + cwd=self.cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + async with asyncio.timeout(30): + stdout, _ = await process.communicate() + except TimeoutError: + process.kill() + raise + context.add(ai.assistant_message(work.text)) + context.add( + ai.user_message( + "You are the reviewer. hello.py contains:\n" + + source + + f"\nRunning it printed: {stdout.decode().strip()!r} " + + f"(exit code {process.returncode}).\n" + + "In one sentence: does it meet the task? Start with YES " + + "or NO." + ) + ) + + # Step 3: a model call, like any other agent step. + async with ai.stream(context=context) as review: + async for event in review: + yield event + context.add(review.message) + + +async def main() -> None: + model = ai.get_model("anthropic/claude-sonnet-4.6") + + with tempfile.TemporaryDirectory() as tmp: + my_agent = CodeThenReview(tmp) + + async with my_agent.run( + model, + [ + ai.user_message( + "Create hello.py that prints 'hello', then run it." + ) + ], + ) as stream: + async for event in stream: + match event: + case ai.events.BuiltinToolEnd(): + print(f"[external harness] {event.tool_call.tool_name}") + case ai.events.TextDelta(): + print(event.chunk, end="", flush=True) + case ai.events.StreamEnd(): + print() + + print(f"review: {stream.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/direct_control.py b/examples/external/direct_control.py new file mode 100644 index 00000000..58a35a70 --- /dev/null +++ b/examples/external/direct_control.py @@ -0,0 +1,39 @@ +"""Direct control: send, steer, read the feed, interrupt.""" + +import asyncio + +import ai + + +async def main() -> None: + async with ai.external.Codex() as harness: + await harness.session.send( + ai.user_message("Count from 1 to 20 slowly, one number per line.") + ) + + steered = False + async for event in harness.session.events(): + match event: + case ai.events.TextDelta(): + print(event.chunk, end="", flush=True) + if not steered and "5" in event.chunk: + steered = True + # A steer: the CLI decides how to fit it in. + await harness.session.send( + ai.user_message("Stop at 8 instead.") + ) + case ai.external.MessageEvent(message=message): + print(f"\n[{message.role}] {message.text[:40]!r}") + case ai.events.StreamEnd(finish_reason="stop"): + break + + await harness.session.send( + ai.user_message("Now count from 100 down to 1, slowly.") + ) + await asyncio.sleep(3) + await harness.session.interrupt() # stop the turn; the session stays + # The cut reply is in the conversation, with the reason in its data. + print(f"\n{harness.session.messages[-1].provider_metadata}") + + +asyncio.run(main()) diff --git a/examples/external/handoff.py b/examples/external/handoff.py new file mode 100644 index 00000000..ce60f4be --- /dev/null +++ b/examples/external/handoff.py @@ -0,0 +1,47 @@ +"""Hand a conversation from one coding-agent CLI to a different one.""" + +import asyncio +import pathlib +import tempfile + +import ai + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + # 1. A conversation with Claude Code. It does real work. + messages = [ + ai.user_message( + "Create notes.txt containing exactly: milk, eggs. Also " + "remember: the deadline is FRIDAY. Reply with exactly: OK" + ) + ] + async with ai.external.ClaudeCode(cwd=tmp) as claude: + reply = await ai.experimental_generate(claude, messages) + print( + f"Claude Code: {reply.text.strip()} " + f"({len(reply.builtin_tool_calls)} tool call(s))" + ) + + # 2. Codex continues it, in the same directory. As with a model: + # give the history to the new external harness. Its new session + # starts with that past, thus it knows what was said and what was + # done. + messages = [ + *messages, + reply, + ai.user_message( + "What is the deadline, and what did we put in notes.txt? " + "Then append 'bread' to notes.txt. Reply in one line." + ), + ] + async with ai.external.Codex(cwd=tmp) as codex: + reply = await ai.experimental_generate(codex, messages) + print("Codex:", reply.text.strip()) + + notes = pathlib.Path(tmp, "notes.txt").read_text() + print("notes.txt:", notes.strip().replace("\n", " / ")) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/resume.py b/examples/external/resume.py new file mode 100644 index 00000000..ba745076 --- /dev/null +++ b/examples/external/resume.py @@ -0,0 +1,44 @@ +"""Save a session id, and continue the conversation in a new process.""" + +import asyncio +import tempfile + +import ai + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + async with ai.external.ClaudeCode(cwd=tmp) as harness: + await ai.experimental_generate( + harness, + [ + ai.user_message( + "Remember this codeword: PLATYPUS. Reply OK." + ) + ], + ) + + # The session id names the conversation in the CLI. Put it in your + # database. + session_id = harness.session.id + + # ... later, another process entirely ... + + # A new external harness with the same settings continues the + # session. The CLI has the conversation: send what is new. The block + # keeps the CLI alive between the calls. + async with ai.external.ClaudeCode(cwd=tmp, resume=session_id) as again: + for question in ( + "What codeword did I give you? Reply with the word only.", + "Spell it backwards. Reply with the word only.", + ): + messages = [ + *again.session.messages, + ai.user_message(question), + ] + answer = await ai.experimental_generate(again, messages) + print(answer.text) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/steering.py b/examples/external/steering.py new file mode 100644 index 00000000..a4486094 --- /dev/null +++ b/examples/external/steering.py @@ -0,0 +1,51 @@ +"""Change an external harness's mind while it works: send during the turn.""" + +import asyncio +import pathlib +import tempfile + +import ai + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + steer = ai.user_message( + "Stop creating numbered files. Write the word STEERED into " + "steered.txt instead, then finish." + ) + + async with ai.external.ClaudeCode(cwd=tmp) as harness: + await harness.session.send( + ai.user_message( + "Create files step1.txt through step40.txt, one at a " + "time, each containing its own number." + ) + ) + + sent = accepted = False + async for event in harness.session.events(): + match event: + case ai.events.BuiltinToolStart() if not sent: + sent = True + print( + f"it started working ({event.tool_name}): steering" + ) + # A message during a turn steers it. The CLI takes it + # at its next step, between tool calls. + await harness.session.send(steer) + case ai.external.MessageEvent(): + # The steer is in the conversation: the reply before + # it ended, and a new reply starts. + accepted = accepted or event.message.id == steer.id + case ai.events.StreamEnd() if accepted: + break + + print([message.role for message in harness.session.messages]) + + # How much of the first plan it finished depends on where its next + # step was. + print("steered.txt:", pathlib.Path(tmp, "steered.txt").read_text()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/streaming.py b/examples/external/streaming.py new file mode 100644 index 00000000..f0f2de84 --- /dev/null +++ b/examples/external/streaming.py @@ -0,0 +1,71 @@ +"""Watch an external harness's turn as it happens, then continue it.""" + +import asyncio +import tempfile + +import ai + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + messages = [ + ai.user_message( + "Write the word HELLO into greeting.txt, then read it back " + "and reply with what it contains." + ) + ] + + async with ai.external.ClaudeCode(cwd=tmp) as harness: + async with ai.stream(harness, messages) as stream: + async for event in stream: + match event: + case ai.events.TextDelta(): + # Live output: print text as it is generated. + print(event.chunk, end="", flush=True) + case ai.events.TextEnd( + message=message, block_id=block_id + ): + # The whole block, already assembled on the message. + part = next( + p + for p in message.parts + if isinstance(p, ai.messages.TextPart) + and p.id == block_id + ) + print(f"\n[text done: {len(part.text)} chars]") + case ai.events.BuiltinToolEnd(tool_call=call): + # The CLI's own tool: the complete call, with the + # native name and the arguments as a JSON string. + print( + f"[tool call] {call.tool_name} " + f"{call.tool_args[:60]}" + ) + case ai.events.BuiltinToolResult(result=result): + # The CLI ran the tool: the result is a part of the + # same assistant message, not a tool message. + print( + f"[tool result] {result.tool_name}: " + f"{str(result.result)[:80]}" + ) + case ai.events.StreamEnd(): + print(f"[turn over] finish={event.finish_reason}") + + # Everything above is also on the message. + print(f"\ntext: {stream.text.strip()!r}") + print(f"usage: {stream.usage}") + print(f"parts in the message: {len(stream.message.parts)}") + + # As with a model: send the history back with what is new. The + # session gives only the new part to the CLI. + messages = [ + *messages, + stream.message, + ai.user_message( + "Which file did you write? Reply with its name only." + ), + ] + print((await ai.experimental_generate(harness, messages)).text) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/external/until_done.py b/examples/external/until_done.py new file mode 100644 index 00000000..1b00e69b --- /dev/null +++ b/examples/external/until_done.py @@ -0,0 +1,83 @@ +"""Your own loop around an external harness: prompt, check, prompt again.""" + +import asyncio +import dataclasses +import tempfile +from collections.abc import AsyncGenerator + +import ai + +EXPECTED = ( + "1\n2\nFizz\n4\nBuzz\nFizz\n7\n8\nFizz\nBuzz\n11\nFizz\n13\n14\nFizzBuzz" +) + + +@dataclasses.dataclass +class Checked: + attempt: int + passed: bool + detail: str + + +# The CLI owns the loop inside a turn. The loop around the turns is plain +# Python: run a turn, check the result in the directory, and send the failure +# back until the check passes. +async def until_passes( + harness: ai.external.ExternalHarness, task: str, *, attempts: int = 3 +) -> AsyncGenerator[ai.events.Event | Checked]: + prompt = task + for attempt in range(1, attempts + 1): + messages = [ + *harness.session.messages, + ai.user_message(prompt), + ] + async with ai.stream(harness, messages) as stream: + async for event in stream: + if not isinstance(event, ai.events.ReasoningDelta): + yield event + + process = await asyncio.create_subprocess_exec( + "python3", + "fizzbuzz.py", + cwd=harness.cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except TimeoutError: + process.kill() + raise + output = stdout.decode().strip() + failure = stderr.decode().strip() + passed = process.returncode == 0 and output == EXPECTED + yield Checked(attempt, passed, output or failure) + if passed: + return + + prompt = ( + "`python3 fizzbuzz.py` should print exactly:\n" + + EXPECTED + + f"\n\nIt printed (exit code {process.returncode}):\n" + + (output or failure) + + "\n\nFix fizzbuzz.py." + ) + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + # Deliberately vague, so the check has something to catch. + task = "Write fizzbuzz.py: a FizzBuzz program." + + async with ai.external.ClaudeCode(cwd=tmp) as harness: + async for item in until_passes(harness, task): + if isinstance(item, Checked): + verdict = "passed" if item.passed else "failed" + print(f"check {item.attempt}: {verdict}") + elif isinstance(item, ai.events.BuiltinToolEnd): + print(f" [external harness] {item.tool_call.tool_name}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 6bfd5968..8b70febc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ [project.optional-dependencies] anthropic = ["anthropic>=0.83.0,<2.0.0"] +claude-code = ["claude-agent-sdk>=0.2.179,<0.3"] mcp = ["mcp>=2.0.0,<3.0.0"] openai = ["openai>=2.34.0,<4.0.0"] otel = [ @@ -69,6 +70,7 @@ bump = true [dependency-groups] dev = [ "anthropic>=0.83.0", + "claude-agent-sdk>=0.2.179,<0.3", "mcp>=2.0.0,<3.0.0", "python-dotenv>=1.2.1", "pytest>=8.0", diff --git a/src/ai/__init__.py b/src/ai/__init__.py index b10faf3e..bee92c7b 100644 --- a/src/ai/__init__.py +++ b/src/ai/__init__.py @@ -2,6 +2,7 @@ durable, errors, experimental_telemetry, + external, models, ops, providers, @@ -201,6 +202,7 @@ "events", "experimental_generate", "experimental_telemetry", + "external", "file_part", "get_hook_registry", "get_model", diff --git a/src/ai/agents/agent.py b/src/ai/agents/agent.py index ec53583c..4cd6d14a 100644 --- a/src/ai/agents/agent.py +++ b/src/ai/agents/agent.py @@ -1042,7 +1042,7 @@ async def _iterate(self) -> AsyncGenerator[events_.ToolCallResult]: class Context(pydantic.BaseModel): """Everything that goes into the LLM.""" - model: models.Model + model: pydantic.SerializeAsAny[models.Model | models.ModelLike] messages: list[types.messages.Message] tools: list[Tool] output_type: type[pydantic.BaseModel] | None = pydantic.Field( @@ -1493,7 +1493,7 @@ async def loop( @overload def run( self, - model: models.Model, + model: models.ModelLike, messages: list[types.messages.Message], *, params: models.InferenceRequestParams | None = None, @@ -1502,7 +1502,7 @@ def run( @overload def run[T: pydantic.BaseModel]( self, - model: models.Model, + model: models.ModelLike, messages: list[types.messages.Message], *, output_type: type[T], @@ -1511,7 +1511,7 @@ def run[T: pydantic.BaseModel]( ) -> AbstractAsyncContextManager[AgentStream[T]]: ... def run( self, - model: models.Model, + model: models.ModelLike, messages: list[types.messages.Message], *, output_type: type[pydantic.BaseModel] | None = None, @@ -1556,7 +1556,7 @@ def run( @contextlib.asynccontextmanager async def _run( self, - model: models.Model, + model: models.ModelLike, messages: list[types.messages.Message], *, output_type: type[pydantic.BaseModel] | None, diff --git a/src/ai/external/__init__.py b/src/ai/external/__init__.py new file mode 100644 index 00000000..56a817d3 --- /dev/null +++ b/src/ai/external/__init__.py @@ -0,0 +1,80 @@ +"""Claude Code, Codex, Fx and Pi coding-agent CLIs as external harnesses. + +Experimental: not part of the stable API, may change or be removed. + +An external harness is a CLI in the shape of a model: settings and a provider. +Open it, and use it where a model goes, with the whole history:: + + import ai + + messages = [ai.user_message("Fix the tests")] + async with ai.external.Codex(cwd=".") as harness: + async with ai.stream(harness, messages) as s: + async for event in s: + ... + + # ai tools reach the CLI through a local MCP bridge + async with ai.external.Codex(cwd=".") as harness: + async with ai.Agent(tools=[search_docs]).run(harness, messages) as run: + ... + +The block holds one live CLI, and the model calls in it run on that CLI. A model +call outside the block raises. One external harness is one conversation: it +comes with one session, `harness.session`, which has the state and gives direct +control:: + + async with ai.external.Codex(cwd=".") as harness: + await harness.session.send(ai.user_message("Fix the tests")) + async for event in harness.session.events(): + ... # send() again to steer, harness.session.interrupt() to stop + +`Codex()` has a new session; `Codex(resume=session_id)` continues one that the +CLI has, and `Codex(history=messages)` has a new one that starts with a past. +The CLI's own tools come as `ai.events.BuiltinTool*` events. The state is plain +data: `harness.session.id` is the CLI's id for the session, and +`harness.session.messages` holds the conversation in `ai` messages. The provider +says where the CLI is: `ai.external.CodexProvider(executable=..., env=...)`. +""" + +from ._errors import ( + ExecutableMissingError, + ExternalHarnessError, + NotAuthenticatedError, + TranscriptMismatchError, + UnsupportedError, +) +from ._harness import ExternalHarness, ExternalHarnessProvider +from ._session import ( + ExternalHarnessEvent, + ExternalHarnessMessageSpanData, + ExternalHarnessSessionSpanData, + MessageEvent, + Session, +) +from .claude_code import ClaudeCode, ClaudeCodeProvider +from .codex import Codex, CodexProvider +from .fx import Fx, FxProvider +from .pi import Pi, PiProvider + +__all__ = [ + "ClaudeCode", + "ClaudeCodeProvider", + "Codex", + "CodexProvider", + "ExecutableMissingError", + "ExternalHarness", + "ExternalHarnessError", + "ExternalHarnessEvent", + "ExternalHarnessMessageSpanData", + "ExternalHarnessProvider", + "ExternalHarnessSessionSpanData", + "Fx", + "FxProvider", + "MessageEvent", + "NotAuthenticatedError", + "Pi", + "PiProvider", + "Session", + "TranscriptMismatchError", + "UnsupportedError", +] diff --git a/src/ai/external/_bridge.py b/src/ai/external/_bridge.py new file mode 100644 index 00000000..cf2c92a8 --- /dev/null +++ b/src/ai/external/_bridge.py @@ -0,0 +1,265 @@ +"""The MCP bridge: `ai` tools, served to a coding-agent CLI. + +An external harness's CLI runs its own loop and its own tools. To give it the +tools an `ai.Agent` holds, the external harness serves them over MCP on +localhost, and the CLI connects to that server like any other. + +The server does not run a tool. It hands each call to `on_call` and waits for +the answer. The external harness turns the call into a normal `ToolCallPart`, +ends its stream with `finish_reason="tool_call"`, and `ai.Agent` runs the tool. +The next `stream()` call brings the result, and the external harness answers the +waiting request with it. While the request waits, the CLI waits too. + +Only what an MCP client needs is implemented: streamable HTTP, POST only, +plain JSON responses (no SSE), and the methods `initialize`, `ping`, +`tools/list` and `tools/call`. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import secrets +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ..types import tools as tools_ + +# The MCP server name the CLI sees. Claude Code calls a tool +# `mcp__ai_sdk__`. +SERVER_NAME = "ai_sdk" + +PROTOCOL_VERSION = "2025-06-18" +TOKEN_ENV = "AI_PYTHON_MCP_TOKEN" + +# `on_call(name, arguments)` returns `(result, is_error)`. A result that is +# not a `str` goes to the CLI as JSON text. +OnCall = Callable[[str, dict[str, Any]], Awaitable[tuple[Any, bool]]] + + +class Bridge: + def __init__(self, on_call: OnCall) -> None: + self._on_call = on_call + self.token = secrets.token_urlsafe(32) + self._tools: dict[str, tools_.Tool] = {} + self._server: asyncio.Server | None = None + self._connections: set[asyncio.Task[None]] = set() + + @property + def url(self) -> str: + if self._server is None: + raise RuntimeError("bridge is not started") + host, port = self._server.sockets[0].getsockname()[:2] + return f"http://{host}:{port}/mcp" + + @property + def names(self) -> set[str]: + return set(self._tools) + + def set_tools(self, tools: list[tools_.Tool]) -> None: + """Serve these tools from now on. + + A CLI reads the list when it connects, so a tool added after that + may stay unseen until the next connection. + """ + self._tools = { + t.name: t for t in tools if t.kind == "function" and t.spec + } + + async def start(self) -> None: + if self._server is None: + self._server = await asyncio.start_server( + self._serve, host="127.0.0.1", port=0 + ) + + async def close(self) -> None: + if self._server is None: + return + self._server.close() + for task in list(self._connections): + task.cancel() + for task in list(self._connections): + with contextlib.suppress(BaseException): + await task + with contextlib.suppress(Exception): + await self._server.wait_closed() + self._server = None + + # -- HTTP ----------------------------------------------------------------- + + async def _serve( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + task = asyncio.current_task() + assert task is not None + self._connections.add(task) + host, port = writer.get_extra_info("sockname")[:2] + authority = f"{host}:{port}" + authorization = f"Bearer {self.token}".encode("ascii") + try: + while True: + request = await _read_head(reader) + if request is None: + return + method, headers = request + # The native clients use the literal loopback authority and + # send no Origin. Neither check replaces client authentication. + if headers.get("host") != authority or "origin" in headers: + await _respond(writer, 403, None) + return + if not secrets.compare_digest( + headers.get("authorization", "").encode("latin-1"), + authorization, + ): + await _respond(writer, 401, None) + return + body = await _read_body(reader, headers) + if method != "POST": + # No SSE stream to offer, and a session needs no teardown. + status = 200 if method == "DELETE" else 405 + await _respond(writer, status, None) + continue + reply = await self._dispatch(body) + await _respond(writer, 200 if reply is not None else 202, reply) + except (ConnectionError, asyncio.IncompleteReadError): + return + finally: + self._connections.discard(task) + with contextlib.suppress(Exception): + writer.close() + + # -- JSON-RPC ------------------------------------------------------------- + + async def _dispatch(self, body: bytes) -> Any: + try: + payload = json.loads(body or b"null") + except json.JSONDecodeError: + return _error(None, -32700, "parse error") + if isinstance(payload, list): + replies = [await self._one(item) for item in payload] + kept = [r for r in replies if r is not None] + return kept or None + return await self._one(payload) + + async def _one(self, message: Any) -> Any: + if not isinstance(message, dict) or "method" not in message: + # A response from the client (we send no requests) or junk. + return None + method, rid = message["method"], message.get("id") + if rid is None: + return None # a notification: nothing to answer + params = message.get("params") or {} + if method == "initialize": + return _result( + rid, + { + "protocolVersion": params.get( + "protocolVersion", PROTOCOL_VERSION + ), + "capabilities": {"tools": {"listChanged": False}}, + "serverInfo": {"name": SERVER_NAME, "version": "0"}, + }, + ) + if method == "ping": + return _result(rid, {}) + if method == "tools/list": + return _result( + rid, + { + "tools": [ + { + "name": t.name, + "description": (t.spec and t.spec.description) + or "", + "inputSchema": (t.spec and t.spec.params) + or {"type": "object"}, + } + for t in self._tools.values() + ] + }, + ) + if method == "tools/call": + name = str(params.get("name", "")) + if name not in self._tools: + return _error(rid, -32602, f"unknown tool: {name}") + arguments = params.get("arguments") or {} + result, is_error = await self._on_call(name, dict(arguments)) + text = result if isinstance(result, str) else json.dumps(result) + return _result( + rid, + { + "content": [{"type": "text", "text": text}], + "isError": is_error, + }, + ) + return _error(rid, -32601, f"method not found: {method}") + + +def _result(rid: Any, result: Any) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": rid, "result": result} + + +def _error(rid: Any, code: int, message: str) -> dict[str, Any]: + return { + "jsonrpc": "2.0", + "id": rid, + "error": {"code": code, "message": message}, + } + + +async def _read_head( + reader: asyncio.StreamReader, +) -> tuple[str, dict[str, str]] | None: + """Read the HTTP head before accepting a body.""" + try: + head = await reader.readuntil(b"\r\n\r\n") + except asyncio.IncompleteReadError: + return None + lines = head.decode("latin-1").split("\r\n") + method = lines[0].split(" ", 1)[0].upper() + headers = {} + for line in lines[1:]: + if ":" in line: + key, value = line.split(":", 1) + key = key.strip().lower() + if key in headers and key in ("host", "authorization", "origin"): + return None + headers[key] = value.strip() + return method, headers + + +async def _read_body( + reader: asyncio.StreamReader, headers: dict[str, str] +) -> bytes: + if headers.get("transfer-encoding", "").lower() == "chunked": + body = b"" + while True: + size = int((await reader.readuntil(b"\r\n")).strip(), 16) + if size == 0: + await reader.readuntil(b"\r\n") + break + body += await reader.readexactly(size) + await reader.readexactly(2) + return body + length = int(headers.get("content-length", "0") or 0) + return await reader.readexactly(length) if length else b"" + + +async def _respond( + writer: asyncio.StreamWriter, status: int, payload: Any +) -> None: + reason = { + 200: "OK", + 202: "Accepted", + 401: "Unauthorized", + 403: "Forbidden", + 405: "Method Not Allowed", + }[status] + body = b"" if payload is None else json.dumps(payload).encode() + head = [f"HTTP/1.1 {status} {reason}", f"Content-Length: {len(body)}"] + if payload is not None: + head.append("Content-Type: application/json") + writer.write(("\r\n".join(head) + "\r\n\r\n").encode() + body) + await writer.drain() diff --git a/src/ai/external/_errors.py b/src/ai/external/_errors.py new file mode 100644 index 00000000..b4ff9780 --- /dev/null +++ b/src/ai/external/_errors.py @@ -0,0 +1,82 @@ +"""The errors an external harness raises. + +Three cases have their own class, because a caller fixes them differently. +Everything else is an `ExternalHarnessError` with a message that says what +happened. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .. import errors + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class ExternalHarnessError(errors.AIError): + """An external harness or its CLI failed.""" + + +class UnsupportedError(ExternalHarnessError): + """The CLI does not support the requested operation.""" + + +class ExecutableMissingError(ExternalHarnessError): + """The CLI is not installed, or `executable=` does not run.""" + + def __init__(self, kind: str, executable: str, install: str) -> None: + super().__init__( + f"{kind} needs {executable!r}, which does not run here. " + f"Install it with: {install}" + ) + self.kind = kind + self.executable = executable + + +class NotAuthenticatedError(ExternalHarnessError): + """The CLI has no usable credentials.""" + + def __init__(self, kind: str, detail: str, hint: str) -> None: + super().__init__(f"{kind} is not authenticated: {detail}\n{hint}") + self.kind = kind + self.detail = detail + + +class TranscriptMismatchError(ExternalHarnessError): + """A session cannot take the messages given to `stream()`. + + A session holds a conversation in its CLI. It can continue it, but it cannot + rewrite it: use `harness.session.messages` as the history. For another + conversation, use a new external harness. A history must always end with a + user message, or with the results of tool calls that the session made. + """ + + +# Phrases that mean "no usable credentials" in any CLI. Narrow on purpose: a +# 403 is authorization, not authentication. A wire adds its CLI's own phrases. +_AUTH_MARKERS = ( + "401", + "unauthorized", + "missing bearer", + "not logged in", + "invalid_api_key", + "invalid api key", + "authentication_error", + "failed to authenticate", + "no api key", +) + + +def turn_failed( + kind: str, detail: str, hint: str, markers: Sequence[str] = () +) -> ExternalHarnessError: + """Name a failed turn for what the caller has to fix. + + `markers` are the CLI's own phrases for "no usable credentials". + """ + detail = (detail or "unknown error")[:500] + if any(m in detail.lower() for m in (*_AUTH_MARKERS, *markers)): + return NotAuthenticatedError(kind, detail, hint) + return ExternalHarnessError(f"{kind} turn failed: {detail}") diff --git a/src/ai/external/_handoff_output.py b/src/ai/external/_handoff_output.py new file mode 100644 index 00000000..ae7a78c4 --- /dev/null +++ b/src/ai/external/_handoff_output.py @@ -0,0 +1,79 @@ +"""Model-facing tool output for the external CLIs' history stores.""" + +import datetime +import decimal +import json +import uuid + +import pydantic + +from ..types import messages + + +def format_result( + part: messages.ToolResultPart | messages.BuiltinToolReturnPart, +) -> str: + """Render a native tool result as text, never falling back on dump errors. + + Multipart output keeps its text and names the media types in place of + files. A message bundle contributes only its last nonempty assistant text, + matching the sub-agent's projection without serializing its transcript. + Ordinary Pydantic models are JSON-encoded only if their fields are plain + data or known scalars; unsupported wrappers are omitted as a whole. + """ + if isinstance(part, messages.BuiltinToolReturnPart): + return ( + part.result + if isinstance(part.result, str) + else json.dumps(part.result) + ) + + value = part.get_model_input() + if isinstance(value, messages.ContentOutput | messages.MessageBundle): + value = _project(value) + return ( + value if isinstance(value, str) else json.dumps(value, default=_project) + ) + + +def _project(value: object) -> object: + if isinstance(value, messages.ContentOutput): + return "\n".join( + item.text + if isinstance(item, messages.TextPart) + else f"[file: {item.media_type}]" + for item in value.value + ) + if isinstance(value, messages.MessageBundle): + return next( + ( + m.text + for m in reversed(value.messages) + if m.role == "assistant" and m.text + ), + "", + ) + if isinstance(value, messages.Message | messages.ToolResultPart): + return "[unsupported structured output omitted]" + if isinstance(value, pydantic.BaseModel): + # Pydantic's serializer would expand nested models without calling + # this hook. Only let it see ordinary data and known scalar types; + # a nested model could be a transcript containing raw tool results. + try: + json.dumps(dict(value), default=_check_scalar) + except (TypeError, ValueError): + return "[unsupported structured output omitted]" + return value.model_dump(mode="json", exclude_computed_fields=True) + raise TypeError(f"Unsupported handoff output type: {type(value).__name__}") + + +def _check_scalar(value: object) -> None: + if not isinstance( + value, + datetime.date + | datetime.time + | datetime.timedelta + | decimal.Decimal + | uuid.UUID, + ): + raise TypeError("unsupported structured output field") diff --git a/src/ai/external/_harness.py b/src/ai/external/_harness.py new file mode 100644 index 00000000..d89a61aa --- /dev/null +++ b/src/ai/external/_harness.py @@ -0,0 +1,252 @@ +"""`ExternalHarness` and its provider: a CLI in the shape of a model. + +An `ai.Model` is an id and a provider, and `ai.stream` calls +`model.provider.stream(model, messages)`. An external harness has the same +shape: + +- `ExternalHarness` is the settings of the CLI. It comes with one `Session`, + `harness.session`, and `async with harness` opens it. +- `ExternalHarnessProvider` is the CLI behind it: where it is and how it runs. + Its `stream` makes one model call. + +The state and direct control are in the session. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +from collections.abc import AsyncGenerator, Sequence +from typing import TYPE_CHECKING, Any, ClassVar, Self + +import pydantic + +from ..types import events as events_ +from ..types import messages as messages_ +from ..types import tools as tools_ +from . import _errors as errors +from . import _session as session_ +from . import _wire + +if TYPE_CHECKING: + from ..models.core import params as params_ + + +class ExternalHarnessProvider(pydantic.BaseModel): + """The CLI behind an external harness: where it is and how it runs. + + External harnesses can share one. A concrete provider (`CodexProvider`, ...) + adds the private wire (`_wire.Wire`) that speaks its CLI's protocol. + """ + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + name: str + """The CLI's name, for telemetry.""" + executable: str | None = None + """The CLI command or path. None: the usual command on PATH.""" + env: dict[str, str] = pydantic.Field(default_factory=dict) + """Added to the CLI's environment: keys, base URLs, config dirs.""" + + def _make_wire(self, harness: Any) -> _wire.Wire: + """Return a new wire for `harness`. Concrete providers implement it.""" + raise NotImplementedError + + async def stream( + self, + harness: ExternalHarness, + messages: list[messages_.Message], + *, + tools: Sequence[tools_.Tool] | None = None, + output_type: type[pydantic.BaseModel] | None = None, + params: params_.InferenceRequestParams | None = None, + ) -> AsyncGenerator[events_.Event]: + """Answer `messages` with one assistant message, as a model does. + + Use `ai.stream(harness, messages)` or `ai.Agent.run(harness, ...)` in + the block of the external harness (`async with harness`): the call + runs on its live CLI. Outside the block, the call raises. A new + session starts with the past in `messages` (lossy). + + The message ends with `finish_reason="tool_call"` when the CLI calls + `ai` tools from `tools=`; the CLI waits for the results, which come with + the next call. To cancel the stream or to stop reading it interrupts the + turn. `params` has no effect: use external harness settings. + """ + if output_type is not None: + raise errors.UnsupportedError( + "external harnesses do not support output_type yet" + ) + session = harness.session + if not session._open: + raise errors.ExternalHarnessError( + "a model call of an external harness needs an open external " + "harness: use `async with harness`" + ) + if session._stream_lock.locked(): + raise errors.ExternalHarnessError( + "an external harness runs one stream() at a time" + ) + seeds = not session.messages + new = session._sync(messages) + seeds = seeds and bool(session.messages) + async with session._stream_lock: + if tools is not None: + harness.tools = [t for t in tools if t.kind == "function"] + run = await session._start_for_stream(seeds=seeds) + queue: asyncio.Queue[Any] = asyncio.Queue() + session._stream_queues.add(queue) + ended = False + try: + for message in new: + await session._send(message) + started = False + while True: + event = await queue.get() + if event is session_._END: + raise errors.ExternalHarnessError( + "the external harness closed during stream()" + ) + if isinstance(event, BaseException): + raise event + if isinstance(event, session_.MessageEvent): + continue + if isinstance(event, events_.StreamStart): + started = True + if not started: + continue # the end of a message that began earlier + if isinstance(event, events_.StreamEnd): + ended = True + error = run.failures.pop(event.message.id, None) + if error is not None: + raise error + yield event + return + yield event + except (asyncio.CancelledError, GeneratorExit) as exc: + if not ended: + # Shielded: a second cancel must not cut the interrupt. + task = asyncio.ensure_future(session.interrupt()) + try: + await asyncio.shield(task) + except asyncio.CancelledError: + if isinstance(exc, GeneratorExit): + raise + except Exception: + pass + raise + finally: + session._stream_queues.discard(queue) + + async def generate( + self, + harness: ExternalHarness, + messages: list[messages_.Message], + *, + tools: Sequence[tools_.Tool] | None = None, + output_type: type[pydantic.BaseModel] | None = None, + params: params_.InferenceRequestParams | None = None, + ) -> messages_.Message: + """Not implemented: `ai.experimental_generate` drains `stream()`.""" + raise NotImplementedError + + +@dataclasses.dataclass(kw_only=True, eq=False) +class ExternalHarness: + """A coding-agent CLI, in the shape of an `ai.Model`. + + One external harness is one conversation. It comes with one session, + `harness.session`, and it has one live CLI at a time. For two + conversations, make two external harnesses. + + Open it, and use it where a model goes. As with a model, give the whole + history:: + + async with ai.external.Codex(cwd=".") as harness: + async with ai.stream(harness, messages) as s: ... + await harness.session.send(ai.user_message("And the tests?")) + + The block holds the live CLI: the model calls in it (`ai.stream`, + `ai.Agent.run`) run on that CLI, and the session gives direct control. A + model call outside the block raises. An external harness that closed + opens again, and continues its session. + + The fields are the settings of the CLI. The conversation is in the + session: `harness.session.id` and `harness.session.messages`. To continue + a conversation later, save `harness.session.id` and give it to a new + external harness as `resume`. + + A copy or a pickle is a closed external harness with the same settings + and the same conversation, not the live CLI. For a new conversation, + make a new external harness. + """ + + harness_class_id: ClassVar[str] + """The name of the kind of CLI, for telemetry and errors.""" + + provider: ExternalHarnessProvider + """The CLI that runs the session.""" + model_name: str | None = None + """The CLI's model. None: the CLI default.""" + cwd: str = dataclasses.field(default_factory=os.getcwd) + """Where the CLI runs.""" + instructions: str | None = None + """Added to the CLI's system prompt when the session starts.""" + tools: list[tools_.Tool] = dataclasses.field(default_factory=list) + """`ai` function tools that the CLI can call through the MCP bridge.""" + + resume: dataclasses.InitVar[str | None] = None + """The id of a session that the CLI has: the session continues it. + + None: a new session. Only the constructor takes it; read the id of the + session from `harness.session.id`. + """ + history: dataclasses.InitVar[list[messages_.Message] | None] = None + """The conversation so far. A new session starts with it (lossy). + + With `resume`, the CLI has it already and nothing goes to the CLI. Only + the constructor takes it; read the conversation from + `harness.session.messages`. + """ + + session: session_.Session = dataclasses.field(init=False, repr=False) + """The session of this external harness: its state and direct control.""" + + def __post_init__( + self, resume: str | None, history: list[messages_.Message] | None + ) -> None: + if type(self) is ExternalHarness: + raise TypeError( + "ExternalHarness must be subclassed, " + "for example ai.external.Codex" + ) + self.session = session_.Session(self, resume, history) + + def __getstate__(self) -> dict[str, Any]: + # For `copy` and `pickle`: the settings and the conversation, not the + # live CLI. + session = self.session + return {**self.__dict__, "session": (session.id, session.messages)} + + def __setstate__(self, state: dict[str, Any]) -> None: + session_id, messages = state["session"] + self.__dict__.update(state) + self.session = session_.Session(self, session_id, messages) + + async def __aenter__(self) -> Self: + """Open the external harness: its model calls and its session work. + + The CLI starts with the first `session.send()` or the first model + call, not here. + """ + self.session._enter() + return self + + async def __aexit__(self, *exc_info: object) -> None: + """Stop the CLI and close. The state of the session stays.""" + await self.session._close() + + @property + def id(self) -> str: + return self.model_name or "default" diff --git a/src/ai/external/_jsonrpc.py b/src/ai/external/_jsonrpc.py new file mode 100644 index 00000000..5149f8da --- /dev/null +++ b/src/ai/external/_jsonrpc.py @@ -0,0 +1,251 @@ +"""JSON-RPC over a CLI process on stdio. + +Line-delimited JSON, request/response futures, and two callbacks for what +a server can start. The wire supplies the protocol-specific meaning. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +from collections.abc import Awaitable, Callable +from typing import Any + +from . import _errors as errors +from . import _process + +NotifyHandler = Callable[[str, dict[str, Any]], Awaitable[None]] +RequestHandler = Callable[[Any, str, dict[str, Any]], Awaitable[None]] + + +class JsonRpcError(Exception): + def __init__(self, error: dict[str, Any]) -> None: + super().__init__(error.get("message") or json.dumps(error)) + self.code = error.get("code") + self.data = error.get("data") + + +class JsonRpcPeer: + def __init__(self, process: _process.Process, *, kind: str) -> None: + self._process = process + self._kind = kind + self._next_id = 0 + self._waiters: dict[int, asyncio.Future[Any]] = {} + self._completions: dict[ + int, Callable[[int, asyncio.Future[Any]], None] + ] = {} + self._pump: asyncio.Task[None] | None = None + self._closed = False + self._on_crash: Callable[[BaseException], None] | None = None + # Server requests being answered. Each runs as its own task: an + # answer can take minutes (a person approving a tool call), and the + # reader must keep reading meanwhile. + self._requests: set[asyncio.Task[None]] = set() + self._answered: set[Any] = set() + + def start( + self, + on_notify: NotifyHandler, + on_request: RequestHandler, + on_crash: Callable[[BaseException], None] | None = None, + ) -> None: + # `on_crash` is for listeners. A turn waits on a stream of + # notifications, not on a future: when the process dies, failing + # the futures wakes nobody who was listening (measured: a SIGKILLed + # codex left the turn blocked until the test timed out). + self._on_crash = on_crash + self._pump = asyncio.create_task(self._read_loop(on_notify, on_request)) + + async def close(self) -> None: + self._closed = True + if self._pump is not None: + self._pump.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._pump + self._pump = None + for task in list(self._requests): + task.cancel() + for task in list(self._requests): + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._requests.clear() + # A close is not a crash: `on_crash` does not fire. + self._fail_pending( + errors.ExternalHarnessError(f"{self._kind}: connection closed"), + crashed=False, + ) + + def _fail_pending( + self, exc: BaseException, *, crashed: bool = True + ) -> None: + for future in list(self._waiters.values()): + if not future.done(): + future.set_exception(exc) + # Prompt futures have no awaiting caller: the crash callback + # reports the failure to their wire's stream instead. + if self._completions: + future.exception() + self._waiters.clear() + self._completions.clear() + if crashed and self._on_crash is not None: + with contextlib.suppress(Exception): + self._on_crash(exc) + + async def call( + self, method: str, params: Any = None, *, timeout: float = 120 + ) -> Any: + request_id, future = await self._begin_call(method, params) + try: + async with asyncio.timeout(timeout): + return await future + finally: + self._waiters.pop(request_id, None) + + async def _begin_call( + self, + method: str, + params: Any = None, + *, + prepared: Callable[[int], None] | None = None, + completed: Callable[[int, asyncio.Future[Any]], None] | None = None, + ) -> tuple[int, asyncio.Future[Any]]: + """Write a request and return its id and response future. + + `prepared` runs before the write. `completed` runs in read order, + before the next record is read, not as a scheduled future callback. + """ + if self._closed: + raise errors.ExternalHarnessError( + f"{self._kind}: connection is closed" + ) + self._next_id += 1 + request_id = self._next_id + future: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + self._waiters[request_id] = future + if completed is not None: + self._completions[request_id] = completed + try: + if prepared is not None: + prepared(request_id) + await self._send( + { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + **({"params": params} if params is not None else {}), + } + ) + except BaseException: + self._waiters.pop(request_id, None) + self._completions.pop(request_id, None) + future.cancel() + raise + return request_id, future + + async def notify(self, method: str, params: Any = None) -> None: + await self._send( + { + "jsonrpc": "2.0", + "method": method, + **({"params": params} if params is not None else {}), + } + ) + + async def respond(self, request_id: Any, result: Any) -> None: + self._answered.add(request_id) + await self._send({"jsonrpc": "2.0", "id": request_id, "result": result}) + + async def _answer( + self, + on_request: RequestHandler, + request_id: Any, + method: str, + params: dict[str, Any], + ) -> None: + """Run one request handler; never leave the server without a reply.""" + try: + await on_request(request_id, method, params) + except asyncio.CancelledError: + raise + except Exception as exc: + if request_id not in self._answered: + with contextlib.suppress(Exception): + await self._send( + { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32603, "message": str(exc)}, + } + ) + finally: + self._answered.discard(request_id) + + async def _send(self, message: dict[str, Any]) -> None: + try: + await self._process.write(json.dumps(message) + "\n") + except Exception as exc: + raise errors.ExternalHarnessError( + f"{self._kind}: could not write to the CLI: {exc}" + ) from exc + + async def _read_loop( + self, on_notify: NotifyHandler, on_request: RequestHandler + ) -> None: + try: + while True: + line = await self._process.readline() + if not line: + note = await self._process.death_note() + self._fail_pending( + errors.ExternalHarnessError( + f"{self._kind} crashed: {note}" + ) + ) + return + try: + message = json.loads(line) + except json.JSONDecodeError: + continue # a stray non-JSON line is noise + if not isinstance(message, dict): + continue + if "method" in message and "id" not in message: + # Inline, so notifications are handled in the order sent. + await on_notify( + message["method"], message.get("params") or {} + ) + elif "method" in message: + task = asyncio.create_task( + self._answer( + on_request, + message["id"], + message["method"], + message.get("params") or {}, + ) + ) + self._requests.add(task) + task.add_done_callback(self._requests.discard) + else: + self._resolve(message) + except asyncio.CancelledError: + raise + except Exception as exc: + self._fail_pending( + errors.ExternalHarnessError(f"{self._kind}: {exc}") + ) + + def _resolve(self, message: dict[str, Any]) -> None: + msg_id = message.get("id") + if not isinstance(msg_id, int): + return + future = self._waiters.pop(msg_id, None) + if future is None or future.done(): + return + if "error" in message: + future.set_exception(JsonRpcError(message["error"])) + else: + future.set_result(message.get("result")) + + completed = self._completions.pop(msg_id, None) + if completed is not None: + completed(msg_id, future) diff --git a/src/ai/external/_process.py b/src/ai/external/_process.py new file mode 100644 index 00000000..fbcb20e5 --- /dev/null +++ b/src/ai/external/_process.py @@ -0,0 +1,142 @@ +"""A CLI process: the version probe, and JSON lines on stdio. + +Shared by the wires. `probe` serves every wire; `spawn` and `Process` serve +the wires that drive a CLI over stdio (Codex, Fx, Pi). +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import re +import warnings +from collections.abc import Mapping, Sequence + +from . import _errors as errors + +# Longest line a CLI may write. One JSON line can be megabytes (a +# `thread/read` of a long thread). With asyncio's default (64 KiB) such a line +# failed as if the CLI had crashed. +LINE_LIMIT = 256 * 1024 * 1024 + +# Seconds a CLI gets to exit after SIGTERM, before SIGKILL. +TERM_GRACE = 3.0 + + +class Process: + """A CLI started by `spawn`: JSON lines on stdin and stdout.""" + + def __init__(self, proc: asyncio.subprocess.Process) -> None: + self._proc = proc + + async def write(self, data: str) -> None: + assert self._proc.stdin is not None + self._proc.stdin.write(data.encode()) + await self._proc.stdin.drain() + + async def readline(self) -> str: + assert self._proc.stdout is not None + return (await self._proc.stdout.readline()).decode() + + async def read_stderr(self) -> str: + if self._proc.stderr is None: + return "" + return (await self._proc.stderr.read()).decode(errors="replace") + + async def wait(self) -> int: + return await self._proc.wait() + + async def terminate(self) -> None: + """Stop the process group: SIGTERM, then SIGKILL.""" + if self._proc.returncode is not None: + return + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(self._proc.pid), 15) + try: + async with asyncio.timeout(TERM_GRACE): + await self._proc.wait() + except TimeoutError: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(self._proc.pid), 9) + with contextlib.suppress(Exception): + await self._proc.wait() + + async def death_note(self) -> str: + """Say how the process ended, and its last words.""" + code: int | None = None + tail = "" + with contextlib.suppress(Exception): + async with asyncio.timeout(2): + code = await self.wait() + with contextlib.suppress(Exception): + async with asyncio.timeout(2): + tail = (await self.read_stderr()).strip() + how = f"exit code {code}" if code is not None else "exited" + if code in (137, -9): + how += " (SIGKILL, typically the kernel's OOM killer)" + return f"{how}; stderr: {tail[-800:]}" if tail else f"{how}; no stderr" + + +async def spawn( + argv: Sequence[str], *, cwd: str, env: Mapping[str, str] +) -> Process: + """Start a CLI in its own process group, with pipes for stdio.""" + try: + proc = await asyncio.create_subprocess_exec( + *argv, + cwd=cwd, + env={**os.environ, **env}, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + # Own the whole tree, so teardown reaches grandchildren too. + start_new_session=True, + limit=LINE_LIMIT, + ) + except OSError as exc: + raise errors.ExternalHarnessError( + f"cannot start {argv[0]!r}: {exc}" + ) from exc + return Process(proc) + + +async def probe( + kind: str, + executable: str, + *, + install: str, + verified: str, + cwd: str, + env: Mapping[str, str], +) -> str: + """Check that the CLI runs; warn if it is older than the verified one.""" + try: + proc = await asyncio.create_subprocess_exec( + executable, + "--version", + cwd=cwd, + env={**os.environ, **env}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + async with asyncio.timeout(60): + stdout, _ = await proc.communicate() + except (OSError, TimeoutError) as exc: + raise errors.ExecutableMissingError(kind, executable, install) from exc + if proc.returncode != 0: + raise errors.ExecutableMissingError(kind, executable, install) + version = (stdout.decode(errors="replace").strip().splitlines() or [""])[0] + have, want = _version(version), _version(verified) + if have is not None and want is not None and have < want: + warnings.warn( + f"{kind} {version} is older than {verified}, the version this " + "SDK was verified with; update it if something misbehaves", + stacklevel=2, + ) + return version + + +def _version(text: str | None) -> tuple[int, ...] | None: + found = re.search(r"\d+(?:\.\d+)+", text or "") + return tuple(int(n) for n in found.group(0).split(".")) if found else None diff --git a/src/ai/external/_session.py b/src/ai/external/_session.py new file mode 100644 index 00000000..5277aeb1 --- /dev/null +++ b/src/ai/external/_session.py @@ -0,0 +1,913 @@ +"""`Session`: the session of an external harness. + +The state is here: `id` and `messages`. So is the low level: `start`, +`send`, `events` and `interrupt`. The model protocol +(`ExternalHarnessProvider.stream`) is built on it. +""" + +from __future__ import annotations + +import asyncio +import collections +import contextlib +import dataclasses +import functools +import json +import warnings +from collections.abc import AsyncGenerator, Awaitable, Callable +from typing import TYPE_CHECKING, Any, Literal + +import pydantic + +from .. import experimental_telemetry as telemetry +from ..types import events as events_ +from ..types import messages as messages_ +from ..types import tools as tools_ +from ..types import usage as usage_ +from . import _bridge, _wire +from . import _errors as errors + +if TYPE_CHECKING: + from . import _harness + +# Seconds to wait for more `ai` tool calls before the message ends: a CLI +# that calls tools in parallel sends the calls one by one. +BATCH_WINDOW = 0.05 +# Seconds `interrupt()` waits for the CLI to end the turn before it closes it. +INTERRUPT_TIMEOUT = 30.0 +# Key in `provider_metadata` for what the external harness records about a +# message. +METADATA_KEY = "external" + +NEW_MESSAGE = "No result: a new user message came before the result." +INTERRUPTED = "No result: the turn was interrupted." +CLOSED = "No result: the external harness closed." +NOT_OPEN = "the external harness is not open: use `async with harness`" + + +# The sessions that are open in this process. Two external harnesses on one +# CLI session are two writers: the second one does not open. +_OPEN: set[Session] = set() + + +class MessageEvent(events_.BaseEvent): + """A complete message went into `session.messages`.""" + + message: messages_.Message + kind: Literal["external_message"] = "external_message" + + +ExternalHarnessEvent = events_.Event | MessageEvent + + +class ExternalHarnessSessionSpanData(pydantic.BaseModel): + """One external harness session, from the start of the CLI to its stop.""" + + kind: Literal["external_session"] = "external_session" + harness: str + model: str | None = None + cwd: str + session_id: str | None = None + + +class ExternalHarnessMessageSpanData(pydantic.BaseModel): + """One assistant message of an external harness.""" + + kind: Literal["external_message"] = "external_message" + harness: str + model: str | None = None + session_id: str | None = None + message: messages_.Message | None = None + usage: usage_.Usage | None = None + finish_reason: str | None = None + + +class Session: + """The session of an external harness: its state and direct control. + + An external harness comes with its session: `harness.session`. Do not + make a session, and do not open or close one: `async with harness` does + that. `Codex(resume=session_id)` gives an external harness whose session + continues one that the CLI has, and `Codex(history=messages)` gives one + whose new session starts with a past. The state is plain data: + + - `id`: the CLI's id for the session. None until a new session starts. + Save it to continue the session later. + - `messages`: the conversation, in `ai` messages. The session keeps it + current. + + Use the session in the block of its external harness. The CLI stays + alive between the turns:: + + async with ai.external.Codex(cwd=".") as harness: + await harness.session.send(ai.user_message("Fix the tests")) + async for event in harness.session.events(): + ... + + In the block, the external harness's model calls + (`ai.stream(harness, messages)`, `ai.Agent(...).run(harness, messages)`) + run on this session; outside the block they raise. After the block, `id` + and `messages` stay. + + The CLI starts with the first `send()` or the first model call, not when + the block opens: by then the tools and the past are known. `start()` + starts it at once. + + `provider_metadata["external"]` marks a message that the external harness + wrote outside a complete model call: `{"stop_reason": "interrupted"}` on a + reply that an interrupt or a close cut, `{"error": text}` on the reply of a + failed turn, and `{"synthetic": True}` on a tool message for calls that got + no result. + """ + + def __init__( + self, + harness: _harness.ExternalHarness, + session_id: str | None = None, + messages: list[messages_.Message] | None = None, + ) -> None: + self.harness = harness + self.id = session_id + self.messages = list(messages or []) + # Open: the CLI stays alive between turns, and `send()` works. + self._open = False + self._run: _CliRun | None = None + # What `events()` did not give yet. + self._pending_events: collections.deque[Any] = collections.deque() + self._pending_events_wake = asyncio.Event() + # `events()` gave something since the last turn began. + self._pending_events_read = False + # The queues of the model calls that read. See the provider's + # `stream()`. + self._stream_queues: set[asyncio.Queue[Any]] = set() + self._start_lock = asyncio.Lock() + # Held by the provider's `stream()`: one model call at a time. + self._stream_lock = asyncio.Lock() + + def _enter(self) -> None: + """Open the session. For `ExternalHarness.__aenter__`.""" + if self._open: + raise errors.ExternalHarnessError( + "the external harness is open already" + ) + if self.id is not None and any(s.id == self.id for s in _OPEN): + raise errors.ExternalHarnessError( + f"the session {self.id} is open already" + ) + self._open = True + _OPEN.add(self) + + async def start(self) -> None: + """Start the CLI now, in the block of the external harness. + + Without this call, the CLI starts with the first `send()` or the first + model call. With `id`, continue that CLI session. Without it, start a + new one, seeded with `messages` (lossy). Started already: nothing + happens. After the CLI stopped, start it again. + """ + if not self._open: + raise errors.ExternalHarnessError(NOT_OPEN) + await self._start() + + async def _start(self) -> None: + async with self._start_lock: + if self._run is not None and self._run.stopped is None: + return + if self._run is not None: + run, self._run = self._run, None + await run.close() + run = _CliRun(self, self.harness.provider._make_wire(self.harness)) + await run.start() + self._run = run + + async def send(self, message: messages_.Message) -> None: + """Give a message to the CLI. + + A user message starts a turn, or steers the turn that runs. The CLI + decides. A tool message gives results to `ai` tool calls that wait. + `send()` returns when the CLI has the message; the message goes into + `messages` when the CLI accepts it. + """ + if message.role not in ("user", "tool"): + raise ValueError( + f"send() takes a user or tool message, not {message.role!r}" + ) + if not self._open: + raise errors.ExternalHarnessError(NOT_OPEN) + await self._send(message) + + async def _send(self, message: messages_.Message) -> None: + if self._run is None: + await self._start() + run = self._run + if run is None: + raise errors.ExternalHarnessError( + "the session closed during send()" + ) + if run.stopped is not None: + raise errors.ExternalHarnessError( + f"the external harness stopped: {run.stopped}" + ) + if message.role == "tool": + await run.run_in_pump( + functools.partial(run.send_tool_results, message) + ) + return + await run.run_in_pump(functools.partial(run.fail_calls, NEW_MESSAGE)) + if run.idle.is_set(): + # A new turn. Nobody read `events()` during the last one: nobody + # reads, so the feed must not grow. + if not self._pending_events_read: + self._pending_events.clear() + self._pending_events_read = False + run.idle.clear() + run.used = True + try: + await run.wire.send(message) + except BaseException: + if run.hydrator is None and not run.waiting_calls: + run.idle.set() + raise + + async def events(self) -> AsyncGenerator[ExternalHarnessEvent]: + """Read the session's events from where the last reader stopped. + + The session keeps the events that nobody read. Thus `send()` and then + `events()` loses nothing, and a reader that stops needs no cleanup: + the next `events()` continues. One reader: with two, each event goes + to one of them. It ends when the external harness closes, and raises + if the CLI fails. + + If nobody reads during a whole turn, the events before the latest + turn are dropped. The messages stay in `messages`. + """ + if not self._open and not self._pending_events: + raise errors.ExternalHarnessError(NOT_OPEN) + while True: + while self._pending_events: + item = self._pending_events.popleft() + if isinstance(item, BaseException): + raise item + self._pending_events_read = True + yield item + if not self._open: + return + self._pending_events_wake.clear() + await self._pending_events_wake.wait() + + async def interrupt(self) -> None: + """Stop the current turn. The CLI keeps the session. + + Return when the CLI ended the turn. If it does not end the turn in + `INTERRUPT_TIMEOUT` seconds, stop the CLI; the next `send()` or model + call starts it again. No turn: nothing happens. + """ + run = self._run + if run is None or run.stopped is not None: + return + await run.run_in_pump(functools.partial(run.fail_calls, INTERRUPTED)) + if run.idle.is_set(): + return + run.interrupting = True + try: + async with asyncio.timeout(INTERRUPT_TIMEOUT): + await run.wire.interrupt() + await run.idle.wait() + except TimeoutError: + await self._stop_cli() + + async def _close(self) -> None: + """Stop the CLI and close the session. `id` and `messages` stay. + + For `ExternalHarness.__aexit__`. A second close does nothing. + """ + await self._stop_cli() + self._open = False + _OPEN.discard(self) + self._pending_events_wake.set() + + async def _stop_cli(self) -> None: + """Stop the CLI. The session stays as it is, open or not.""" + async with self._start_lock: + run, self._run = self._run, None + if run is not None: + await run.close() + # A model call that got the end takes nothing after it. + for queue in self._stream_queues: + queue.put_nowait(_END) + self._stream_queues.clear() + + def _publish_event(self, item: Any) -> None: + """Give an event, or an error, to `events()` and to the model calls.""" + self._pending_events.append(item) + self._pending_events_wake.set() + for queue in self._stream_queues: + queue.put_nowait(item) + + async def _start_for_stream(self, *, seeds: bool) -> _CliRun: + """Start the CLI for a model call. For the provider's `stream()`. + + A CLI reads the tool list once, when it connects to the bridge, and + takes a past only when its session starts. `seeds`: the session just + took a past. If the tools or the past changed and no turn runs, start + the CLI again. + """ + harness = self.harness + run = self._run + if run is not None and run.stopped is None: + restart = (seeds and run.fresh) or ( + _dump_tool_specs(harness.tools) != run.served_tool_specs + ) + if restart and run.idle.is_set() and not run.waiting_calls: + async with self._start_lock: + self._run = None + await run.close() + if run.fresh and not run.used: + # The session is new and empty: start a new one, which also + # takes the past. + self.id = None + elif restart: + warnings.warn( + "the tools changed during a turn: the CLI keeps the " + "tools it saw at the start until the external harness " + "restarts", + stacklevel=2, + ) + await self._start() + run = self._run + if run is None: + raise errors.ExternalHarnessError( + "the session closed during stream()" + ) + run.bridge.set_tools(harness.tools) + return run + + def _sync( + self, messages: list[messages_.Message] + ) -> list[messages_.Message]: + """Match a whole history with the conversation; return the new part. + + First use: the history up to its last user messages becomes the + conversation. After that, the history must continue the + conversation. + """ + if not self.messages: + # Without an id the past seeds the new CLI session; with one, the + # CLI has it already. + start = len(messages) + while start and messages[start - 1].role == "user": + start -= 1 + if start == len(messages): + raise errors.TranscriptMismatchError( + "the last message must be a user message: an external " + "harness cannot continue tool calls that it did not make" + ) + self.messages = list(messages[:start]) + return list(messages[start:]) + # The external harness wrote these outside a complete model call: the + # caller does not always have them. + own = { + m.id + for m in self.messages + if METADATA_KEY in (m.provider_metadata or {}) + } + known = [m.id for m in self.messages if m.id not in own] + given = [m for m in messages if m.id not in own] + if [m.id for m in given[: len(known)]] != known: + raise errors.TranscriptMismatchError( + "the messages do not continue this session's conversation; " + "use harness.session.messages as the history, or a new " + "external harness" + ) + new = given[len(known) :] + if not new: + raise errors.TranscriptMismatchError( + "the messages have nothing new for this session: " + "add a user message" + ) + for message in new: + if message.role not in ("user", "tool"): + raise errors.TranscriptMismatchError( + f"a new {message.role} message cannot go to the " + "CLI: only user and tool messages can" + ) + return new + + +@dataclasses.dataclass +class _Call: + """An `ai` tool call that the CLI made through the bridge. The CLI waits.""" + + tool_call_id: str + name: str + arguments: dict[str, Any] + answer: asyncio.Future[tuple[Any, bool]] + + +@dataclasses.dataclass +class _Command: + """Work for the pump: only the pump task changes the conversation.""" + + run: Callable[[], Awaitable[None]] + done: asyncio.Future[None] + + +@dataclasses.dataclass(frozen=True) +class _Stopped: + """The wire stopped: the CLI exited or failed.""" + + error: BaseException + + +_END = object() # put on a model call's queue when the CLI stops + + +class _CliRun: + """One run of the CLI process: the wire, the bridge and the pump. + + The pump is the only task that changes the conversation state. Other + tasks give it items through `queue`: wire items, bridge calls, commands. + What it makes goes to the session's readers. + """ + + def __init__(self, session: Session, wire: _wire.Wire) -> None: + self.session = session + self.harness = harness = session.harness + self.wire = wire + self.bridge = _bridge.Bridge(self._handle_bridge_call) + self.queue: asyncio.Queue[Any] = asyncio.Queue() + self.tasks: list[asyncio.Task[None]] = [] + self.span: telemetry.Span[ExternalHarnessSessionSpanData] = ( + telemetry.create_span( + ExternalHarnessSessionSpanData( + harness=harness.harness_class_id, + model=harness.model_name, + cwd=harness.cwd, + session_id=session.id, + ), + set_as_current=False, + ) + ) + # Set when no turn runs. `interrupt()` waits on it. + self.idle = asyncio.Event() + self.idle.set() + self.interrupting = False + self.stopped: BaseException | None = None + + # For the provider's `stream()` only. The run records these; only + # `stream()` reads them, to match the history it gets with this run. + # A new CLI session, not one that continues `session_id`. + self.fresh = session.id is None + # The tools the CLI saw when it connected to the bridge. + self.served_tool_specs: list[dict[str, Any]] = [] + # True after the first user message went to the CLI. + self.used = False + # Turn errors, by the id of the assistant message they ended. + # `stream()` raises them; `events()` gives only the message. + self.failures: dict[str, BaseException] = {} + + # The open assistant message. + self.hydrator: events_.MessageHydrator | None = None + self.message_id = "" + self.message_span: ( + telemetry.Span[ExternalHarnessMessageSpanData] | None + ) = None + self.tool_spans: dict[ + str, telemetry.Span[telemetry.ToolExecutionSpanData] + ] = {} + # Text and reasoning blocks and CLI tool calls that did not end yet. + # A steer waits for none to be open, so no block is cut in two. + self.open: set[str] = set() + # User messages that the CLI took while the message was busy. + self.inputs: list[messages_.Message] = [] + # `ai` tool calls without a result. While there are some, other items + # wait in `held_items`: they come after the results. + self.waiting_calls: dict[str, _Call] = {} + self.held_items: list[Any] = [] + # When the message with `ai` tool calls ends, if no call comes first. + self.close_at: float | None = None + + # -- life ----------------------------------------------------------------- + + async def start(self) -> None: + harness = self.harness + self.span.stamp_start() + await self.span.push() + await self.bridge.start() + self.bridge.set_tools(harness.tools) + self.served_tool_specs = _dump_tool_specs(harness.tools) + session = self.session + system = [m.text for m in session.messages if m.role == "system"] + instructions = "\n\n".join( + t for t in (harness.instructions, *system) if t + ) + seed = ( + [m for m in session.messages if m.role != "system"] + if session.id is None + else [] + ) + try: + session.id = await self.wire.start( + session_id=session.id, + bridge_url=self.bridge.url, + bridge_token=self.bridge.token, + instructions=instructions or None, + seed=seed, + ) + except BaseException as exc: + with contextlib.suppress(Exception): + await self.wire.close() + with contextlib.suppress(Exception): + await self.bridge.close() + await self.span.stamp_end(error=exc).push() + raise + self.span.data.session_id = session.id + self.tasks = [ + asyncio.create_task(self._read_wire()), + asyncio.create_task(self._run_pump()), + ] + + async def close(self) -> None: + if self.stopped is None: + # End what is open, so the state stays valid. + with contextlib.suppress(Exception): + await self.run_in_pump(self._end_open_work) + for task in self.tasks: + task.cancel() + for task in self.tasks: + with contextlib.suppress(BaseException): + await task + with contextlib.suppress(Exception): + await self.wire.close() + with contextlib.suppress(Exception): + await self.bridge.close() + self.stopped = self.stopped or errors.ExternalHarnessError( + "the external harness is closed" + ) + self.idle.set() + self._fail_commands() + await self.span.stamp_end().push() + + async def _end_open_work(self) -> None: + await self.fail_calls(CLOSED) + if self.hydrator is not None: + await self._end_message("other", interrupted=True) + + async def run_in_pump(self, run: Callable[[], Awaitable[None]]) -> None: + """Run `run` in the pump task and wait for it.""" + if self.stopped is not None: + raise errors.ExternalHarnessError( + f"the external harness stopped: {self.stopped}" + ) + done = asyncio.get_running_loop().create_future() + self.queue.put_nowait(_Command(run, done)) + await done + + def _fail_commands(self) -> None: + while not self.queue.empty(): + item = self.queue.get_nowait() + if isinstance(item, _Command) and not item.done.done(): + item.done.set_exception( + errors.ExternalHarnessError( + f"the external harness stopped: {self.stopped}" + ) + ) + + # -- tasks ---------------------------------------------------------------- + + async def _read_wire(self) -> None: + error: BaseException + try: + async for item in self.wire.events(): + self.queue.put_nowait(item) + error = errors.ExternalHarnessError( + f"{self.harness.harness_class_id}: the CLI stopped" + ) + except asyncio.CancelledError: + raise + except Exception as exc: + error = exc + self.queue.put_nowait(_Stopped(error)) + + async def _run_pump(self) -> None: + while True: + try: + async with asyncio.timeout_at(self.close_at): + item = await self.queue.get() + except TimeoutError: + self.close_at = None + if self.hydrator is not None and self.waiting_calls: + await self._end_message("tool_call") + continue + if isinstance(item, _Command): + try: + await item.run() + except Exception as exc: + if not item.done.done(): + item.done.set_exception(exc) + else: + if not item.done.done(): + item.done.set_result(None) + continue + try: + if isinstance(item, _Stopped): + raise item.error + await self._handle_item(item) + except Exception as exc: + await self._mark_stopped(exc) + return + + async def _mark_stopped(self, error: BaseException) -> None: + if not isinstance(error, errors.ExternalHarnessError): + wrapped = errors.ExternalHarnessError( + f"{self.harness.harness_class_id} failed: {error}" + ) + wrapped.__cause__ = error + error = wrapped + for call in self.waiting_calls.values(): + if not call.answer.done(): + call.answer.set_result((str(error), True)) + self.waiting_calls.clear() + if self.hydrator is not None: + with contextlib.suppress(Exception): + await self._end_message("error", error=error) + self.stopped = error + self.idle.set() + self._fail_commands() + self.session._publish_event(error) + self.session._stream_queues.clear() + + async def _handle_bridge_call( + self, name: str, arguments: dict[str, Any] + ) -> tuple[Any, bool]: + """Report a bridge call to an `ai` tool; wait for its result.""" + if self.stopped is not None: + return f"the external harness stopped: {self.stopped}", True + call = _Call( + tool_call_id=messages_.generate_id("call"), + name=name, + arguments=arguments, + answer=asyncio.get_running_loop().create_future(), + ) + self.queue.put_nowait(call) + return await call.answer + + # -- the conversation (pump task only) ------------------------------------ + + async def _handle_item(self, item: Any) -> None: + if self.waiting_calls and ( + self.hydrator is None or not isinstance(item, _Call) + ): + # Waiting for `ai` tool results: what comes now goes after them. + if self.hydrator is not None: + await self._end_message("tool_call") + self.held_items.append(item) + return + match item: + case _Call(): + await self._handle_tool_call(item) + case _wire.Accepted(message=message): + await self._handle_accepted(message) + case _wire.TurnEnd(): + await self._handle_turn_end(item) + case _: + await self._handle_content(item) + + async def _handle_content(self, event: _wire.Content) -> None: + if self.hydrator is None: + await self._begin_message() + self.idle.clear() + match event: + case ( + events_.TextStart(block_id=block) + | events_.ReasoningStart(block_id=block) + ): + self.open.add(block) + case ( + events_.TextEnd(block_id=block) + | events_.ReasoningEnd(block_id=block) + ): + self.open.discard(block) + case events_.BuiltinToolStart(tool_call_id=call_id, tool_name=name): + self.open.add(call_id) + tool_span = telemetry.create_span( + telemetry.ToolExecutionSpanData( + tool_name=name, tool_call_id=call_id + ), + parent=self.message_span, + set_as_current=False, + ) + self.tool_spans[call_id] = tool_span + await tool_span.stamp_start().push() + case events_.BuiltinToolEnd(tool_call=part): + started = self.tool_spans.get(part.tool_call_id) + if started is not None: + with contextlib.suppress(ValueError): + args = json.loads(part.tool_args or "{}") + started.data.args = ( + args if isinstance(args, dict) else None + ) + case events_.BuiltinToolResult(result=result): + self.open.discard(result.tool_call_id) + ended = self.tool_spans.pop(result.tool_call_id, None) + if ended is not None: + ended.data.result = result.result + ended.data.is_error = result.is_error + await ended.stamp_end().push() + case _: + pass + self._emit(event) + if self.inputs and not self.open: + await self._end_message("other") + + async def _handle_accepted(self, message: messages_.Message) -> None: + self.idle.clear() + if self.hydrator is None: + await self._append_message(message) + return + # A steer: the assistant message ends, the user message follows, + # and a new assistant message starts. + self.inputs.append(message) + if not self.open: + await self._end_message("other") + + async def _handle_tool_call(self, call: _Call) -> None: + if self.hydrator is None: + await self._begin_message() + self.waiting_calls[call.tool_call_id] = call + args = json.dumps(call.arguments) + part = messages_.ToolCallPart( + tool_call_id=call.tool_call_id, tool_name=call.name, tool_args=args + ) + self._emit( + events_.ToolStart( + tool_call_id=call.tool_call_id, tool_name=call.name + ) + ) + self._emit( + events_.ToolDelta(tool_call_id=call.tool_call_id, chunk=args) + ) + self._emit( + events_.ToolEnd(tool_call_id=call.tool_call_id, tool_call=part) + ) + self.close_at = asyncio.get_running_loop().time() + BATCH_WINDOW + + async def _handle_turn_end(self, end: _wire.TurnEnd) -> None: + interrupted = end.error is None and ( + end.reason == "interrupted" or self.interrupting + ) + reason: str = "other" if interrupted else end.reason + # Every turn ends with one StreamEnd, also a turn that said nothing. + await self._end_message( + reason, + usage=end.usage, + interrupted=interrupted, + error=end.error, + metadata=end.metadata, + ) + self.interrupting = False + self.idle.set() + + async def send_tool_results(self, message: messages_.Message) -> None: + """Give `ai` tool results to the CLI calls that wait for them.""" + results = message.tool_results + if not results: + raise ValueError("a tool message needs at least one tool result") + unknown = [ + p.tool_call_id + for p in results + if p.tool_call_id not in self.waiting_calls + ] + if unknown: + raise ValueError(f"no tool call waits for these results: {unknown}") + if self.hydrator is not None: + await self._end_message("tool_call") + for part in results: + call = self.waiting_calls.pop(part.tool_call_id) + value = part.model_input + if value is messages_.MODEL_INPUT_UNSET: + value = part.result + if not call.answer.done(): + call.answer.set_result((value, part.result_kind == "error")) + await self._append_message(message) + if not self.waiting_calls: + await self._handle_held_items() + + async def fail_calls(self, reason: str) -> None: + """Answer the waiting `ai` tool calls with an error.""" + if not self.waiting_calls: + return + if self.hydrator is not None: + await self._end_message("tool_call") + parts: list[messages_.Part] = [] + for call in self.waiting_calls.values(): + if not call.answer.done(): + call.answer.set_result((reason, True)) + parts.append( + messages_.ToolResultPart( + tool_call_id=call.tool_call_id, + tool_name=call.name, + result=reason, + result_kind="error", + ) + ) + self.waiting_calls.clear() + # A tool message must follow the calls, or no model takes the history. + await self._append_message( + messages_.Message( + role="tool", + parts=parts, + provider_metadata={METADATA_KEY: {"synthetic": True}}, + ) + ) + await self._handle_held_items() + + async def _handle_held_items(self) -> None: + held, self.held_items = self.held_items, [] + for item in held: + await self._handle_item(item) + + async def _begin_message(self) -> None: + harness = self.harness + self.message_id = messages_.generate_id("msg") + self.hydrator = events_.MessageHydrator() + self.open.clear() + self.message_span = telemetry.create_span( + ExternalHarnessMessageSpanData( + harness=harness.harness_class_id, + model=harness.model_name, + session_id=self.session.id, + ), + parent=self.span, + set_as_current=False, + ) + await self.message_span.stamp_start().push() + self._emit(events_.StreamStart()) + + async def _end_message( + self, + reason: str, + *, + usage: usage_.Usage | None = None, + interrupted: bool = False, + error: BaseException | None = None, + metadata: dict[str, Any] | None = None, + ) -> None: + if self.hydrator is None: + await self._begin_message() + assert self.hydrator is not None + self.close_at = None + metadata = dict(metadata or {}) + if error is not None: + metadata[METADATA_KEY] = {"error": str(error)} + self.failures[self.message_id] = error + elif interrupted: + # Not in the finish-reason vocabulary: `other`, and the raw value. + metadata[METADATA_KEY] = {"stop_reason": "interrupted"} + self._emit( + events_.StreamEnd( + finish_reason=reason, + usage=usage, + provider_metadata=metadata or None, + ) + ) + message = self.hydrator.message + self.hydrator = None + self.open.clear() + # Keep history current when readers receive StreamEnd. A telemetry + # push can yield to them, so publish the messages first. + await self._append_message(message) + inputs, self.inputs = self.inputs, [] + if reason == "tool_call": + # The tool results must follow the calls: steers wait for them. + self.held_items[:0] = [_wire.Accepted(m) for m in inputs] + else: + for m in inputs: + await self._append_message(m) + for span in self.tool_spans.values(): + await span.stamp_end().push() + self.tool_spans.clear() + if self.message_span is not None: + self.message_span.data.message = message + self.message_span.data.usage = message.usage + self.message_span.data.finish_reason = reason + await self.message_span.stamp_end(error=error).push() + self.message_span = None + + def _emit(self, event: events_.Event) -> None: + assert self.hydrator is not None + stamp = messages_.Message( + id=self.message_id, role="assistant", parts=[] + ) + self.session._publish_event( + self.hydrator.feed(event.model_copy(update={"message": stamp})) + ) + + async def _append_message(self, message: messages_.Message) -> None: + # A new list, not append: a caller can hold the old one. + self.session.messages = [*self.session.messages, message] + self.session._publish_event(MessageEvent(message=message)) + + +def _dump_tool_specs(tools: list[tools_.Tool]) -> list[dict[str, Any]]: + return [t.model_dump(mode="json") for t in tools] diff --git a/src/ai/external/_utils.py b/src/ai/external/_utils.py new file mode 100644 index 00000000..d0922300 --- /dev/null +++ b/src/ai/external/_utils.py @@ -0,0 +1,203 @@ +"""Conversions that every wire does the same way. Only wires use them. + +Prompt files go in as bytes or text. The CLI's own tool calls come out as +`BuiltinTool*` events. A past seeds a new CLI session by one rule: + +An external harness given `[*past, user_message]` on its first turn starts a new +conversation from `past`. Each CLI stores it natively so the agent truly +has the context, and the rule for what becomes what is the same everywhere: + +- text: verbatim, same role. +- tool calls and results: the NATIVE tool items, original name and + arguments unaltered. Measured: the receiving agent reads them as history + and acts with its own tools; it does not try to call a tool it lacks. + Names are not translated — mapping `Write` to `fileChange` would be the + invented taxonomy this SDK refuses everywhere else. Nor are PATHS: a call + that wrote /Users/x/proj/recipe.txt says so, and that is where it was + written. Across machines the receiver must be told where things are now + (or the workspace copied to the same path) — as any colleague would be. +- reasoning: a user-role handoff note after the turn it belonged to — where + the destination accepts it. Measured: fed back as the agent's OWN thoughts it + is rejected (Claude signs thinking blocks) or silently ignored (codex accepts + a reasoning item and never reads it). As a note, codex uses it. Claude does + NOT accept it in any form: Anthropic's safeguards flag real model reasoning + replayed as user content (`[reasoning_extraction]`) — hand-written notes + passed, genuine Opus thinking was refused. So the Claude renderer omits + reasoning, and the external harness warns with the count, because dropping + context silently is the one thing this SDK never does. Pi omits it too, for + both reasons: pi drops thinking another model wrote, and pi often runs Claude. + Fx omits it as well: its session store keeps reasoning only as the provider's + own records, with no portable form. +- files: images go to every harness, as the image input each takes (pi and + fx take bytes, so an image known only by URL becomes a note naming it); + PDFs to Claude as documents. Anything else (a PDF into codex, a text + file) is left out and counted (`unsupported_files`), and the external harness + warns, like it does for reasoning into Claude. +""" + +from __future__ import annotations + +import base64 +import os +import pathlib +import warnings +from collections.abc import Sequence +from typing import Any + +from ..models.core.helpers import files +from ..types import events, media +from ..types import messages as messages_ +from . import _errors as errors + +# -- files in a prompt --------------------------------------------------------- +# +# Files are turned into bytes HERE: a URL is downloaded with the SDK's own +# SSRF-checked downloader, not handed to the CLI. Measured: Claude Code drops +# an image given by URL without a word. + + +async def extract_data(part: messages_.FilePart) -> bytes: + """Return a file's bytes, downloading a URL first.""" + if isinstance(part.data, bytes): + return part.data + if media.is_downloadable_url(part.data): + body, _ = await files.download(part.data) + return body + return base64.b64decode(media.data_to_base64(part.data)) + + +async def format_text_file(part: messages_.FilePart) -> str: + """Return a text file as text a CLI reads inline, named if it has one.""" + body = (await extract_data(part)).decode("utf-8", errors="replace") + name = part.filename or "attached file" + return f"\n{body}\n" + + +def raise_for_unsupported_parts( + message: messages_.Message, + cli_name: str, + accepted_media_types: Sequence[str], +) -> None: + """Refuse a part the CLI cannot take, before anything runs. + + Text parts go everywhere. Text files (`text/*`) go everywhere, as text. + `accepted_media_types` lists the other media type prefixes the CLI takes. + """ + for part in message.parts: + if isinstance(part, messages_.TextPart): + continue + if not isinstance(part, messages_.FilePart): + raise errors.UnsupportedError( + f"{cli_name} does not take {part.kind} parts in a user message" + ) + if not any( + part.media_type.startswith(p) + for p in ("text/", *accepted_media_types) + ): + raise errors.UnsupportedError( + f"{cli_name} does not take {part.media_type} files" + ) + + +# -- the CLI's own tools as events -------------------------------------------- + + +def get_builtin_call_events( + tool_call_id: str, + tool_name: str, + tool_args: str, + *, + started_call_ids: set[str], +) -> list[ + events.BuiltinToolStart | events.BuiltinToolDelta | events.BuiltinToolEnd +]: + """Return the events of one finished CLI tool call. + + `started_call_ids` holds the calls whose start was already sent: the + start is sent here otherwise, because the hydrator only completes a call + it saw start. The arguments travel as a delta: the hydrator builds a + call's arguments only from its deltas. + """ + start_events: list[events.BuiltinToolStart] = [] + if tool_call_id not in started_call_ids: + started_call_ids.add(tool_call_id) + start_events.append( + events.BuiltinToolStart( + tool_call_id=tool_call_id, tool_name=tool_name + ) + ) + return [ + *start_events, + events.BuiltinToolDelta(tool_call_id=tool_call_id, chunk=tool_args), + events.BuiltinToolEnd( + tool_call_id=tool_call_id, + tool_call=messages_.BuiltinToolCallPart( + tool_call_id=tool_call_id, + tool_name=tool_name, + tool_args=tool_args, + ), + ), + ] + + +def get_builtin_result_event( + tool_call_id: str, tool_name: str, result: Any, *, is_error: bool +) -> events.BuiltinToolResult: + return events.BuiltinToolResult( + tool_call_id=tool_call_id, + result=messages_.BuiltinToolReturnPart( + tool_call_id=tool_call_id, + tool_name=tool_name, + result=result, + is_error=is_error, + ), + ) + + +# -- seeding a CLI with a past ------------------------------------------------- + + +def write_seed_transcript(path: pathlib.Path, text: str) -> None: + """Create a private transcript without overwriting an existing path.""" + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(text) + + +def warn_unsupported_files( + messages: list[messages_.Message], + cli_name: str, + *, + accepted_media_types: Sequence[str], + accepted_description: str, +) -> None: + """Say how many files a history held that `cli_name` was not given. + + `accepted_media_types` lists the media type prefixes `cli_name` takes + from a history, and `accepted_description` names them for the warning. + """ + left = sum( + 1 + for m in messages + for p in m.parts + if isinstance(p, messages_.FilePart) + and not any(p.media_type.startswith(a) for a in accepted_media_types) + ) + if left: + warnings.warn( + f"{left} file(s) in the history were not carried into " + f"{cli_name}: it takes {accepted_description} from a history. " + "Text, tool calls and images " + "were carried.", + stacklevel=3, + ) + + +def count_reasoning_parts(messages: list[messages_.Message]) -> int: + return sum( + 1 + for m in messages + for p in m.parts + if isinstance(p, messages_.ReasoningPart) and p.text.strip() + ) diff --git a/src/ai/external/_wire.py b/src/ai/external/_wire.py new file mode 100644 index 00000000..07bda881 --- /dev/null +++ b/src/ai/external/_wire.py @@ -0,0 +1,102 @@ +"""The private contract between a session and one CLI. + +A wire speaks one CLI's native protocol and nothing else. It turns native +records into `ai` events. It keeps no conversation state, knows no +subscribers, and does not know about `ai.Agent`: the session does all that. + +A wire yields three kinds of items: + +- `Content`: `ai` model events for what the assistant says and does. The CLI's + own tools are `BuiltinTool*` events. Calls to the MCP bridge (`ai` tools) + must NOT appear: the bridge reports them to the session itself. +- `Accepted`: the CLI took a message from `send()`. It must come before any + content that the message causes. +- `TurnEnd`: the CLI finished a turn. +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any, Literal, Protocol + +from ..types import events +from ..types import messages as messages_ +from ..types import usage as usage_ + +if TYPE_CHECKING: + from . import _errors as errors + +Content = ( + events.TextStart + | events.TextDelta + | events.TextEnd + | events.ReasoningStart + | events.ReasoningDelta + | events.ReasoningEnd + | events.BuiltinToolStart + | events.BuiltinToolDelta + | events.BuiltinToolEnd + | events.BuiltinToolResult +) + + +@dataclasses.dataclass(frozen=True) +class Accepted: + """The CLI took this message: it starts a turn or steers one.""" + + message: messages_.Message + + +@dataclasses.dataclass(frozen=True) +class TurnEnd: + """The CLI finished a turn. + + `error` is set when the turn failed; a model call raises it. + `metadata` goes into the `provider_metadata` of the turn's last + assistant message, for example which model wrote it. + """ + + reason: Literal["stop", "length", "interrupted", "error"] = "stop" + usage: usage_.Usage | None = None + error: errors.ExternalHarnessError | None = None + metadata: dict[str, Any] | None = None + + +Item = Content | Accepted | TurnEnd + + +class Wire(Protocol): + """One CLI process, driven by its native protocol.""" + + async def start( + self, + *, + session_id: str | None, + bridge_url: str, + bridge_token: str, + instructions: str | None, + seed: list[messages_.Message], + ) -> str: + """Start the CLI and open the session; return the session id. + + Resume `session_id` if it is set. Otherwise, start a new session + and give it `seed` as its history (lossy). + """ + ... + + async def send(self, message: messages_.Message) -> None: + """Give a user message to the CLI: a new turn, or a steer.""" + ... + + def events(self) -> AsyncIterator[Item]: + """Yield the items of the whole session. One reader only.""" + ... + + async def interrupt(self) -> None: + """Ask the CLI to stop the current turn. Do not wait for it.""" + ... + + async def close(self) -> None: + """Stop the CLI. Safe to call more than once.""" + ... diff --git a/src/ai/external/claude_code/__init__.py b/src/ai/external/claude_code/__init__.py new file mode 100644 index 00000000..7b5bbb51 --- /dev/null +++ b/src/ai/external/claude_code/__init__.py @@ -0,0 +1,3 @@ +from .harness import ClaudeCode, ClaudeCodeProvider + +__all__ = ["ClaudeCode", "ClaudeCodeProvider"] diff --git a/src/ai/external/claude_code/handoff.py b/src/ai/external/claude_code/handoff.py new file mode 100644 index 00000000..43f5d912 --- /dev/null +++ b/src/ai/external/claude_code/handoff.py @@ -0,0 +1,136 @@ +"""Seeding Claude Code with a past: records for its transcript store. + +The rule for what becomes what is the same for every CLI: see `_utils`. +""" + +from __future__ import annotations + +import base64 +import datetime +import json +import uuid +from typing import Any + +from ...types import messages as messages_ +from .. import _handoff_output, _utils + + +async def to_claude_records( + messages: list[messages_.Message], session_id: str, cwd: str +) -> list[dict[str, Any]]: + """Build Claude Code transcript records for the CLI's store. + + Chained by parentUuid. Reasoning parts are omitted: see `_utils`. + Callers report the omission; `_utils.count_reasoning_parts()` says how + many. + Files go in as base64, a URL downloaded first: Claude Code drops an + image given by URL without a word (measured on prompts). + """ + records: list[dict[str, Any]] = [] + parent: str | None = None + + def add(role: str, content: Any) -> None: + nonlocal parent + uid = str(uuid.uuid4()) + ts = ( + datetime.datetime.now(datetime.UTC) + .isoformat() + .replace("+00:00", "Z") + ) + records.append( + { + "parentUuid": parent, + "isSidechain": False, + "type": role, + "message": {"role": role, "content": content}, + "uuid": uid, + "timestamp": ts, + "sessionId": session_id, + "cwd": cwd, + } + ) + parent = uid + + for m in messages: + role = "user" if m.role in ("user", "tool") else "assistant" + blocks: list[dict[str, Any]] = [] + results: list[dict[str, Any]] = [] + for p in m.parts: + if isinstance(p, messages_.BuiltinToolReturnPart): + # A tool the CLI ran: the external harness holds its result in + # the assistant message, the store in the NEXT user record. So + # the assistant record ends here, and the results follow. + if blocks: + add(role, blocks) + blocks = [] + results.append( + { + "type": "tool_result", + "tool_use_id": p.tool_call_id, + "content": _handoff_output.format_result(p), + **({"is_error": True} if p.is_error else {}), + } + ) + continue + if results: + add("user", results) + results = [] + if isinstance(p, messages_.TextPart) and p.text.strip(): + blocks.append({"type": "text", "text": p.text}) + elif isinstance(p, messages_.FilePart) and m.role == "user": + if p.media_type.startswith("image/"): + kind = "image" + elif p.media_type == "application/pdf": + kind = "document" + else: + continue # counted by `_utils.warn_unsupported_files` + encoded = base64.b64encode( + await _utils.extract_data(p) + ).decode() + blocks.append( + { + "type": kind, + "source": { + "type": "base64", + "media_type": p.media_type, + "data": encoded, + }, + } + ) + elif isinstance( + p, messages_.ToolCallPart | messages_.BuiltinToolCallPart + ): + try: + args = json.loads(p.tool_args) if p.tool_args else {} + except json.JSONDecodeError: + args = {"raw": p.tool_args} + blocks.append( + { + "type": "tool_use", + "id": p.tool_call_id or p.id, + "name": p.tool_name, + "input": args, + } + ) + elif isinstance(p, messages_.ToolResultPart): + # A tool an `ai.Agent` ran: its result is in a `tool` + # message, which becomes a user record of its own. + blocks.append( + { + "type": "tool_result", + "tool_use_id": p.tool_call_id, + "content": _handoff_output.format_result(p), + **( + {"is_error": True} + if p.result_kind == "error" + else {} + ), + } + ) + # ReasoningPart: omitted on purpose. Anthropic's safeguards + # refuse replayed reasoning. The wire warns with the count. + if blocks: + add(role, blocks) + if results: + add("user", results) + return records diff --git a/src/ai/external/claude_code/harness.py b/src/ai/external/claude_code/harness.py new file mode 100644 index 00000000..b8045273 --- /dev/null +++ b/src/ai/external/claude_code/harness.py @@ -0,0 +1,39 @@ +"""Claude Code: the provider and the settings. `wire.py` drives the CLI.""" + +from __future__ import annotations + +import dataclasses +from typing import ClassVar + +from .. import _harness +from . import wire + + +class ClaudeCodeProvider(_harness.ExternalHarnessProvider): + """The Claude Code CLI.""" + + name: str = "claude-code" + + def _make_wire(self, harness: ClaudeCode) -> wire.ClaudeCodeWire: + return wire.ClaudeCodeWire(harness) + + +@dataclasses.dataclass(kw_only=True, eq=False) +class ClaudeCode(_harness.ExternalHarness): + """Claude Code, through `claude-agent-sdk`. Needs the `claude-code` extra. + + `allowed_tools` limits Claude Code's own tools (`["Read", "Grep"]`, or + `[]` for none), and `disallowed_tools` takes some away. `max_turns` caps + the agent's steps in one turn; a turn that hits it ends with + `finish_reason="length"`. Claude Code never asks for approval: it runs + with `bypassPermissions`. + """ + + harness_class_id: ClassVar[str] = "claude-code" + + provider: ClaudeCodeProvider = dataclasses.field( + default_factory=ClaudeCodeProvider + ) + allowed_tools: list[str] | None = None + disallowed_tools: list[str] = dataclasses.field(default_factory=list) + max_turns: int | None = None diff --git a/src/ai/external/claude_code/wire.py b/src/ai/external/claude_code/wire.py new file mode 100644 index 00000000..e7b00950 --- /dev/null +++ b/src/ai/external/claude_code/wire.py @@ -0,0 +1,539 @@ +"""Claude Code, driven through claude-agent-sdk. + +The session id we give the CLI is its own session id: `claude --resume ` +opens the same conversation, here or in the CLI's own UI. + +The comments record what was MEASURED against a real `claude` CLI (see +VERIFIED_VERSION), not what the API suggested. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import json +import os +import pathlib +import types +import uuid +import warnings +from collections.abc import AsyncGenerator, AsyncIterator, Iterator +from typing import TYPE_CHECKING, Any + +from ...providers import _optional +from ...types import events as events_ +from ...types import messages as messages_ +from ...types import usage as usage_ +from .. import _bridge, _process, _utils, _wire +from .. import _errors as errors +from . import handoff + +if TYPE_CHECKING: + from .harness import ClaudeCode + +KIND = "claude-code" +INSTALL = "npm install -g @anthropic-ai/claude-code" +# The CLI version the live tests last passed with. Older ones get a warning. +VERIFIED_VERSION = "2.1.319" +# Claude Code names a bridge tool `mcp__ai_sdk__`. +BRIDGE_PREFIX = f"mcp__{_bridge.SERVER_NAME}__" + +AUTH_HINT = ( + "claude-code uses your own login: run `claude` once and sign in, or " + "pass env={'ANTHROPIC_API_KEY': ...}." +) +# The CLI's own phrases for "no usable credentials". +AUTH_MARKERS = ("please run /login",) + + +class ClaudeCodeWire: + def __init__(self, harness: ClaudeCode) -> None: + self.harness = harness + # The CLI keys its store by the real path of its directory. + self.cwd = os.path.realpath(harness.cwd) + self.executable = harness.provider.executable or "claude" + self.client: Any = None + # Messages written to the CLI and not yet echoed, by the uuid we gave + # them. See `send()`. + self.pending: dict[str, messages_.Message] = {} + # We asked for an interrupt: the next result is not a failure. + self.interrupted = False + + def _client(self) -> Any: + if self.client is None: + raise errors.ExternalHarnessError(f"{KIND}: the CLI is not running") + return self.client + + async def start( + self, + *, + session_id: str | None, + bridge_url: str, + bridge_token: str, + instructions: str | None, + seed: list[messages_.Message], + ) -> str: + harness = self.harness + await _process.probe( + KIND, + self.executable, + install=INSTALL, + verified=VERIFIED_VERSION, + cwd=self.cwd, + env=harness.provider.env, + ) + claude_agent_sdk = _sdk() + if session_id is not None: + await self._require_known(session_id) + resume: str | None = session_id + elif seed: + session_id = resume = await self._stage(seed) + else: + session_id, resume = str(uuid.uuid4()), None + options: dict[str, Any] = { + "cwd": self.cwd, + "cli_path": self.executable, + # Raw Anthropic stream events, which map 1:1 onto Start/Delta/End + # triples. Without them the SDK yields only whole messages. + "include_partial_messages": True, + "session_id": session_id if resume is None else None, + "resume": resume, + # The CLI echoes each user message from stdin when it takes it + # into the conversation; `send()` uses that as the acceptance. + "extra_args": {"replay-user-messages": None}, + "mcp_servers": { + _bridge.SERVER_NAME: { + "type": "http", + "url": bridge_url, + # The SDK passes these settings on the command line. + # Let Claude expand the token from its environment. + "headers": { + "Authorization": f"Bearer ${{{_bridge.TOKEN_ENV}}}" + }, + } + }, + # External harnesses do not ask for approval. No gate at all: an + # always-allow callback froze the agent at its next tool call once + # nobody answered (measured). The allow rule keeps the bridge tools + # usable even if the user's settings narrow the mode. + "permission_mode": "bypassPermissions", + "allowed_tools": [f"mcp__{_bridge.SERVER_NAME}"], + } + # claude-agent-sdk spawns the CLI itself: pass the env along. An + # `ai` tool can take longer than Claude Code's default MCP tool + # timeout; the user's env wins. + options["env"] = { + "MCP_TOOL_TIMEOUT": str(24 * 3600 * 1000), + **harness.provider.env, + _bridge.TOKEN_ENV: bridge_token, + } + if harness.model_name: + options["model"] = harness.model_name + if instructions: + # Added to Claude Code's own system prompt, never replacing it. + options["system_prompt"] = { + "type": "preset", + "preset": "claude_code", + "append": instructions, + } + if harness.allowed_tools is not None: + options["tools"] = harness.allowed_tools + if harness.disallowed_tools: + options["disallowed_tools"] = harness.disallowed_tools + if harness.max_turns is not None: + options["max_turns"] = harness.max_turns + # Set before connect: a failed connect must still be closed. + self.client = claude_agent_sdk.ClaudeSDKClient( + options=claude_agent_sdk.ClaudeAgentOptions(**options) + ) + try: + await self.client.connect() + except claude_agent_sdk.ClaudeSDKError as exc: + raise _sdk_error(exc) from exc + return session_id + + def _project_dir(self, *, for_cli: bool = True) -> pathlib.Path: + config = os.environ.get("CLAUDE_CONFIG_DIR", "") + if for_cli: + config = self.harness.provider.env.get("CLAUDE_CONFIG_DIR", config) + root = ( + pathlib.Path(self.cwd if for_cli else os.getcwd(), config) + if config + else pathlib.Path.home() / ".claude" + ) + key: str = _sdk().project_key_for_directory(self.cwd) + return root / "projects" / key + + async def _stage(self, seed: list[messages_.Message]) -> str: + """Write a past into the CLI's store under a new id; run nothing.""" + session_id = str(uuid.uuid4()) + _utils.warn_unsupported_files( + seed, + KIND, + accepted_media_types=("image/", "application/pdf"), + accepted_description="images and PDFs", + ) + dropped = _utils.count_reasoning_parts(seed) + if dropped: + warnings.warn( + f"{dropped} reasoning part(s) in the history were not carried " + "into claude-code: Anthropic's safeguards refuse replayed " + "model reasoning ([reasoning_extraction]). Text and tool " + "calls were carried in full.", + stacklevel=2, + ) + path = self._project_dir() / f"{session_id}.jsonl" + records = await handoff.to_claude_records(seed, session_id, self.cwd) + text = "".join( + json.dumps(r, separators=(",", ":")) + "\n" for r in records + ) + await asyncio.to_thread(_utils.write_seed_transcript, path, text) + return session_id + + async def _require_known(self, session_id: str) -> None: + """Refuse an unknown session before connecting to it. + + Connecting to a ghost surfaces as claude-agent-sdk's own error + somewhere downstream. + """ + try: + project = self._project_dir() + if project == self._project_dir(for_cli=False): + known = ( + await asyncio.to_thread( + _sdk().get_session_info, session_id, self.cwd + ) + is not None + ) + else: + # SDK lookup reads the Python process's environment, not the + # CLI's. Check the CLI's own project transcript without + # changing process-wide state or treating an ID as a path. + try: + valid = str(uuid.UUID(session_id)) == session_id.lower() + except ValueError: + valid = False + path = project / f"{session_id}.jsonl" + known = valid and await asyncio.to_thread(path.is_file) + except Exception as exc: + raise errors.ExternalHarnessError( + f"{KIND}: cannot read session {session_id}: {exc}" + ) from exc + if not known: + raise errors.ExternalHarnessError( + f"{KIND}: no session {session_id} in {self.cwd}" + ) + + async def send(self, message: messages_.Message) -> None: + """Write a user message to the CLI's stdin: a new turn or a steer. + + Acceptance is the CLI's own echo (`--replay-user-messages`), matched + by a uuid we set: the CLI keeps it (measured). The echo comes when + the CLI takes the message into the conversation, before the content + it causes. A message written during a turn is injected into that + turn at the next step, and the echo comes there, not at the write + (measured on 2.1.319: written during a Bash call, echoed after its + result; one result for both messages). An `Accepted` emitted at the + write would split the reply at the wrong place. + """ + claude_agent_sdk = _sdk() + _utils.raise_for_unsupported_parts( + message, KIND, ("image/", "application/pdf") + ) + content = await _content(message) + client = self._client() + uid = str(uuid.uuid4()) + # Before the write: the echo can come before `query()` returns. + self.pending[uid] = message + try: + await client.query(_one_message(content, uid)) + except BaseException as exc: + self.pending.pop(uid, None) + if isinstance(exc, claude_agent_sdk.ClaudeSDKError): + raise _sdk_error(exc) from exc + raise + + async def interrupt(self) -> None: + # The CLI answers with an error result (`error_during_execution`), + # which `events()` reads as the interrupted turn's end. + if self.client is None: + return + self.interrupted = True + with contextlib.suppress(Exception): + await self.client.interrupt() + + async def close(self) -> None: + client, self.client = self.client, None + self.pending.clear() + if client is not None: + with contextlib.suppress(Exception): + await client.disconnect() + + # -- from the CLI --------------------------------------------------------- + + async def events(self) -> AsyncGenerator[_wire.Item]: + claude_agent_sdk = _sdk() + client = self._client() + turn = _Turn() + try: + async for message in client.receive_messages(): + for item in self._translate(message, turn): + yield item + if isinstance(message, claude_agent_sdk.ResultMessage): + turn = _Turn() + except claude_agent_sdk.ClaudeSDKError as exc: + raise _sdk_error(exc) from exc + raise errors.ExternalHarnessError(f"{KIND}: the CLI exited") + + def _translate(self, message: Any, turn: _Turn) -> Iterator[_wire.Item]: + claude_agent_sdk = _sdk() + if getattr(message, "parent_tool_use_id", None) is not None: + # A subagent's own steps (inside a Task call). The Task call and + # its result come at the top level; its inner steps do not go + # into this conversation. + return + if isinstance(message, claude_agent_sdk.StreamEvent): + yield from turn.stream_event(message) + elif isinstance(message, claude_agent_sdk.AssistantMessage): + if message.error: + turn.error = str(message.error) + streamed = message.message_id in turn.streamed + for index, block in enumerate(message.content): + if isinstance(block, claude_agent_sdk.ToolUseBlock): + if block.name.startswith(BRIDGE_PREFIX): + continue # an `ai` tool: the bridge reports it + turn.names[block.id] = block.name + yield from _utils.get_builtin_call_events( + block.id, + block.name, + json.dumps(block.input), + started_call_ids=turn.started, + ) + elif ( + isinstance(block, claude_agent_sdk.TextBlock) + and not streamed + and block.text + ): + # Not streamed: the CLI made it up, for example + # "Invalid API key" (measured, model ""). + block_id = f"{message.message_id or message.uuid}:{index}" + yield events_.TextStart(block_id=block_id) + yield events_.TextDelta(chunk=block.text, block_id=block_id) + yield events_.TextEnd(block_id=block_id) + elif isinstance(message, claude_agent_sdk.UserMessage): + accepted = self.pending.pop(message.uuid or "", None) + if accepted is not None: + yield _wire.Accepted(accepted) + return + # Tool results, or a user record the CLI wrote itself (for + # example "[Request interrupted by user for tool use]"). + blocks = ( + message.content if isinstance(message.content, list) else [] + ) + for block in blocks: + if ( + isinstance(block, claude_agent_sdk.ToolResultBlock) + and block.tool_use_id in turn.names + ): + yield _utils.get_builtin_result_event( + block.tool_use_id, + turn.names[block.tool_use_id], + block.content, + is_error=bool(block.is_error), + ) + elif isinstance(message, claude_agent_sdk.ResultMessage): + yield from turn.close_blocks() + usage = _usage(message.usage or {}, message.total_cost_usd) + interrupted, self.interrupted = self.interrupted, False + if interrupted: + # We asked for this. The CLI reports an error result after + # an interrupt; it is not a failure. + yield _wire.TurnEnd(reason="interrupted", usage=usage) + elif message.subtype == "error_max_turns": + yield _wire.TurnEnd(reason="length", usage=usage) + elif message.is_error: + detail = ( + message.result + or "; ".join(message.errors or []) + or turn.error + or message.subtype + ) + error = ( + errors.NotAuthenticatedError(KIND, detail, AUTH_HINT) + if turn.error == "authentication_failed" + else errors.turn_failed( + KIND, detail, AUTH_HINT, AUTH_MARKERS + ) + ) + yield _wire.TurnEnd(reason="error", usage=usage, error=error) + else: + yield _wire.TurnEnd(reason="stop", usage=usage) + + +class _Turn: + """What one turn's translation remembers.""" + + def __init__(self) -> None: + # The API message that streams now, and all that streamed. + self.message = "" + self.streamed: set[str] = set() + # Open text and thinking blocks: content-block index -> (kind, id). + self.blocks: dict[int, tuple[str, str]] = {} + # The CLI's own tool calls: started ones, and names by id. Bridge + # calls are in neither, so their results are skipped too. + self.started: set[str] = set() + self.names: dict[str, str] = {} + # The error an assistant message reported, e.g. authentication_failed. + self.error = "" + + def stream_event(self, message: Any) -> Iterator[_wire.Content]: + """One raw Anthropic stream event -> text and reasoning events. + + Block indexes restart at 0 in each API message, and one assistant + message of the external harness can hold several: the block id carries + the API message id. + """ + raw: dict[str, Any] = message.event + kind = raw.get("type") + index = raw.get("index", 0) + if kind == "message_start": + self.message = str( + (raw.get("message") or {}).get("id") or message.uuid + ) + self.streamed.add(self.message) + return + block_id = f"{self.message}:{index}" + if kind == "content_block_start": + block = raw.get("content_block") or {} + if block.get("type") == "text": + self.blocks[index] = ("text", block_id) + yield events_.TextStart(block_id=block_id) + elif block.get("type") == "thinking": + self.blocks[index] = ("thinking", block_id) + yield events_.ReasoningStart(block_id=block_id) + elif block.get("type") == "tool_use": + name = str(block.get("name") or "") + call_id = str(block.get("id") or "") + if not name.startswith(BRIDGE_PREFIX): + self.started.add(call_id) + yield events_.BuiltinToolStart( + tool_call_id=call_id, tool_name=name + ) + elif kind == "content_block_delta": + delta = raw.get("delta") or {} + if delta.get("type") == "text_delta" and delta.get("text"): + yield events_.TextDelta(chunk=delta["text"], block_id=block_id) + elif delta.get("type") == "thinking_delta" and delta.get( + "thinking" + ): + yield events_.ReasoningDelta( + chunk=delta["thinking"], block_id=block_id + ) + elif kind == "content_block_stop" and index in self.blocks: + yield from self._end(self.blocks.pop(index)) + + def close_blocks(self) -> Iterator[_wire.Content]: + """End the blocks that an interrupt or a failure cut.""" + blocks, self.blocks = self.blocks, {} + for block in blocks.values(): + yield from self._end(block) + + @staticmethod + def _end(block: tuple[str, str]) -> Iterator[_wire.Content]: + kind, block_id = block + if kind == "text": + yield events_.TextEnd(block_id=block_id) + else: + yield events_.ReasoningEnd(block_id=block_id) + + +def _sdk() -> types.ModuleType: + return _optional.import_optional_sdk( + "claude_agent_sdk", provider="claude-code", extra="claude-code" + ) + + +async def _content(message: messages_.Message) -> list[dict[str, Any]]: + """Return a user message as Anthropic content blocks. + + Images and PDFs travel as base64 blocks (measured: both read correctly). + """ + blocks: list[dict[str, Any]] = [] + for part in message.parts: + if isinstance(part, messages_.TextPart): + blocks.append({"type": "text", "text": part.text}) + elif isinstance(part, messages_.FilePart): + if part.media_type.startswith("text/"): + blocks.append( + { + "type": "text", + "text": await _utils.format_text_file(part), + } + ) + continue + kind = ( + "image" if part.media_type.startswith("image/") else "document" + ) + encoded = base64.b64encode(await _utils.extract_data(part)).decode() + blocks.append( + { + "type": kind, + "source": { + "type": "base64", + "media_type": part.media_type, + "data": encoded, + }, + } + ) + return blocks + + +async def _one_message( + content: list[dict[str, Any]], uid: str +) -> AsyncIterator[dict[str, Any]]: + """Yield one stream-json user message, with the uuid the echo carries.""" + yield { + "type": "user", + "message": {"role": "user", "content": content}, + "parent_tool_use_id": None, + "session_id": "default", + "uuid": uid, + } + + +def _usage(raw: dict[str, Any], cost: float | None) -> usage_.Usage: + cache_read = int(raw.get("cache_read_input_tokens") or 0) + details = raw.get("output_tokens_details") or {} + thinking = details.get("thinking_tokens") + return usage_.Usage( + # As the Anthropic provider: input includes the cache reads. + input_tokens=int(raw.get("input_tokens") or 0) + cache_read, + output_tokens=int(raw.get("output_tokens") or 0), + reasoning_tokens=thinking if isinstance(thinking, int) else None, + cache_read_tokens=cache_read, + cache_write_tokens=int(raw.get("cache_creation_input_tokens") or 0), + # Cost has no home in Usage; the CLI's own number rides in `raw`. + raw={"cost_usd": cost, "claude": raw}, + ) + + +def _sdk_error(exc: BaseException) -> BaseException: + """Turn a claude-agent-sdk exception into an external harness error.""" + claude_agent_sdk = _sdk() + if isinstance(exc, claude_agent_sdk.CLINotFoundError): + return errors.ExecutableMissingError(KIND, "claude", INSTALL) + if isinstance(exc, claude_agent_sdk.ProcessError): + code = getattr(exc, "exit_code", None) + return errors.ExternalHarnessError( + f"{KIND} crashed (exit {code}): {exc}" + ) + if isinstance(exc, claude_agent_sdk.CLIConnectionError): + return errors.ExternalHarnessError( + f"{KIND}: lost the CLI connection: {exc}" + ) + if isinstance(exc, claude_agent_sdk.CLIJSONDecodeError): + return errors.ExternalHarnessError( + f"{KIND}: unparseable CLI output: {exc}" + ) + return errors.ExternalHarnessError(f"{KIND} failed: {exc}") diff --git a/src/ai/external/codex/__init__.py b/src/ai/external/codex/__init__.py new file mode 100644 index 00000000..2b49b262 --- /dev/null +++ b/src/ai/external/codex/__init__.py @@ -0,0 +1,3 @@ +from .harness import Codex, CodexProvider + +__all__ = ["Codex", "CodexProvider"] diff --git a/src/ai/external/codex/handoff.py b/src/ai/external/codex/handoff.py new file mode 100644 index 00000000..7e4ba0b9 --- /dev/null +++ b/src/ai/external/codex/handoff.py @@ -0,0 +1,97 @@ +"""Seeding Codex with a past: Responses API items for `thread/inject_items`. + +The rule for what becomes what is the same for every CLI: see `_utils`. +""" + +from __future__ import annotations + +from typing import Any + +from ...types import media +from ...types import messages as messages_ +from .. import _handoff_output + +NOTE_PREFIX = "[Handoff from a previous session — earlier reasoning: " + + +def _note(reasoning: list[str]) -> dict[str, Any]: + """Return the reasoning of a turn as a user-role note.""" + text = " ".join(r.strip() for r in reasoning if r.strip()) + return { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": NOTE_PREFIX + text + "]"}], + } + + +def _image_url(part: messages_.FilePart) -> str: + """Return an http(s) URL as it is, anything else as a data URL.""" + return media.data_to_data_url(part.data, part.media_type) + + +def to_codex_items(messages: list[messages_.Message]) -> list[dict[str, Any]]: + """Responses API items for `thread/inject_items`. + + The items are a flat list, so each message goes in as it is: a call and + its result are two items that follow each other. That holds for a tool + an `ai.Agent` ran (a `ToolCallPart`, its result in a `tool` message) and + for a tool the CLI ran (built-in parts, the result in the same message). + A turn's reasoning goes in as a note after its calls, before their + results. + """ + items: list[dict[str, Any]] = [] + for m in messages: + reasoning: list[str] = [] + for p in m.parts: + if isinstance(p, messages_.TextPart) and p.text.strip(): + ctype = "input_text" if m.role == "user" else "output_text" + items.append( + { + "type": "message", + "role": m.role, + "content": [{"type": ctype, "text": p.text}], + } + ) + elif ( + isinstance(p, messages_.FilePart) + and m.role == "user" + and p.media_type.startswith("image/") + ): + items.append( + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_image", "image_url": _image_url(p)} + ], + } + ) + elif isinstance( + p, messages_.ToolCallPart | messages_.BuiltinToolCallPart + ): + items.append( + { + "type": "function_call", + "call_id": p.tool_call_id or p.id, + "name": p.tool_name, + "arguments": p.tool_args or "{}", + } + ) + elif isinstance( + p, messages_.ToolResultPart | messages_.BuiltinToolReturnPart + ): + if reasoning: + items.append(_note(reasoning)) + reasoning = [] + items.append( + { + "type": "function_call_output", + "call_id": p.tool_call_id, + "output": _handoff_output.format_result(p), + } + ) + elif isinstance(p, messages_.ReasoningPart) and p.text.strip(): + reasoning.append(p.text) + if reasoning: + items.append(_note(reasoning)) + return items diff --git a/src/ai/external/codex/harness.py b/src/ai/external/codex/harness.py new file mode 100644 index 00000000..68d8dfdc --- /dev/null +++ b/src/ai/external/codex/harness.py @@ -0,0 +1,37 @@ +"""Codex: the provider and the settings. `wire.py` drives `codex app-server`.""" + +from __future__ import annotations + +import dataclasses +from typing import Any, ClassVar, Literal + +from .. import _harness +from . import wire + + +class CodexProvider(_harness.ExternalHarnessProvider): + """The Codex CLI.""" + + name: str = "codex" + + def _make_wire(self, harness: Codex) -> wire.CodexWire: + return wire.CodexWire(harness) + + +@dataclasses.dataclass(kw_only=True, eq=False) +class Codex(_harness.ExternalHarness): + """Codex, over `codex app-server`. + + `sandbox` is Codex's own sandbox mode: `"read-only"`, + `"workspace-write"` or `"danger-full-access"` (the default). Codex never + asks for approval. `config` takes Codex `-c` overrides, for example + `{"model_reasoning_effort": "high"}`. + """ + + harness_class_id: ClassVar[str] = "codex" + + provider: CodexProvider = dataclasses.field(default_factory=CodexProvider) + sandbox: ( + Literal["read-only", "workspace-write", "danger-full-access"] | None + ) = None + config: dict[str, Any] = dataclasses.field(default_factory=dict) diff --git a/src/ai/external/codex/wire.py b/src/ai/external/codex/wire.py new file mode 100644 index 00000000..4472bc3a --- /dev/null +++ b/src/ai/external/codex/wire.py @@ -0,0 +1,579 @@ +"""Codex, driven through `codex app-server` (JSON-RPC over stdio). + +`codex exec --json` cannot be controlled: one process per turn and no input +during a turn. app-server has `turn/start`, `turn/steer` and +`turn/interrupt`. + +The protocol shapes were learned by running the server and reading what it +sent (see VERIFIED_VERSION), not from the schema alone. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import importlib.metadata +import json +import os +from collections.abc import AsyncGenerator, Iterator +from typing import TYPE_CHECKING, Any + +from ...types import events as events_ +from ...types import messages as messages_ +from ...types import usage as usage_ +from .. import _bridge, _process, _utils, _wire +from .. import _errors as errors +from .. import _jsonrpc as jsonrpc +from . import handoff + +if TYPE_CHECKING: + from .harness import Codex + +KIND = "codex" +INSTALL = "npm install -g @openai/codex" +# The CLI version the live tests last passed with. Older ones get a warning. +VERIFIED_VERSION = "0.160.1" + +# Server requests that ask for approval. The policy is "never" (external +# harnesses do not ask for approval), so these should not come. If one comes, +# the answer follows the sandbox: accept. +APPROVAL_METHODS = ( + "item/commandExecution/requestApproval", + "item/fileChange/requestApproval", + "item/permissions/requestApproval", + "execCommandApproval", + "applyPatchApproval", +) + +# Item types that are the agent doing something, not speaking. +TOOL_ITEMS = ( + "fileChange", + "commandExecution", + "mcpToolCall", + "webSearch", + "toolCall", +) + +AUTH_HINT = ( + "codex uses your own login: run `codex login`, or pass " + "env={'OPENAI_API_KEY': ...}, or point it at another provider with " + "config={'model_provider': ..., 'model_providers..base_url': ...}." +) +# The CLI's own phrases for "no usable credentials". +AUTH_MARKERS = ("please run `codex login`",) + + +class CodexWire: + def __init__(self, harness: Codex) -> None: + self.harness = harness + # Codex records a thread's directory as its real path (on macOS a + # temp dir /var/... is recorded as /private/var/..., measured). + self.cwd = os.path.realpath(harness.cwd) + self.process: _process.Process | None = None + self.peer: jsonrpc.JsonRpcPeer | None = None + self.thread = "" + self.instructions: str | None = None + # Unbounded: the reader fills it inline and must not wait. + self.queue: asyncio.Queue[tuple[str, Any, Any]] = asyncio.Queue() + # The turn that runs, from the turn/start response. `send` steers it. + self.running: str | None = None + self.interrupted = False + + def _rpc(self) -> jsonrpc.JsonRpcPeer: + if self.peer is None: + raise errors.ExternalHarnessError( + f"{KIND}: app-server is not running" + ) + return self.peer + + async def start( + self, + *, + session_id: str | None, + bridge_url: str, + bridge_token: str, + instructions: str | None, + seed: list[messages_.Message], + ) -> str: + harness = self.harness + executable = harness.provider.executable or "codex" + self.instructions = instructions + await _process.probe( + KIND, + executable, + install=INSTALL, + verified=VERIFIED_VERSION, + cwd=self.cwd, + env=harness.provider.env, + ) + config = { + # An `ai` tool can take longer than Codex's default (60 seconds). + f"mcp_servers.{_bridge.SERVER_NAME}.tool_timeout_sec": 24 * 3600, + **harness.config, + f"mcp_servers.{_bridge.SERVER_NAME}.url": bridge_url, + f"mcp_servers.{_bridge.SERVER_NAME}.bearer_token_env_var": ( + _bridge.TOKEN_ENV + ), + } + self.process = await _process.spawn( + [executable, "app-server", *config_args(config)], + cwd=self.cwd, + env={**harness.provider.env, _bridge.TOKEN_ENV: bridge_token}, + ) + self.peer = jsonrpc.JsonRpcPeer(self.process, kind=KIND) + self.peer.start( + self._on_notify, self._on_request, on_crash=self._on_crash + ) + try: + await self.peer.call( + "initialize", + { + "clientInfo": { + "name": "ai-python", + "title": "AI SDK for Python", + "version": importlib.metadata.version("ai"), + } + }, + timeout=60, + ) + await self.peer.notify("initialized") + return await self._open(session_id, seed) + except BaseException: + # A failed start must not leave the server running. + await self.close() + raise + + async def _open( + self, session_id: str | None, seed: list[messages_.Message] + ) -> str: + params = { + "sandbox": self.harness.sandbox or "danger-full-access", + # External harnesses do not ask for approval. A request that nobody + # answers would freeze the agent (measured). + "approvalPolicy": "never", + **( + {"model": self.harness.model_name} + if self.harness.model_name + else {} + ), + **( + {"developerInstructions": self.instructions} + if self.instructions + else {} + ), + } + if session_id is not None: + await self._require_here(session_id) + try: + result = await self._rpc().call( + "thread/resume", {"threadId": session_id, **params} + ) + except jsonrpc.JsonRpcError as exc: + raise errors.ExternalHarnessError( + f"{KIND}: cannot open thread {session_id}: {exc}" + ) from exc + self.thread = str( + (result or {}).get("thread", {}).get("id") or session_id + ) + return self.thread + result = await self._rpc().call( + "thread/start", {"cwd": self.cwd, **params} + ) + self.thread = str((result or {}).get("thread", {}).get("id") or "") + if not self.thread: + raise errors.ExternalHarnessError( + f"{KIND}: thread/start gave no id: {result}" + ) + if seed: + # The thread's model-visible history, before the first turn. + _utils.warn_unsupported_files( + seed, + KIND, + accepted_media_types=("image/",), + accepted_description="images", + ) + await self._rpc().call( + "thread/inject_items", + { + "threadId": self.thread, + "items": handoff.to_codex_items(seed), + }, + ) + return self.thread + + async def _require_here(self, session_id: str) -> None: + """Refuse a thread that belongs to another directory. + + Codex's thread store is the whole machine's. A thread from another + project resumes natively, and the agent then talks about files that + are not here. + """ + try: + result = await self._rpc().call( + "thread/read", {"threadId": session_id} + ) + except jsonrpc.JsonRpcError as exc: + raise errors.ExternalHarnessError( + f"{KIND}: cannot read thread {session_id}: {exc}" + ) from exc + thread = (result or {}).get("thread") or {} + if not thread: + raise errors.ExternalHarnessError(f"{KIND}: no thread {session_id}") + cwd = thread.get("cwd") + if cwd is not None and cwd != self.cwd: + raise errors.ExternalHarnessError( + f"{KIND}: thread {session_id} belongs to {cwd}, not {self.cwd}" + ) + + async def send(self, message: messages_.Message) -> None: + _utils.raise_for_unsupported_parts(message, KIND, ("image/",)) + items = await _input_items(message) + turn = self.running + if turn is not None: + try: + await self._rpc().call( + "turn/steer", + { + "threadId": self.thread, + # A precondition: to steer the wrong turn is worse + # than to fail. + "expectedTurnId": turn, + "input": items, + }, + ) + except jsonrpc.JsonRpcError as exc: + # The turn ended before the steer. Only this explicit + # rejection proves that Codex did not take the input. + if exc.code != -32600 or str(exc) not in ( + "no active turn", + "no active turn to steer", + ): + raise errors.ExternalHarnessError( + f"{KIND}: steer rejected: {exc}" + ) from exc + else: + self.queue.put_nowait(("accepted", message, turn)) + return + try: + begun = await self._rpc().call( + "turn/start", {"threadId": self.thread, "input": items} + ) + except jsonrpc.JsonRpcError as exc: + raise errors.turn_failed( + KIND, str(exc), AUTH_HINT, AUTH_MARKERS + ) from exc + # The turn id comes from the RESPONSE: this server sends no + # `turn/started`, and steer and interrupt need the id. + turn = ((begun or {}).get("turn") or {}).get("id") + if not isinstance(turn, str) or not turn: + raise errors.ExternalHarnessError( + f"{KIND}: turn/start gave no id: {begun}" + ) + self.running = turn + self.interrupted = False + self.queue.put_nowait(("accepted", message, turn)) + + async def interrupt(self) -> None: + turn = self.running + if turn is None or self.peer is None: + return + self.interrupted = True + with contextlib.suppress(Exception): + await self.peer.call( + "turn/interrupt", + {"threadId": self.thread, "turnId": turn}, + timeout=30, + ) + + async def close(self) -> None: + if self.peer is not None: + await self.peer.close() + self.peer = None + if self.process is not None: + with contextlib.suppress(Exception): + await self.process.terminate() + self.process = None + + # -- from the server ------------------------------------------------------ + + async def _on_notify(self, method: str, params: dict[str, Any]) -> None: + thread = params.get("threadId") or (params.get("thread") or {}).get( + "id" + ) + if thread == self.thread: + self.queue.put_nowait(("notify", method, params)) + + def _on_crash(self, exc: BaseException) -> None: + self.queue.put_nowait(("crashed", str(exc), None)) + + async def _on_request( + self, request_id: Any, method: str, params: dict[str, Any] + ) -> None: + # Answer, never leave the agent waiting on a question. + decision = "accept" if method in APPROVAL_METHODS else "decline" + await self._rpc().respond(request_id, {"decision": decision}) + + async def events(self) -> AsyncGenerator[_wire.Item]: + current: str | None = None # the turn whose notifications count + old: set[str] = set() + # Notifications of a turn whose turn/start response is not read yet. + early: dict[str, list[tuple[str, dict[str, Any]]]] = {} + state = _TurnState() + while True: + kind, first, second = await self.queue.get() + if kind == "crashed": + raise errors.ExternalHarnessError(f"{KIND} crashed: {first}") + if kind == "accepted": + yield _wire.Accepted(first) + if second != current: + if current is not None: + old.add(current) + current = second + state = _TurnState() + for method, params in early.pop(second, []): + for item in self._translate(method, params, state): + yield item + continue + method, params = first, second + turn = params.get("turnId") or (params.get("turn") or {}).get("id") + if turn is not None and turn != current: + if turn not in old: + early.setdefault(turn, []).append((method, params)) + # An interrupted turn's late notifications: without this, + # the next turn ended on the OLD turn's completion. + continue + for item in self._translate(method, params, state): + yield item + + def _translate( + self, method: str, params: dict[str, Any], state: _TurnState + ) -> Iterator[_wire.Item]: + if method == "thread/tokenUsage/updated": + state.usage = _usage(params.get("tokenUsage") or {}) + elif method == "item/agentMessage/delta": + item_id = params.get("itemId", "") + if item_id not in state.text: + state.text.add(item_id) + yield events_.TextStart(block_id=item_id) + yield events_.TextDelta( + chunk=params.get("delta", ""), block_id=item_id + ) + elif method in ( + "item/reasoning/textDelta", + "item/reasoning/summaryTextDelta", + ): + item_id = params.get("itemId", "") + if item_id not in state.reasoning: + state.reasoning.add(item_id) + yield events_.ReasoningStart(block_id=item_id) + yield events_.ReasoningDelta( + chunk=params.get("delta", ""), block_id=item_id + ) + elif method == "item/started": + item = params.get("item") or {} + if item.get("type") in TOOL_ITEMS and not _is_bridge(item): + state.tools.add(item.get("id", "")) + yield events_.BuiltinToolStart( + tool_call_id=item.get("id", ""), tool_name=_tool_name(item) + ) + elif method == "item/completed": + yield from _completed(params.get("item") or {}, state) + elif method == "turn/completed": + completed = params.get("turn") or {} + status = completed.get("status") + if self.running == completed.get("id"): + self.running = None + if status == "failed" and not self.interrupted: + # Reported as a normal stop, "401 Unauthorized" was silence. + error = errors.turn_failed( + KIND, + _error_message(completed) or state.last_error, + AUTH_HINT, + AUTH_MARKERS, + ) + yield _wire.TurnEnd( + reason="error", usage=state.usage, error=error + ) + elif status == "interrupted" or self.interrupted: + yield _wire.TurnEnd(reason="interrupted", usage=state.usage) + else: + yield _wire.TurnEnd(reason="stop", usage=state.usage) + elif method == "turn/failed": + self.running = None + error = errors.turn_failed( + KIND, _error_message(params), AUTH_HINT, AUTH_MARKERS + ) + yield _wire.TurnEnd(reason="error", usage=state.usage, error=error) + elif method == "error": + state.last_error = _error_message(params) or state.last_error + + +class _TurnState: + """What one turn's translation remembers.""" + + def __init__(self) -> None: + self.text: set[str] = set() + self.reasoning: set[str] = set() + self.tools: set[str] = set() + self.usage: usage_.Usage | None = None + # The last transport error codex reported while it retried: if the + # turn fails without a message of its own, the only explanation. + self.last_error = "" + + +def _completed(item: dict[str, Any], state: _TurnState) -> Iterator[_wire.Item]: + item_id = item.get("id", "") + kind = item.get("type") + if kind == "agentMessage": + if item_id in state.text: + state.text.discard(item_id) + yield events_.TextEnd(block_id=item_id) + elif item.get("text"): + # A message that was not streamed. + yield events_.TextStart(block_id=item_id) + yield events_.TextDelta(chunk=item["text"], block_id=item_id) + yield events_.TextEnd(block_id=item_id) + elif kind == "reasoning": + if item_id in state.reasoning: + state.reasoning.discard(item_id) + yield events_.ReasoningEnd(block_id=item_id) + elif text := _reasoning_text(item): + yield events_.ReasoningStart(block_id=item_id) + yield events_.ReasoningDelta(chunk=text, block_id=item_id) + yield events_.ReasoningEnd(block_id=item_id) + elif kind in TOOL_ITEMS and not _is_bridge(item): + name = _tool_name(item) + yield from _utils.get_builtin_call_events( + item_id, + name, + json.dumps(_tool_args(item)), + started_call_ids=state.tools, + ) + yield _utils.get_builtin_result_event( + item_id, + name, + _item_result(item), + is_error=item.get("status") == "failed", + ) + + +def _is_bridge(item: dict[str, Any]) -> bool: + """Tell if `item` calls the MCP bridge (external harnesses report those).""" + return ( + item.get("type") == "mcpToolCall" + and item.get("server") == _bridge.SERVER_NAME + ) + + +async def _input_items(message: messages_.Message) -> list[dict[str, Any]]: + """Return a user message as app-server `UserInput` items. + + An image goes as a data URL, built here from its bytes. + """ + items: list[dict[str, Any]] = [] + for part in message.parts: + if isinstance(part, messages_.TextPart): + items.append({"type": "text", "text": part.text}) + elif isinstance(part, messages_.FilePart): + if part.media_type.startswith("text/"): + items.append( + { + "type": "text", + "text": await _utils.format_text_file(part), + } + ) + else: + encoded = base64.b64encode( + await _utils.extract_data(part) + ).decode("ascii") + items.append( + { + "type": "image", + "url": f"data:{part.media_type};base64,{encoded}", + } + ) + return items + + +def _tool_name(item: dict[str, Any]) -> str: + # Never the command line: that made every shell call a tool with + # another name. The command is in the arguments. + return ( + item.get("toolName") or item.get("server") or item.get("type") or "tool" + ) + + +def _tool_args(item: dict[str, Any]) -> dict[str, Any]: + return { + k: v + for k, v in item.items() + if k in ("command", "changes", "arguments", "query", "path", "cwd") + } + + +def _item_result(item: dict[str, Any]) -> Any: + """Return what a tool item produced, under the name codex uses. + + A command's output is `aggregatedOutput` (codex 0.160; with only `output` + every command result was None), a file change's `changes`, an MCP call's + `result`. + """ + for key in ("aggregatedOutput", "output", "changes", "result"): + if item.get(key) is not None: + return item[key] + return None + + +def _reasoning_text(item: dict[str, Any]) -> str: + """Codex reports reasoning as text, or as a LIST of summary parts.""" + text = item.get("text") + if isinstance(text, str) and text: + return text + summary = item.get("summary") + if isinstance(summary, str): + return summary + if isinstance(summary, list): + return "".join(part for part in summary if isinstance(part, str)) + return "" + + +def _usage(raw: dict[str, Any]) -> usage_.Usage: + totals = raw.get("lastTurn") or raw.get("total") or raw + + def count(*names: str) -> int: + for name in names: + value = totals.get(name) + if isinstance(value, int): + return value + return 0 + + return usage_.Usage( + input_tokens=count("inputTokens", "input_tokens"), + output_tokens=count("outputTokens", "output_tokens"), + cache_read_tokens=count("cachedInputTokens", "cached_input_tokens"), + reasoning_tokens=count( + "reasoningOutputTokens", "reasoning_output_tokens" + ), + raw={"codex": raw}, + ) + + +def _error_message(payload: dict[str, Any]) -> str: + error = payload.get("error") + if isinstance(error, dict): + detail = error.get("additionalDetails") or "" + message = str(error.get("message") or "") + return f"{message} {detail}".strip() + return str(error or "") + + +def config_args(config: dict[str, Any]) -> list[str]: + """`-c key=value` pairs, the values as TOML. + + JSON string, number and bool literals are valid TOML. + """ + args: list[str] = [] + for key, value in config.items(): + args += ["-c", f"{key}={json.dumps(value)}"] + return args diff --git a/src/ai/external/fx/__init__.py b/src/ai/external/fx/__init__.py new file mode 100644 index 00000000..2cd77428 --- /dev/null +++ b/src/ai/external/fx/__init__.py @@ -0,0 +1,3 @@ +from .harness import Fx, FxProvider + +__all__ = ["Fx", "FxProvider"] diff --git a/src/ai/external/fx/handoff.py b/src/ai/external/fx/handoff.py new file mode 100644 index 00000000..4c980ded --- /dev/null +++ b/src/ai/external/fx/handoff.py @@ -0,0 +1,400 @@ +"""Write a past into the internal fx sessions/v2 store. Verified on 0.0.17.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import re +import secrets +import shutil +import time +from typing import Any + +from ...types import media +from ...types import messages as messages_ +from .. import _handoff_output + +FORMAT_DIR = "v2" +FORMAT_VERSION = 1 +PREVIEW_BYTES = 4096 # fx keeps at most this much of a result in the log line + +_CRC_TABLE = [] +for _i in range(256): + _c = _i + for _ in range(8): + _c = (_c >> 1) ^ 0x82F63B78 if _c & 1 else _c >> 1 + _CRC_TABLE.append(_c) + + +def crc32c(data: bytes) -> int: + crc = 0xFFFFFFFF + for byte in data: + crc = _CRC_TABLE[(crc ^ byte) & 0xFF] ^ (crc >> 8) + return crc ^ 0xFFFFFFFF + + +def _dumps(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def frame(payload: str) -> bytes: + """Seal one line. `payload` is a JSON object without its closing brace.""" + raw = payload.encode() + return raw + b',"crc":"%08x"}\n' % crc32c(raw) + + +class _Log: + def __init__(self, ts: int) -> None: + self.lines: list[bytes] = [] + self.seq = 0 + self.ts = ts + + def add(self, kind: str, fields: str = "") -> None: + self.seq += 1 + self.ts += 1 + head = ( + f'{{"v":{FORMAT_VERSION},"seq":{self.seq},' + f'"ts":{self.ts},"kind":"{kind}"' + ) + self.lines.append(frame(head + fields)) + + def item( + self, + turn: int, + kind: str, + data: dict[str, Any], + blobs: list[str] | None = None, + ) -> None: + fields = f',"turn":{turn},"type":"{kind}","data":{_dumps(data)}' + if blobs: + fields += f',"blobs":{_dumps(blobs)}' + self.add("item", fields) + + def set(self, key: str, value: Any) -> None: + self.add("set", f',"key":"{key}","value":{_dumps(value)}') + + +def _preview(text: str) -> str: + raw = text.encode() + if len(raw) <= PREVIEW_BYTES: + return text + return raw[:PREVIEW_BYTES].decode(errors="ignore") + + +def _turns(messages: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + """Split into turns. Each turn starts with exactly one user message.""" + turns: list[list[dict[str, Any]]] = [] + for message in messages: + if message["role"] == "user": + if ( + turns and len(turns[-1]) == 1 + ): # two user messages in a row: join them + last = turns[-1][0] + last["text"] = (last["text"] + "\n\n" + message["text"]).strip() + last.setdefault("images", []).extend(message.get("images", [])) + else: + turns.append( + [dict(message, images=list(message.get("images", [])))] + ) + else: + if not turns: # the past starts with the assistant + turns.append( + [ + { + "role": "user", + "text": "(start of the earlier conversation)", + "images": [], + } + ] + ) + turns[-1].append(message) + return turns + + +def write_session( + messages: list[messages_.Message], + *, + fx_home: str, + cwd: str, + prefs: dict[str, Any], + instructions: str | None, + session_id: str | None = None, + now_ms: int | None = None, +) -> tuple[str, int]: + """Write a new session. Return its id and turn count.""" + records = to_fx_records(messages) + cwd = os.path.realpath(cwd) + sid = ( + session_id or base64.urlsafe_b64encode(secrets.token_bytes(9)).decode() + ) + if not re.fullmatch(r"[A-Za-z0-9_-][A-Za-z0-9_.-]{0,254}", sid): + raise ValueError("invalid fx session id") + root = os.path.join(fx_home, "sessions", FORMAT_DIR) + final = os.path.join(root, sid) + if os.path.exists(final): + raise FileExistsError(final) + staging = os.path.join(root, ".tmp", sid) + os.makedirs(fx_home, mode=0o700, exist_ok=True) + for parent in ( + os.path.join(fx_home, "sessions"), + root, + os.path.join(root, ".tmp"), + ): + try: + os.mkdir(parent, mode=0o700) + except FileExistsError: + pass + os.mkdir(staging, mode=0o700) + + try: + os.mkdir(os.path.join(staging, "blobs"), mode=0o700) + + def put_blob(data: bytes) -> str: + digest = hashlib.sha256(data).hexdigest() + path = os.path.join(staging, "blobs", digest) + if not os.path.exists(path): + with open(path, "wb") as file: + file.write(data) + os.chmod(path, 0o400) + return digest + + log = _Log( + now_ms if now_ms is not None else int(time.time() * 1000) - 1000 + ) + log.add( + "session_created", + f',"id":{_dumps(sid)},"workspace":{_dumps(cwd)},"role":"root","host":"acp"', + ) + log.set("prefs", prefs) + log.set( + "permissions", + {"schema_version": 2, "next_generation": 1, "rules": []}, + ) + log.set("language", "und") + if instructions: + log.set("client_prompt", instructions) + + image_id = 0 + turns = _turns(records) + for number, turn in enumerate(turns, start=1): + log.add("turn_started", f',"turn":{number}') + user = turn[0] + text, images, blobs = user["text"], [], [] + for image in user.get("images", []): + image_id += 1 + digest = put_blob(image["data"]) + blobs.append(digest) + text = (text + "\n" if text else "") + f"[Image #{image_id}]" + ext = image["media_type"].split("/")[-1] + images.append( + { + "id": image_id, + "path": f"seed-image-{image_id}.{ext}", + "media_type": image["media_type"], + "blob": digest, + } + ) + log.item( + number, + "user", + {"text": text or "(empty)", "images": images, "work_id": None}, + blobs, + ) + + open_calls: dict[str, str] = {} + for message in turn[1:]: + if message["role"] == "assistant": + calls = message.get("tool_calls", []) + if message.get("text") or calls: + log.item( + number, + "assistant", + { + "text": message.get("text", ""), + "provider_replay": None, + "standalone_response": False, + }, + ) + for call in calls: + open_calls[call["id"]] = call["name"] + log.item( + number, + "tool_call", + { + "call_id": call["id"], + "tool_name": call["name"], + "arguments_json": _dumps( + call.get("arguments", {}) + ), + "argument_integrity": "valid", + "provisional_id": None, + "provider_result": None, + "final_identity": "valid", + "provenance": "fx_local", + }, + ) + else: + for result in message["results"]: + open_calls.pop(result["id"], None) + body = result["output"].encode() + digest = put_blob(body) + log.item( + number, + "tool_result", + { + "call_id": result["id"], + "tool_name": result["name"], + "status": "failure" + if result.get("is_error") + else "success", + "artifact_ref": f"result-{digest}.txt", + "tool_image_handle": None, + "output_bytes": len(body), + "stored_bytes": len(body), + "completeness": "complete", + "preview": _preview(result["output"]), + "provider_native": False, + "created_at_ms": log.ts, + "permission_feedback": [], + "committed_file_presentation": None, + "command_replay_ref": None, + "command_replay_bytes": None, + "command_process_presentation": None, + "terminal_action_presentation": None, + }, + [digest], + ) + for ( + call_id, + name, + ) in ( + open_calls.items() + ): # a call with no result: fx refuses the turn + body = b"(no result was recorded)" + digest = put_blob(body) + log.item( + number, + "tool_result", + { + "call_id": call_id, + "tool_name": name, + "status": "failure", + "artifact_ref": f"result-{digest}.txt", + "tool_image_handle": None, + "output_bytes": len(body), + "stored_bytes": len(body), + "completeness": "complete", + "preview": body.decode(), + "provider_native": False, + "created_at_ms": log.ts, + "permission_feedback": [], + "committed_file_presentation": None, + "command_replay_ref": None, + "command_replay_bytes": None, + "command_process_presentation": None, + "terminal_action_presentation": None, + }, + [digest], + ) + log.item(number, "turn_end", {"files": [], "turn_summary": None}) + log.add("turn_committed", f',"turn":{number}') + + log.add("closed") + + with open(os.path.join(staging, "lock"), "wb"): + pass + with open(os.path.join(staging, "log.jsonl"), "wb") as file: + file.write(b"".join(log.lines)) + file.flush() + os.fsync(file.fileno()) + os.chmod(os.path.join(staging, "log.jsonl"), 0o600) + os.chmod(os.path.join(staging, "lock"), 0o600) + os.rename( + staging, final + ) # atomic publish; fx never sees a half-written session + return sid, len(turns) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def to_fx_records(messages: list[messages_.Message]) -> list[dict[str, Any]]: + """Normalize SDK messages for the internal fx turn writer.""" + out: list[dict[str, Any]] = [] + for message in messages: + role = "user" if message.role == "system" else message.role + record: dict[str, Any] = { + "role": role, + "text": "", + "images": [], + "tool_calls": [], + } + + def flush(record: dict[str, Any] = record) -> None: + if record["text"] or record["images"] or record["tool_calls"]: + out.append(dict(record)) + record.update(text="", images=[], tool_calls=[]) + + for part in message.parts: + if isinstance(part, messages_.TextPart): + record["text"] += part.text + elif ( + isinstance(part, messages_.FilePart) + and role == "user" + and part.media_type.startswith("image/") + ): + if isinstance(part.data, str) and media.is_downloadable_url( + part.data + ): + record["text"] += f"[image: {part.data}]" + else: + record["images"].append( + { + "media_type": part.media_type, + "data": base64.b64decode( + media.data_to_base64(part.data) + ), + } + ) + elif ( + isinstance( + part, messages_.ToolCallPart | messages_.BuiltinToolCallPart + ) + and role == "assistant" + ): + try: + arguments = ( + json.loads(part.tool_args) if part.tool_args else {} + ) + except json.JSONDecodeError: + arguments = {"raw": part.tool_args} + record["tool_calls"].append( + { + "id": part.tool_call_id or part.id, + "name": part.tool_name, + "arguments": arguments, + } + ) + elif isinstance( + part, messages_.ToolResultPart | messages_.BuiltinToolReturnPart + ): + flush() + out.append( + { + "role": "tool", + "results": [ + { + "id": part.tool_call_id, + "name": part.tool_name, + "output": _handoff_output.format_result(part), + "is_error": part.result_kind == "error" + if isinstance(part, messages_.ToolResultPart) + else part.is_error, + } + ], + } + ) + flush() + return out diff --git a/src/ai/external/fx/harness.py b/src/ai/external/fx/harness.py new file mode 100644 index 00000000..c1c9e9cf --- /dev/null +++ b/src/ai/external/fx/harness.py @@ -0,0 +1,39 @@ +"""Fx: the provider and settings for `fx acp`.""" + +from __future__ import annotations + +import dataclasses +from typing import ClassVar + +from .. import _harness +from . import wire + + +class FxProvider(_harness.ExternalHarnessProvider): + """The fx CLI. `env={"FX_HOME": ...}` isolates its settings and store.""" + + name: str = "fx" + + def _make_wire(self, harness: Fx) -> wire.FxWire: + return wire.FxWire(harness) + + +@dataclasses.dataclass(kw_only=True, eq=False) +class Fx(_harness.ExternalHarness): + """Fx over `fx acp`, with full access to its built-in tools. + + `model_provider` and `effort` select the provider and reasoning effort. A + bridge tool must answer within 5 minutes. All streamed tool results are + short previews. Instructions are fixed at session start; resume does not + change them. Built-in tools cannot be limited. Fx loads user settings and + project AGENTS.md; use `FxProvider(env={"FX_HOME": ...})` to isolate + settings. Seeding writes fx's internal session format. A new fx release can + break it: the external harness raises instead of starting a session with a + wrong past. + """ + + harness_class_id: ClassVar[str] = "fx" + + provider: FxProvider = dataclasses.field(default_factory=FxProvider) + model_provider: str | None = None + effort: str | None = None diff --git a/src/ai/external/fx/wire.py b/src/ai/external/fx/wire.py new file mode 100644 index 00000000..1c12dab3 --- /dev/null +++ b/src/ai/external/fx/wire.py @@ -0,0 +1,683 @@ +"""Fx ACP v1 over JSON-RPC stdio. Shapes verified with fx 0.0.17.""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import dataclasses +import json +import os +import pathlib +import shutil +import warnings +from collections.abc import AsyncGenerator, Iterator +from typing import TYPE_CHECKING, Any, Literal + +from ...types import events as events_ +from ...types import messages as messages_ +from ...types import usage as usage_ +from .. import _bridge, _jsonrpc, _process, _utils, _wire +from .. import _errors as errors +from . import handoff + +if TYPE_CHECKING: + from .harness import Fx + +KIND = "fx" +INSTALL = "curl -fsSL https://fx.sh/setup.sh | bash" +VERIFIED_VERSION = "0.0.17" +AUTH_HINT = "fx uses your own login: run `fx login`, or set AI_GATEWAY_API_KEY." +AUTH_MARKERS = ( + "fx needs access", + "fx needs a", + "credential refresh failed", + "authentication failed", + "run fx login", +) + + +class FxWire: + def __init__(self, harness: Fx) -> None: + self.harness = harness + self.cwd = os.path.realpath(harness.cwd) + self.process: _process.Process | None = None + self.peer: _jsonrpc.JsonRpcPeer | None = None + self.session_id = "" + self.version = VERIFIED_VERSION + self.model: str | None = None + self.queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + self.running: int | None = None + self.idle = asyncio.Event() + self.idle.set() + self.interrupted = False + # A request stays busy until its response, not until its content ends. + self.requests: dict[int, _Prompt] = {} + self.order = 0 + self.retries: list[_Prompt] = [] + self.retry_task: asyncio.Task[None] | None = None + self.loading: set[str] | None = None + self.writing = False + self.early: list[tuple[str, Any]] = [] + self.closed = False + self.failure: errors.ExternalHarnessError | None = None + + def _rpc(self) -> _jsonrpc.JsonRpcPeer: + if self.closed: + raise errors.ExternalHarnessError("fx: connection is closed") + if self.failure is not None: + raise self.failure + if self.peer is None: + raise errors.ExternalHarnessError("fx: ACP is not running") + return self.peer + + async def start( + self, + *, + session_id: str | None, + bridge_url: str, + bridge_token: str, + instructions: str | None, + seed: list[messages_.Message], + ) -> str: + harness = self.harness + executable = harness.provider.executable or "fx" + await _process.probe( + KIND, + executable, + install=INSTALL, + verified=VERIFIED_VERSION, + cwd=self.cwd, + env=harness.provider.env, + ) + argv = [executable, "acp", "--permission-mode", "full-access"] + for flag, value in ( + ("--provider", harness.model_provider), + ("--model", harness.model_name), + ("--effort", harness.effort), + ): + if value is not None: + argv.extend((flag, value)) + self.process = await _process.spawn( + argv, cwd=self.cwd, env=harness.provider.env + ) + self.peer = _jsonrpc.JsonRpcPeer(self.process, kind=KIND) + self.peer.start(self._on_notify, self._on_request, self._on_crash) + try: + try: + initialized = await self.peer.call( + "initialize", + { + "protocolVersion": 1, + "clientCapabilities": {"session": {"notices": {}}}, + }, + ) + except _jsonrpc.JsonRpcError as exc: + raise errors.turn_failed( + KIND, str(exc), AUTH_HINT, AUTH_MARKERS + ) from exc + self.version = ( + initialized.get("agentInfo", {}).get("version") or self.version + ) + params: dict[str, Any] = { + "cwd": self.cwd, + "mcpServers": [ + { + "type": "http", + "name": _bridge.SERVER_NAME, + "url": bridge_url, + "headers": [ + { + "name": "Authorization", + "value": f"Bearer {bridge_token}", + } + ], + } + ], + } + if session_id is not None: + cursor = None + while True: + page = await self.peer.call( + "session/list", + { + "cwd": self.cwd, + **({"cursor": cursor} if cursor else {}), + }, + ) + if any( + s.get("sessionId") == session_id + and os.path.realpath(s.get("cwd", self.cwd)) == self.cwd + for s in page.get("sessions", []) + ): + break + cursor = page.get("nextCursor") + if not cursor: + raise errors.ExternalHarnessError( + f"fx: session {session_id} is unknown or belongs " + f"to another directory, not {self.cwd}" + ) + self.session_id = session_id + result = await self.peer.call( + "session/resume", {"sessionId": session_id, **params} + ) + else: + result = await self.peer.call( + "session/new", + { + **params, + **( + { + "systemPrompt": [ + {"type": "text", "text": instructions} + ] + } + if instructions + else {} + ), + }, + ) + if seed: + values = { + o["id"]: o.get("currentValue") + for o in result.get("configOptions", []) + } + prefs = { + k: values.get(k) + for k in ("provider", "model", "effort") + } + prefs["fast_mode"] = values.get("fast") in ( + True, + "true", + "on", + ) + home = ( + harness.provider.env.get("FX_HOME") + or os.environ.get("FX_HOME") + or str(pathlib.Path.home() / ".fx") + ) + _utils.warn_unsupported_files( + seed, + KIND, + accepted_media_types=("image/",), + accepted_description="images", + ) + dropped = _utils.count_reasoning_parts(seed) + if dropped: + warnings.warn( + f"{dropped} reasoning part(s) in the history " + "were not carried into fx: no portable reasoning " + "format. Text and tool calls were carried.", + stacklevel=2, + ) + self.session_id, count = await asyncio.to_thread( + handoff.write_session, + seed, + fx_home=home, + cwd=self.cwd, + prefs=prefs, + instructions=instructions, + ) + self.loading = set() + try: + result = await self.peer.call( + "session/load", + {"sessionId": self.session_id, **params}, + ) + if len(self.loading) != count: + raise ValueError( + f"replayed {len(self.loading)} user messages, " + f"expected {count}" + ) + except BaseException as exc: + with contextlib.suppress(Exception): + await self.peer.call( + "session/close", + {"sessionId": self.session_id}, + timeout=2, + ) + shutil.rmtree( + pathlib.Path(home) + / "sessions" + / "v2" + / self.session_id, + ignore_errors=True, + ) + self.session_id = "" + if not isinstance(exc, Exception): + raise + raise errors.ExternalHarnessError( + f"fx {self.version}: this version of ai cannot " + f"seed an fx session: {exc}" + ) from exc + finally: + self.loading = None + else: + self.session_id = str(result.get("sessionId") or "") + if not self.session_id: + raise errors.ExternalHarnessError( + "fx: session/new gave no session id" + ) + self.model = next( + ( + o.get("currentValue") + for o in result.get("configOptions", []) + if o.get("id") == "model" + ), + None, + ) + return self.session_id + except BaseException: + await self.close() + raise + + async def send(self, message: messages_.Message) -> None: + self._rpc() + _utils.raise_for_unsupported_parts(message, KIND, ("image/",)) + prompt: list[dict[str, Any]] = [] + text_only = True + for part in message.parts: + if isinstance(part, messages_.TextPart): + prompt.append({"type": "text", "text": part.text}) + elif isinstance(part, messages_.FilePart): + text_only = False + if part.media_type.startswith("text/"): + prompt.append( + { + "type": "resource", + "resource": { + "uri": "file:///" + + (part.filename or "attached-file"), + "text": ( + await _utils.extract_data(part) + ).decode(errors="replace"), + }, + } + ) + else: + prompt.append( + { + "type": "image", + "mimeType": part.media_type, + "data": base64.b64encode( + await _utils.extract_data(part) + ).decode("ascii"), + } + ) + while True: + self._rpc() + if self.retry_task is not None: + try: + await asyncio.shield(self.retry_task) + except asyncio.CancelledError: + self._rpc() # A close must fail send(), not cancel it. + raise + elif not text_only and self.requests: + await self.idle.wait() + else: + break + self.order += 1 + await self._prompt(_Prompt(self.order, message, prompt)) + + async def _prompt(self, record: _Prompt) -> None: + steer = bool(self.requests) + params = { + "sessionId": self.session_id, + "prompt": record.prompt, + **({"_meta": {"fx": {"steer": True}}} if steer else {}), + } + + def prepared(request_id: int) -> None: + self.requests[request_id] = record + self.idle.clear() + if not steer: + self.running = request_id + self.interrupted = False + record.accepted = True + + if not steer: + self.writing = True + try: + await self._rpc()._begin_call( + "session/prompt", + params, + prepared=prepared, + completed=self._finished, + ) + if not steer: + self.queue.put_nowait(("accepted", record.message)) + except BaseException: + # A failed write must not leave a request that can never resolve. + for key, value in list(self.requests.items()): + if value is record: + self.requests.pop(key) + if not self.requests: + self.idle.set() + raise + finally: + if not steer: + self.writing = False + for item in self.early: + self.queue.put_nowait(item) + self.early.clear() + + def _emit(self, kind: str, value: Any) -> None: + if self.writing: + self.early.append((kind, value)) + else: + self.queue.put_nowait((kind, value)) + + def _accept(self, request_id: int, record: _Prompt) -> None: + if not record.accepted: + record.accepted = True + self._emit("accepted", record.message) + if self.running is None: + self.running = request_id + + def _finished(self, request_id: int, future: asyncio.Future[Any]) -> None: + record = self.requests.pop(request_id, None) + if record is None: + if not future.cancelled(): + future.exception() + return + if not self.requests: + self.idle.set() + if future.cancelled(): + return + exc = future.exception() + result = {} if exc else (future.result() or {}) + outcome = (result.get("_meta") or {}).get("fx", {}).get("steering") + # Once a request owns a turn, its response always ends that turn. + # Accepted messages are never retried, whatever the steering outcome. + if self.running != request_id: + if not record.accepted and (outcome == "dropped" or exc): + self.retries.append(record) + if self.retry_task is None and not self.closed: + self.retry_task = asyncio.create_task(self._retry()) + return + if outcome == "absorbed" or record.accepted: + if not record.accepted: + self._accept(request_id, record) + else: + self._accept(request_id, record) + if self.running != request_id: + return + self.running = None + self.interrupted = ( + self.interrupted or result.get("stopReason") == "cancelled" + ) + self._emit("end", (result, exc, self.interrupted)) + + async def _retry(self) -> None: + try: + while self.retries and not self.closed: + await self.idle.wait() + self._rpc() + self.retries.sort(key=lambda record: record.order) + record = self.retries.pop(0) + await self._prompt(record) + except Exception as exc: + self._on_crash(exc) + finally: + self.retry_task = None + + async def _on_notify(self, method: str, params: dict[str, Any]) -> None: + if method != "session/update": + return + update = params.get("update") or {} + if self.loading is not None: + if update.get("sessionUpdate") == "user_message_chunk": + self.loading.add(str(update.get("messageId"))) + return + if params.get("sessionId") != self.session_id: + return + # After a cancel, late updates cannot start a waiting steer's turn. + # Its response decides whether it was dropped or ran on its own. + if self.running is None and self.interrupted: + return + kind = update.get("sessionUpdate") + if kind == "user_message_chunk": + request_id = ( + (update.get("_meta") or {}) + .get("fx", {}) + .get("steering", {}) + .get("requestId") + ) + record = self.requests.get(request_id) + if record is not None: + self._accept(request_id, record) + elif self.running is None and kind in ( + "agent_message_chunk", + "agent_thought_chunk", + "tool_call", + ): + candidate = next( + ( + (key, value) + for key, value in self.requests.items() + if not value.accepted + ), + None, + ) + if candidate is not None: + self._accept(*candidate) + if self.running is not None: + self._emit("update", update) + + async def _on_request( + self, request_id: Any, method: str, params: dict[str, Any] + ) -> None: + if method == "session/request_permission": + options = params.get("options") or [] + option = next( + ( + o["optionId"] + for o in options + if o.get("kind") in ("allow_once", "allow_always") + or o.get("optionId") == "allow_once" + ), + None, + ) + outcome = ( + {"outcome": "selected", "optionId": option} + if option + else {"outcome": "cancelled"} + ) + await self._rpc().respond(request_id, {"outcome": outcome}) + else: + raise ValueError(f"fx: unsupported server request {method}") + + def _on_crash(self, exc: BaseException) -> None: + if self.closed or self.failure is not None: + return + self.failure = errors.ExternalHarnessError(str(exc)) + self.requests.clear() + self.retries.clear() + self.running = None + self.idle.set() + self.queue.put_nowait(("crashed", self.failure)) + + async def interrupt(self) -> None: + if self.requests and self.peer is not None: + self.interrupted = True + await self.peer.notify( + "session/cancel", {"sessionId": self.session_id} + ) + + async def close(self) -> None: + if self.closed: + return + self.closed = True + self.requests.clear() + self.retries.clear() + self.running = None + self.idle.set() + task = self.retry_task + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + if self.peer is not None: + if self.session_id: + with contextlib.suppress(Exception): + await self.peer.call( + "session/close", + {"sessionId": self.session_id}, + timeout=2, + ) + await self.peer.close() + self.peer = None + if self.process is not None: + with contextlib.suppress(Exception): + await self.process.terminate() + self.process = None + + async def events(self) -> AsyncGenerator[_wire.Item]: + state = _TurnState() + while True: + kind, value = await self.queue.get() + if kind == "crashed": + raise errors.ExternalHarnessError(str(value)) + if kind == "accepted": + yield _wire.Accepted(value) + elif kind == "update": + for item in state.translate(value): + yield item + elif kind == "end": + for item in state.close_block(): + yield item + result, exc, cancelled = value + stop = result.get("stopReason") + reason: Literal["stop", "length", "interrupted", "error"] = ( + "stop" + ) + error = None + if cancelled or stop == "cancelled": + reason = "interrupted" + elif exc is not None or stop == "refused": + reason = "error" + error = errors.turn_failed( + KIND, + str(exc) if exc else state.recovery or state.last_text, + AUTH_HINT, + AUTH_MARKERS, + ) + elif stop in ("max_output_tokens", "max_model_turns"): + reason = "length" + raw = result.get("usage") + usage = ( + None + if raw is None + else usage_.Usage( + input_tokens=raw.get("inputTokens") or 0, + output_tokens=raw.get("outputTokens") or 0, + cache_read_tokens=raw.get("cacheReadTokens"), + cache_write_tokens=raw.get("cacheWriteTokens"), + reasoning_tokens=raw.get("reasoningTokens"), + raw=raw, + ) + ) + yield _wire.TurnEnd( + reason=reason, + error=error, + usage=usage, + metadata={"model": self.model, "fx_version": self.version}, + ) + state = _TurnState() + + +@dataclasses.dataclass +class _Prompt: + order: int + message: messages_.Message + prompt: list[dict[str, Any]] + accepted: bool = False + + +class _TurnState: + def __init__(self) -> None: + self.block: tuple[str, str] | None = None + self.counter = 0 + self.tools: dict[str, str] = {} + self.started: set[str] = set() + self.completed: set[str] = set() + self.last_text = "" + self.recovery = "" + + def close_block(self) -> Iterator[_wire.Content]: + if self.block is not None: + kind, block_id = self.block + self.block = None + if kind == "agent_message_chunk": + yield events_.TextEnd(block_id=block_id) + else: + yield events_.ReasoningEnd(block_id=block_id) + + def translate(self, update: dict[str, Any]) -> Iterator[_wire.Content]: + kind = update.get("sessionUpdate") + if kind in ("agent_message_chunk", "agent_thought_chunk"): + block_id = ( + str(update.get("messageId") or "") + if kind == "agent_message_chunk" + else ( + self.block[1] + if self.block and self.block[0] == kind + else f"reasoning-{self.counter}" + ) + ) + if self.block != (kind, block_id): + yield from self.close_block() + self.counter += 1 + self.block = (kind, block_id) + if kind == "agent_message_chunk": + self.last_text = "" + yield events_.TextStart(block_id=block_id) + else: + yield events_.ReasoningStart(block_id=block_id) + text = (update.get("content") or {}).get("text", "") + if kind == "agent_message_chunk": + self.last_text += text + yield events_.TextDelta(block_id=block_id, chunk=text) + else: + yield events_.ReasoningDelta(block_id=block_id, chunk=text) + return + if kind in ("tool_call", "tool_call_update", "user_message_chunk"): + yield from self.close_block() + call_id = str(update.get("toolCallId") or "") + if kind == "tool_call": + native = ( + (update.get("_meta") or {}).get("fx", {}).get("toolCall", {}) + ) + if native.get("mcp", {}).get("server") == _bridge.SERVER_NAME: + return + name = str(update.get("name") or "tool") + self.tools[call_id] = name + args = update.get("rawInput") or {} + yield from _utils.get_builtin_call_events( + call_id, + name, + args if isinstance(args, str) else json.dumps(args), + started_call_ids=self.started, + ) + elif kind == "tool_call_update" and update.get("status") in ( + "completed", + "failed", + ): + if call_id not in self.tools or call_id in self.completed: + return + self.completed.add(call_id) + text = "".join( + (p.get("content") or {}).get("text", "") + for p in update.get("content", []) + ) + yield _utils.get_builtin_result_event( + call_id, + self.tools.get(call_id, "tool"), + text, + is_error=update.get("status") == "failed", + ) + elif kind == "session_info_update": + recovery = (update.get("_meta") or {}).get("fx", {}).get( + "modelResponseRecovery" + ) or {} + self.recovery = recovery.get("message") or self.recovery diff --git a/src/ai/external/pi/__init__.py b/src/ai/external/pi/__init__.py new file mode 100644 index 00000000..ffa86c65 --- /dev/null +++ b/src/ai/external/pi/__init__.py @@ -0,0 +1,3 @@ +from .harness import Pi, PiProvider + +__all__ = ["Pi", "PiProvider"] diff --git a/src/ai/external/pi/extension.js b/src/ai/external/pi/extension.js new file mode 100644 index 00000000..ce7135bc --- /dev/null +++ b/src/ai/external/pi/extension.js @@ -0,0 +1,29 @@ +// Loaded into every `pi` the ai-python SDK launches (`pi -e `). +// +// AI_PYTHON_PI, a JSON object in the environment, configures it: +// +// - "mcp": MCP servers, registered for this session only. The external +// harness's MCP bridge (the `ai` tools) comes this way: pi has no flag for +// an MCP server, and a registration is not written to any config file. +// +// pi has no permission prompt, and the SDK asks for no approvals, +// so there is no gate here. + +// Registered LAST: its presence in `get_commands` is how the SDK knows every +// part above loaded. No marker, no session. +const READY = "ai-python-ready"; + +export default function (pi) { + // Unparseable configuration fails the load (and so the session) rather + // than starting without the bridge. + const config = JSON.parse(process.env.AI_PYTHON_PI || "{}") || {}; + + for (const [name, server] of Object.entries(config.mcp || {})) { + pi.registerMcpServer(name, server); + } + + pi.registerCommand(READY, { + description: "Marks the ai-python SDK's extension as loaded", + handler: async () => {}, + }); +} diff --git a/src/ai/external/pi/handoff.py b/src/ai/external/pi/handoff.py new file mode 100644 index 00000000..2f343cce --- /dev/null +++ b/src/ai/external/pi/handoff.py @@ -0,0 +1,165 @@ +"""Seeding pi with a past: `AgentMessage`s for a session file. + +The rule for what becomes what is the same for every CLI: see `_utils`. +""" + +from __future__ import annotations + +import json +from typing import Any + +from ...types import media +from ...types import messages as messages_ +from .. import _handoff_output + +# What names a reply's author in a pi session: its API and model. +AUTHOR = ("api", "provider", "model") +SYSTEM_PREFIX = "[Handoff from a previous session — instructions it was given: " + + +def to_pi_messages( + messages: list[messages_.Message], *, timestamp: int +) -> list[dict[str, Any]]: + """Build pi `AgentMessage`s for a session file. + + Assistant messages pi wrote keep their author (`provider_metadata["pi"]`): + pi restores a session's model from it. Any other is attributed to a + model pi does not run (`ai-python/handoff`), so pi treats it as another + model's — it keeps its text and tool calls and normalizes the call ids + for its own provider. Consecutive assistant messages are merged, the way + one model response holds text and calls together. Reasoning is omitted: + see `_utils`. + """ + out: list[dict[str, Any]] = [] + usage = { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "totalTokens": 0, + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "total": 0, + }, + } + + def result( + part: messages_.ToolResultPart | messages_.BuiltinToolReturnPart, + ) -> None: + out.append( + { + "role": "toolResult", + "toolCallId": part.tool_call_id, + "toolName": part.tool_name, + "content": [ + { + "type": "text", + "text": _handoff_output.format_result(part), + } + ], + "isError": part.is_error, + "timestamp": timestamp, + } + ) + + def add(m: messages_.Message, blocks: list[dict[str, Any]]) -> None: + if not blocks: + return + if m.role == "system": + # Measured (pi 1.3.0): a system message in a session file is + # pi's own prompt state, and replaying one REPLACED pi's system + # prompt and its tools. So instructions travel as what they are + # to the agent — the caller's words — never as its own output. + text = " ".join(b["text"] for b in blocks if b["type"] == "text") + blocks = [{"type": "text", "text": SYSTEM_PREFIX + text + "]"}] + if m.role in ("user", "system"): + out.append( + {"role": "user", "content": blocks, "timestamp": timestamp} + ) + return + if out and out[-1]["role"] == "assistant": + out[-1]["content"] += blocks + else: + # A reply pi wrote keeps its author: pi restores a session's model + # from its last reply's, and a model it does not know (ours) + # puts the conversation on pi's default one (measured). + author = (m.provider_metadata or {}).get("pi") or {} + known = all(isinstance(author.get(k), str) for k in AUTHOR) + out.append( + { + "role": "assistant", + "content": blocks, + **( + {k: author[k] for k in AUTHOR} + if known + else { + "api": "ai-python", + "provider": "ai-python", + "model": "handoff", + } + ), + "usage": usage, + "stopReason": "stop", + "timestamp": timestamp, + } + ) + if any(b["type"] == "toolCall" for b in out[-1]["content"]): + out[-1]["stopReason"] = "toolUse" + + for m in messages: + blocks: list[dict[str, Any]] = [] + for p in m.parts: + if isinstance(p, messages_.ToolResultPart): + # A tool an `ai.Agent` ran: its result is in a `tool` message. + result(p) + elif isinstance(p, messages_.BuiltinToolReturnPart): + # A tool the CLI ran: the external harness holds its result in + # the assistant message, pi as a message of its own. So the + # assistant message ends here, and the result follows. + add(m, blocks) + blocks = [] + result(p) + elif isinstance(p, messages_.TextPart) and p.text.strip(): + blocks.append({"type": "text", "text": p.text}) + elif ( + isinstance(p, messages_.FilePart) + and m.role == "user" + and p.media_type.startswith("image/") + ): + if isinstance(p.data, str) and media.is_downloadable_url( + p.data + ): + blocks.append( + {"type": "text", "text": f"[image: {p.data}]"} + ) + else: + blocks.append( + { + "type": "image", + "data": media.data_to_base64(p.data), + "mimeType": p.media_type, + } + ) + elif ( + isinstance( + p, messages_.ToolCallPart | messages_.BuiltinToolCallPart + ) + and m.role == "assistant" + ): + try: + args = json.loads(p.tool_args) if p.tool_args else {} + except json.JSONDecodeError: + args = {"raw": p.tool_args} + blocks.append( + { + "type": "toolCall", + "id": p.tool_call_id or p.id, + "name": p.tool_name, + "arguments": args if isinstance(args, dict) else {}, + } + ) + add(m, blocks) + return out diff --git a/src/ai/external/pi/harness.py b/src/ai/external/pi/harness.py new file mode 100644 index 00000000..a7aeb529 --- /dev/null +++ b/src/ai/external/pi/harness.py @@ -0,0 +1,42 @@ +"""Pi: the provider and the settings. `wire.py` drives `pi --mode rpc`.""" + +from __future__ import annotations + +import dataclasses +from typing import ClassVar, Literal + +from .. import _harness +from . import wire + + +class PiProvider(_harness.ExternalHarnessProvider): + """The pi CLI.""" + + name: str = "pi" + + def _make_wire(self, harness: Pi) -> wire.PiWire: + return wire.PiWire(harness) + + +@dataclasses.dataclass(kw_only=True, eq=False) +class Pi(_harness.ExternalHarness): + """Pi, over `pi --mode rpc`. + + `model_provider` picks the provider of `model_name` (`--provider`), and + `thinking` the thinking level. `allowed_tools` limits pi's own tools + (`["read", "grep"]`, or `[]` for none; `ai` tools stay), and + `disallowed_tools` takes some away. `args` go to pi as they are. pi never + asks for approval. + """ + + harness_class_id: ClassVar[str] = "pi" + + provider: PiProvider = dataclasses.field(default_factory=PiProvider) + model_provider: str | None = None + thinking: ( + Literal["off", "minimal", "low", "medium", "high", "xhigh", "max"] + | None + ) = None + allowed_tools: list[str] | None = None + disallowed_tools: list[str] | None = None + args: list[str] = dataclasses.field(default_factory=list) diff --git a/src/ai/external/pi/wire.py b/src/ai/external/pi/wire.py new file mode 100644 index 00000000..f0372710 --- /dev/null +++ b/src/ai/external/pi/wire.py @@ -0,0 +1,788 @@ +"""Pi, driven through `pi --mode rpc`. + +One pi process per session: an RPC process holds one session. The id we +hand pi (`--session-id`) IS its session id, so `pi --session-id ` opens +the same conversation in pi's own TUI. + +Every pi an external harness starts loads `extension.js`: pi has no flag for an +MCP server, so the extension registers the external harness's MCP bridge, and a +marker command it registers last proves that it loaded. + +The protocol shapes here were measured against a real pi (see +VERIFIED_VERSION), not taken from the docs alone. +""" + +from __future__ import annotations + +import asyncio +import base64 +import collections +import contextlib +import datetime +import json +import os +import pathlib +import re +import uuid +import warnings +from collections.abc import AsyncGenerator, Iterator +from typing import TYPE_CHECKING, Any, Literal + +from ...types import events as events_ +from ...types import messages as messages_ +from ...types import usage as usage_ +from .. import _bridge, _process, _utils, _wire +from .. import _errors as errors +from . import handoff + +if TYPE_CHECKING: + from .harness import Pi + +KIND = "pi" +INSTALL = "npm install -g @earendil-works/pi-coding-agent" +# The CLI version the live tests last passed with. Older ones get a warning. +VERIFIED_VERSION = "1.5.0" +EXTENSION = str(pathlib.Path(__file__).with_name("extension.js")) +# The command the extension registers last: proof that it loaded. +READY = "ai-python-ready" +# The environment variable that configures the extension. +CONFIG_VAR = "AI_PYTHON_PI" +# pi names an MCP tool `mcp____` (with a hash suffix when the +# name is longer than 64 characters; the prefix stays). +BRIDGE_PREFIX = f"mcp__{_bridge.SERVER_NAME}__" +# pi's rule for a session id: letters, digits, `.`, `_` and `-`, starting and +# ending with a letter or digit. Never a path. +SESSION_ID = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]{0,198}[A-Za-z0-9])?\Z") +# Extension UI methods that wait for an answer; the rest are notices. +DIALOGS = ("select", "confirm", "input", "editor") + +AUTH_HINT = ( + "pi uses its own login (`/login`, auth.json) or a provider key in the " + "environment: pass env={'ANTHROPIC_API_KEY': ...}." +) +# The CLI's own phrases for "no usable credentials". +AUTH_MARKERS = ("please run /login",) + + +class _CommandError(Exception): + """pi answered a command with `success: false`.""" + + +class PiWire: + def __init__(self, harness: Pi) -> None: + self.harness = harness + # pi records a session's directory as its real path (on macOS a temp + # dir /tmp/... is recorded as /private/tmp/..., measured). + self.cwd = os.path.realpath(harness.cwd) + self.process: _process.Process | None = None + self.reader: asyncio.Task[None] | None = None + self.dead: str | None = None + self.waiters: dict[str, asyncio.Future[Any]] = {} + # Unbounded: the reader fills it inline and must not wait. + self.queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + # User messages that pi took but did not put in the conversation yet, + # in the order sent. See `send`. + self.pending: collections.deque[messages_.Message] = collections.deque() + # Between `agent_start` and `agent_settled`, as the reader saw it. + self.streaming = False + # Assistant messages seen in the session: block ids are unique. + self.count = 0 + + # -- life ----------------------------------------------------------------- + + async def start( + self, + *, + session_id: str | None, + bridge_url: str, + bridge_token: str, + instructions: str | None, + seed: list[messages_.Message], + ) -> str: + harness = self.harness + executable = harness.provider.executable or "pi" + await _process.probe( + KIND, + executable, + install=INSTALL, + verified=VERIFIED_VERSION, + cwd=self.cwd, + env=harness.provider.env, + ) + if session_id is not None: + # Checked first: `--session-id` CREATES a session it cannot + # find, so a typo would silently start an empty conversation. + await asyncio.to_thread(self._find, session_id) + else: + session_id = str(uuid.uuid4()) + if seed: + await asyncio.to_thread(self._stage, session_id, seed) + argv = [ + executable, + "--mode", + "rpc", + "--session-id", + session_id, + "-e", + EXTENSION, + ] + if harness.model_name: + argv += ["--model", harness.model_name] + if harness.model_provider: + argv += ["--provider", harness.model_provider] + if harness.thinking: + argv += ["--thinking", harness.thinking] + if instructions: + argv += ["--append-system-prompt", instructions] + if harness.allowed_tools is not None: + # Not `--no-tools`: it disables extension tools too, and the + # MCP bridge's tools are extension tools. + argv += ( + ["--tools", ",".join(harness.allowed_tools)] + if harness.allowed_tools + else ["--no-builtin-tools"] + ) + if harness.disallowed_tools: + argv += ["--exclude-tools", ",".join(harness.disallowed_tools)] + argv += harness.args + bridge = { + "url": bridge_url, + "headers": {"Authorization": f"Bearer {bridge_token}"}, + # pi's default exposure is "codemode": the model reaches MCP + # tools through a script, as nested calls. "direct" declares + # each tool to the model, and the first prompt waits until the + # bridge is connected. + "exposure": "direct", + # An `ai` tool can take longer than pi's default (60 seconds). + "timeout": 24 * 3600, + } + config = {"mcp": {_bridge.SERVER_NAME: bridge}} + self.process = await _process.spawn( + argv, + cwd=self.cwd, + env={**harness.provider.env, CONFIG_VAR: json.dumps(config)}, + ) + self.reader = asyncio.create_task(self._read(self.process)) + try: + # A pi that cannot start (an unknown model, a bad flag) exits at + # once, and these calls report its last words. + state = await self._call({"type": "get_state"}, timeout=60) + reported = (state or {}).get("sessionId") + if reported and reported != session_id: + raise errors.ExternalHarnessError( + f"{KIND} opened session {reported}, not {session_id}" + ) + listed = await self._call({"type": "get_commands"}, timeout=60) + names = { + c.get("name") for c in (listed or {}).get("commands") or [] + } + if READY not in names: + raise errors.ExternalHarnessError( + f"{KIND} did not load the SDK's extension ({EXTENSION}); " + "without it pi has no `ai` tools" + ) + except BaseException: + # A failed start must not leave pi running. + await self.close() + raise + return session_id + + async def close(self) -> None: + reader, self.reader = self.reader, None + if reader is not None: + reader.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await reader + self._fail("closed") + process, self.process = self.process, None + if process is not None: + with contextlib.suppress(Exception): + await process.terminate() + + # -- input ---------------------------------------------------------------- + + async def send(self, message: messages_.Message) -> None: + _utils.raise_for_unsupported_parts(message, KIND, ("image/",)) + command = await _prompt_command(message) + # `Accepted` comes from pi's own record of the message: the + # `message_start` of a user message, which pi emits when the message + # enters the conversation and before anything the message causes + # (measured, for a prompt and for a steer). The command response + # does not order anything: for a new run, it can come after the + # run's records; for a steer, it comes long before the message is + # delivered. So the message waits in `pending`, registered BEFORE + # the write, and the reader matches user messages to it in order. + self.pending.append(message) + try: + # Always `prompt` with `steer`: when pi is idle it starts a run, + # when a run streams it queues the message for the next model + # call. pi chooses atomically, so a send at the very end of a run + # is never stranded (`steer` alone queues even when idle, and + # nothing then delivers it). Measured: "started" when idle, + # "queued" while streaming. A queued steer is delivered before + # the run settles, also after `abort`. + accepted = await self._call( + {**command, "streamingBehavior": "steer"} + ) + except _CommandError as exc: + # Rejected before acceptance: pi will never report it. + # Measured: pi refuses a prompt itself when it has no key. + self._forget(message) + raise errors.turn_failed( + KIND, str(exc), AUTH_HINT, AUTH_MARKERS + ) from exc + disposition = (accepted or {}).get("disposition") + if disposition == "handled": + # An extension command or input handler consumed it: no user + # message comes. A command may start a run of its own (measured: + # then pi already streams, and that run follows); if nothing + # runs, the turn is over. + self._forget(message) + state = await self._call({"type": "get_state"}) + self.queue.put_nowait(("accepted", message)) + if not (state or {}).get("isStreaming") and not self.streaming: + self.queue.put_nowait(("handled", None)) + elif disposition not in ("started", "queued"): + # An ambiguous acceptance must never be retried as a new turn. + raise errors.ExternalHarnessError( + f"{KIND}: unknown prompt disposition: {accepted}" + ) + + def _forget(self, message: messages_.Message) -> None: + with contextlib.suppress(ValueError): + self.pending.remove(message) + + async def interrupt(self) -> None: + if self.process is None or self.dead is not None: + return + # `abort` answers only after the run settled (measured: the aborted + # message, `agent_end` and `agent_settled {aborted: true}` come + # first). So it is sent, not awaited: the end comes as a TurnEnd. + with contextlib.suppress(Exception): + await self._write({"type": "abort", "id": uuid.uuid4().hex}) + + # -- from pi -------------------------------------------------------------- + + async def _call( + self, command: dict[str, Any], *, timeout: float = 120 + ) -> Any: + if self.dead is not None: + raise errors.ExternalHarnessError(f"{KIND} crashed: {self.dead}") + request_id = uuid.uuid4().hex + future: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + self.waiters[request_id] = future + try: + await self._write({**command, "id": request_id}) + async with asyncio.timeout(timeout): + return await future + finally: + self.waiters.pop(request_id, None) + + async def _write(self, record: dict[str, Any]) -> None: + if self.process is None: + raise errors.ExternalHarnessError(f"{KIND}: pi is not running") + try: + await self.process.write(json.dumps(record) + "\n") + except Exception as exc: + raise errors.ExternalHarnessError( + f"{KIND}: could not write to pi: {exc}" + ) from exc + + def _fail(self, detail: str) -> None: + self.dead = self.dead or detail + for future in self.waiters.values(): + if not future.done(): + future.set_exception( + errors.ExternalHarnessError(f"{KIND} crashed: {detail}") + ) + self.waiters.clear() + + async def _read(self, process: _process.Process) -> None: + try: + while True: + line = await process.readline() + if not line: + detail = await process.death_note() + self._fail(detail) + self.queue.put_nowait(("crashed", detail)) + return + try: + record = json.loads(line) + except json.JSONDecodeError: + continue # stdout is protocol only; tolerate noise + if not isinstance(record, dict): + continue + kind = record.get("type") + if kind == "response": + future = self.waiters.get(str(record.get("id"))) + if future is None or future.done(): + continue + if record.get("success") is False: + future.set_exception( + _CommandError(str(record.get("error") or "")) + ) + else: + future.set_result(record.get("data")) + elif kind == "extension_ui_request": + if record.get("method") in DIALOGS: + # Another extension asks a person, who is not there: + # external harnesses do not ask for approval. + # Unanswered, the agent would wait. + await self._write( + { + "type": "extension_ui_response", + "id": record.get("id"), + "cancelled": True, + } + ) + else: + if kind == "agent_start": + self.streaming = True + elif kind == "agent_settled": + self.streaming = False + self.queue.put_nowait(("record", record)) + except asyncio.CancelledError: + raise + except Exception as exc: + self._fail(str(exc)) + self.queue.put_nowait(("crashed", str(exc))) + + async def events(self) -> AsyncGenerator[_wire.Item]: + run = _Run() + while True: + kind, value = await self.queue.get() + if kind == "crashed": + raise errors.ExternalHarnessError(f"{KIND} crashed: {value}") + if kind == "accepted": + yield _wire.Accepted(value) + elif kind == "handled": + yield _wire.TurnEnd() + else: + if value.get("type") == "agent_start": + run = _Run() + for item in self._translate(value, run): + yield item + + def _translate( + self, record: dict[str, Any], run: _Run + ) -> Iterator[_wire.Item]: + kind = record.get("type") + message = record.get("message") or {} + role = message.get("role") + if kind == "message_start" and role == "user": + # pi's own system message (`role: "system"`, pi 1.5) and other + # roles are not ours. A user message nobody sent here (an + # extension's) has nothing pending. + if self.pending: + yield _wire.Accepted(self.pending.popleft()) + elif kind == "message_start" and role == "assistant": + self.count += 1 + elif kind == "message_update": + yield from _stream_events( + record.get("assistantMessageEvent") or {}, str(self.count), run + ) + elif kind == "message_end" and role == "assistant": + run.last = message + raw = message.get("usage") or {} + run.usage = run.usage + _usage(raw) + run.cost += float((raw.get("cost") or {}).get("total") or 0) + yield from _close_blocks(run) + if message.get("stopReason") != "error": + # What the message holds that was never streamed (a run an + # extension command started, measured). + yield from _unstreamed( + message.get("content"), str(self.count), run + ) + elif kind == "tool_execution_end": + call_id = str(record.get("toolCallId") or "") + name = str(record.get("toolName") or "") + # A nested call (a codemode script's) is not in the transcript: + # only the tool that made it reports it. A bridge call is the + # external harness's to report. + if ( + not record.get("parentToolCallId") + and call_id not in run.hidden + and not name.startswith(BRIDGE_PREFIX) + ): + yield _utils.get_builtin_result_event( + call_id, + name, + _tool_result(record.get("result") or {}), + is_error=bool(record.get("isError")), + ) + elif kind == "auto_retry_end" and not record.get("success"): + run.retry_error = str(record.get("finalError") or "") + elif kind == "agent_settled": + # `agent_end` closes one low-level run, but retries, compaction + # and queued steers can follow it. `agent_settled` means pi will + # not continue on its own: the turn is over. + yield from _close_blocks(run) + stop = run.last.get("stopReason") + usage = run.usage.model_copy(update={"raw": {"cost_usd": run.cost}}) + written = _written_by(run.last) + if record.get("aborted") or stop == "aborted": + yield _wire.TurnEnd( + reason="interrupted", usage=usage, metadata=written + ) + elif stop == "error": + error = errors.turn_failed( + KIND, + str(run.last.get("errorMessage") or run.retry_error), + AUTH_HINT, + AUTH_MARKERS, + ) + yield _wire.TurnEnd(reason="error", usage=usage, error=error) + else: + reason: Literal["stop", "length"] = ( + "length" if stop == "length" else "stop" + ) + yield _wire.TurnEnd( + reason=reason, usage=usage, metadata=written + ) + + # -- the store ------------------------------------------------------------ + + def _session_dir(self) -> pathlib.Path: + """Where pi keeps this directory's sessions, decided as pi does. + + The environment, then project settings, then global ones; else one + directory per working directory under the agent dir. + """ + env = {**os.environ, **self.harness.provider.env} + agent = pathlib.Path( + os.path.expanduser(env.get("PI_CODING_AGENT_DIR") or "~/.pi/agent") + ) + custom = ( + env.get("PI_CODING_AGENT_SESSION_DIR") + or _setting(pathlib.Path(self.cwd, ".pi", "settings.json")) + or _setting(agent / "settings.json") + ) + if custom: + return pathlib.Path(self.cwd, os.path.expanduser(custom)) + safe = self.cwd.lstrip("/\\") + for c in "/\\:": + safe = safe.replace(c, "-") + return agent / "sessions" / f"--{safe}--" + + def _find(self, session_id: str) -> None: + """Refuse a session that is not this directory's. + + A session file can come from anyone (a project's settings choose the + directory), and its id becomes a CLI argument: one that breaks pi's + rule is not a session. + """ + if not SESSION_ID.match(session_id): + raise errors.ExternalHarnessError( + f"{KIND}: {session_id!r} is not a pi session id" + ) + try: + paths = sorted(self._session_dir().glob("*.jsonl")) + except OSError: + paths = [] + for path in paths: + try: + with path.open() as f: + header = json.loads(f.readline() or "{}") + except Exception: + continue + if ( + not isinstance(header, dict) + or header.get("type") != "session" + or header.get("id") != session_id + ): + continue + cwd = header.get("cwd") + if cwd not in (self.cwd, self.harness.cwd): + raise errors.ExternalHarnessError( + f"{KIND}: session {session_id} belongs to {cwd}, " + f"not {self.cwd}" + ) + return + raise errors.ExternalHarnessError( + f"{KIND}: no session {session_id} in {self.cwd}" + ) + + def _stage(self, session_id: str, seed: list[messages_.Message]) -> None: + """Write a session file with this past; start nothing on it.""" + _utils.warn_unsupported_files( + seed, + KIND, + accepted_media_types=("image/",), + accepted_description="images", + ) + dropped = _utils.count_reasoning_parts(seed) + if dropped: + # pi drops thinking another model produced, and replaying it as + # user text is refused by Anthropic's safeguards. + warnings.warn( + f"{dropped} reasoning part(s) in the history were not carried " + "into pi: a model only reads its own reasoning. Text, tool " + "calls and images were carried in full.", + stacklevel=2, + ) + now = datetime.datetime.now(datetime.UTC) + stamp = now.isoformat(timespec="milliseconds").replace("+00:00", "Z") + lines: list[dict[str, Any]] = [ + { + "type": "session", + "version": 3, + "id": session_id, + "timestamp": stamp, + "cwd": self.cwd, + } + ] + # The model the conversation was on, as pi records a switch: pi + # restores a session's model from its last reply's author. Measured: + # without it, a seeded conversation ran on pi's default model. The + # wire records the author of each turn's last reply in + # `provider_metadata["pi"]` (`_written_by`). + written: dict[str, Any] = next( + ( + found + for m in reversed(seed) + if m.role == "assistant" + and isinstance( + found := (m.provider_metadata or {}).get(KIND), dict + ) + ), + {}, + ) + entries: list[dict[str, Any]] = [] + if written.get("provider") and written.get("model"): + entries.append( + { + "type": "model_change", + "provider": written["provider"], + "modelId": written["model"], + } + ) + if written.get("thinkingLevel"): + entries.append( + { + "type": "thinking_level_change", + "thinkingLevel": written["thinkingLevel"], + } + ) + entries += [ + {"type": "message", "message": message} + for message in handoff.to_pi_messages( + seed, timestamp=int(now.timestamp() * 1000) + ) + ] + parent: str | None = None + for entry in entries: + # A chain, root first: pi reads it as one branch. + entry_id = uuid.uuid4().hex[:8] + lines.append( + { + **entry, + "id": entry_id, + "parentId": parent, + "timestamp": stamp, + } + ) + parent = entry_id + name = stamp.replace(":", "-").replace(".", "-") + path = self._session_dir() / f"{name}_{session_id}.jsonl" + _utils.write_seed_transcript( + path, "".join(json.dumps(line) + "\n" for line in lines) + ) + + +class _Run: + """What one pi run's translation remembers.""" + + def __init__(self) -> None: + # Open text and thinking blocks, by block id: "text" or "thinking". + self.open: dict[str, str] = {} + self.streamed: set[str] = set() + self.started: set[str] = set() + # Bridge calls: the external harness reports them, not the wire. + self.hidden: set[str] = set() + self.last: dict[str, Any] = {} + self.usage = usage_.Usage() + self.cost = 0.0 + # The last error pi reported while it retried: if the run fails + # without a message of its own, the only explanation. + self.retry_error = "" + + +async def _prompt_command(message: messages_.Message) -> dict[str, Any]: + """Return a user message as pi's `prompt`: text, and images as base64.""" + texts: list[str] = [] + images: list[dict[str, Any]] = [] + for part in message.parts: + if isinstance(part, messages_.TextPart): + texts.append(part.text) + elif isinstance(part, messages_.FilePart): + if part.media_type.startswith("text/"): + texts.append(await _utils.format_text_file(part)) + else: + encoded = base64.b64encode(await _utils.extract_data(part)) + images.append( + { + "type": "image", + "data": encoded.decode("ascii"), + "mimeType": part.media_type, + } + ) + command: dict[str, Any] = {"type": "prompt", "message": "\n\n".join(texts)} + if images: + command["images"] = images + return command + + +def _stream_events( + update: dict[str, Any], message: str, run: _Run +) -> list[_wire.Content]: + """One `assistantMessageEvent` -> events. + + Block ids are `.`: pi numbers content blocks per + assistant message, and a turn holds several. Tool-call arguments travel + as ONE delta, the final arguments, at `toolcall_end`: streamed fragments + are not guaranteed to add up to what pi settles on. + """ + kind = update.get("type") or "" + block = f"{message}.{update.get('contentIndex', 0)}" + out: list[_wire.Content] = [] + if kind == "text_start": + run.open[block] = "text" + run.streamed.add(block) + out.append(events_.TextStart(block_id=block)) + elif kind == "text_delta" and update.get("delta"): + out.append(events_.TextDelta(chunk=update["delta"], block_id=block)) + elif kind == "thinking_start": + run.open[block] = "thinking" + run.streamed.add(block) + out.append(events_.ReasoningStart(block_id=block)) + elif kind == "thinking_delta" and update.get("delta"): + out.append( + events_.ReasoningDelta(chunk=update["delta"], block_id=block) + ) + elif kind in ("text_end", "thinking_end"): + out += _close_blocks(run, only=block) + elif kind == "toolcall_start" and update.get("id"): + call_id = str(update["id"]) + name = str(update.get("toolName") or "") + if name.startswith(BRIDGE_PREFIX): + run.hidden.add(call_id) + else: + run.started.add(call_id) + out.append( + events_.BuiltinToolStart(tool_call_id=call_id, tool_name=name) + ) + elif kind == "toolcall_end": + out += _builtin_call(update.get("toolCall") or {}, run) + return out + + +def _unstreamed( + content: Any, message: str, run: _Run +) -> Iterator[_wire.Content]: + """Yield the events of a finished message's unstreamed blocks.""" + for index, block in enumerate(content if isinstance(content, list) else []): + if not isinstance(block, dict): + continue + block_id = f"{message}.{index}" + kind = block.get("type") + if ( + kind == "text" + and block.get("text") + and block_id not in run.streamed + ): + yield events_.TextStart(block_id=block_id) + yield events_.TextDelta(chunk=block["text"], block_id=block_id) + yield events_.TextEnd(block_id=block_id) + elif ( + kind == "thinking" + and block.get("thinking") + and block_id not in run.streamed + ): + yield events_.ReasoningStart(block_id=block_id) + yield events_.ReasoningDelta( + chunk=block["thinking"], block_id=block_id + ) + yield events_.ReasoningEnd(block_id=block_id) + elif kind == "toolCall" and str(block.get("id")) not in run.started: + yield from _builtin_call(block, run) + + +def _builtin_call(block: dict[str, Any], run: _Run) -> list[_wire.Content]: + """Return the events of a finished pi tool call; none for a bridge's.""" + call_id = str(block.get("id") or "") + name = str(block.get("name") or "") + if call_id in run.hidden or name.startswith(BRIDGE_PREFIX): + run.hidden.add(call_id) + return [] + args = json.dumps(block.get("arguments") or {}) + return list( + _utils.get_builtin_call_events( + call_id, name, args, started_call_ids=run.started + ) + ) + + +def _close_blocks(run: _Run, only: str | None = None) -> list[_wire.Content]: + """End the text and reasoning blocks still open (an aborted message's).""" + out: list[_wire.Content] = [] + for block in [only] if only is not None else list(run.open): + opened = run.open.pop(block, None) + if opened == "text": + out.append(events_.TextEnd(block_id=block)) + elif opened == "thinking": + out.append(events_.ReasoningEnd(block_id=block)) + return out + + +def _tool_result(result: Any) -> Any: + """Return what a tool returned: its text, when that is all it is. + + pi tools return content blocks; text-only content (nearly every call) + reads as the string it is, anything else stays the blocks themselves. + """ + content = result.get("content") if isinstance(result, dict) else result + if isinstance(content, list) and all( + isinstance(b, dict) and b.get("type") == "text" for b in content + ): + content = "".join(b.get("text") or "" for b in content) + return content + + +def _usage(raw: dict[str, Any]) -> usage_.Usage: + """Return pi's per-message usage. pi's `input` excludes cached input.""" + + def count(name: str) -> int: + value = raw.get(name) + return value if isinstance(value, int) else 0 + + reasoning = raw.get("reasoning") + return usage_.Usage( + input_tokens=count("input") + count("cacheRead") + count("cacheWrite"), + output_tokens=count("output"), + cache_read_tokens=count("cacheRead"), + cache_write_tokens=count("cacheWrite"), + reasoning_tokens=reasoning if isinstance(reasoning, int) else None, + ) + + +def _setting(path: pathlib.Path) -> str | None: + """Return the `sessionDir` a pi settings file names, if any.""" + try: + value = json.loads(path.read_text()).get("sessionDir") + except Exception: + return None + return value if isinstance(value, str) and value else None + + +def _written_by(message: dict[str, Any]) -> dict[str, Any] | None: + """Say which provider and model wrote an assistant message, as pi does. + + Kept on the message (`provider_metadata["pi"]`) so a conversation + seeded into another pi continues on the model it was on. + """ + written = { + k: message[k] + for k in ("api", "provider", "model", "thinkingLevel") + if isinstance(message.get(k), str) + } + return {KIND: written} if written else None diff --git a/src/ai/models/__init__.py b/src/ai/models/__init__.py index 1b3b99d8..eb9f80ac 100644 --- a/src/ai/models/__init__.py +++ b/src/ai/models/__init__.py @@ -39,7 +39,7 @@ probe, stream, ) -from .core.model import Model, get_model +from .core.model import Model, ModelLike, ProviderLike, get_model from .core.params import ( DEFAULT, GLOBAL, @@ -85,9 +85,11 @@ "InferenceRequestParams", "MinPSamplerParams", "Model", + "ModelLike", "ModelProviderDefault", "OutputParams", "Provider", + "ProviderLike", "ProviderProtocol", "ProviderRankingStrategy", "ProviderServiceParams", diff --git a/src/ai/models/core/api.py b/src/ai/models/core/api.py index 4e6f10bf..8f0c6203 100644 --- a/src/ai/models/core/api.py +++ b/src/ai/models/core/api.py @@ -37,7 +37,7 @@ @dataclasses.dataclass(frozen=True) class _StreamRequest: - model: model_.Model + model: model_.ModelLike messages: list[types.messages.Message] tools: Sequence[types.tools.Tool] | None = None output_type: type[pydantic.BaseModel] | None = None @@ -66,14 +66,19 @@ async def _do_stream( self, request: _StreamRequest, ) -> AsyncGenerator[types.events.Event]: - async for ev in request.model.provider.stream( - request.model, - request.messages, - tools=request.tools, - output_type=request.output_type, - params=request.params, - ): - yield ev + # Closed with the outer generator, not left to the garbage + # collector: a provider with state cleans up in it. + async with contextlib.aclosing( + request.model.provider.stream( + request.model, + request.messages, + tools=request.tools, + output_type=request.output_type, + params=request.params, + ) + ) as inner: + async for ev in inner: + yield ev async def _do_generate( self, request: _StreamRequest @@ -307,7 +312,7 @@ class StreamContext(Protocol): """ @property - def model(self) -> model_.Model: ... + def model(self) -> model_.ModelLike: ... @property def messages(self) -> list[types.messages.Message]: ... @property @@ -333,7 +338,7 @@ def stream[T: pydantic.BaseModel]( ) -> AbstractAsyncContextManager[Stream[T]]: ... @overload def stream( - model: model_.Model, + model: model_.ModelLike, messages: list[types.messages.Message], *, tools: Sequence[types.tools.Tool] | None = None, @@ -341,7 +346,7 @@ def stream( ) -> AbstractAsyncContextManager[Stream[str]]: ... @overload def stream[T: pydantic.BaseModel]( - model: model_.Model, + model: model_.ModelLike, messages: list[types.messages.Message], *, tools: Sequence[types.tools.Tool] | None = None, @@ -349,7 +354,7 @@ def stream[T: pydantic.BaseModel]( params: params_.InferenceRequestParams | None = None, ) -> AbstractAsyncContextManager[Stream[T]]: ... def stream( - model: model_.Model | None = None, + model: model_.ModelLike | None = None, messages: list[types.messages.Message] | None = None, *, context: StreamContext | None = None, @@ -401,7 +406,7 @@ def stream( @contextlib.asynccontextmanager async def _stream( *, - model: model_.Model, + model: model_.ModelLike, messages: list[types.messages.Message], tools: Sequence[types.tools.Tool] | None, output_type: type[pydantic.BaseModel] | None, @@ -467,7 +472,7 @@ async def experimental_generate( ) -> types.messages.Message: ... @overload async def experimental_generate( - model: model_.Model, + model: model_.ModelLike, messages: list[types.messages.Message], *, tools: Sequence[types.tools.Tool] | None = None, @@ -475,7 +480,7 @@ async def experimental_generate( params: params_.InferenceRequestParams | None = None, ) -> types.messages.Message: ... async def experimental_generate( - model: model_.Model | None = None, + model: model_.ModelLike | None = None, messages: list[types.messages.Message] | None = None, *, context: StreamContext | None = None, diff --git a/src/ai/models/core/model.py b/src/ai/models/core/model.py index e944767d..89d1ae7d 100644 --- a/src/ai/models/core/model.py +++ b/src/ai/models/core/model.py @@ -1,18 +1,23 @@ """Model metadata types.""" +from __future__ import annotations + import os import weakref -from typing import TYPE_CHECKING, Any, Self, cast +from typing import TYPE_CHECKING, Any, Protocol, Self, cast, runtime_checkable import pydantic -if TYPE_CHECKING: - from collections.abc import Callable - from ... import _modelsdev from ...errors import ConfigurationError from ...providers import base +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable, Sequence + + from ... import types + from . import params as params_ + _DEFAULT_MODEL_ENV = "AI_SDK_DEFAULT_MODEL" @@ -24,6 +29,62 @@ def _exclude_if_none(v: Any) -> bool: return v is None +class ProviderLike[M](Protocol): + """What answers for a model.""" + + @property + def name(self) -> str: ... + + def stream( + self, + model: M, + messages: list[types.messages.Message], + /, + *, + tools: Sequence[types.tools.Tool] | None = None, + output_type: type[pydantic.BaseModel] | None = None, + params: params_.InferenceRequestParams | None = None, + ) -> AsyncGenerator[types.events.Event]: + """Answer the WHOLE conversation with ONE assistant message. + + Yield its events, ending with `StreamEnd`. + """ + ... + + async def generate( + self, + model: M, + messages: list[types.messages.Message], + /, + *, + tools: Sequence[types.tools.Tool] | None = None, + output_type: type[pydantic.BaseModel] | None = None, + params: params_.InferenceRequestParams | None = None, + ) -> types.messages.Message: + """Answer without streaming. + + Optional: raise `NotImplementedError`, and + `ai.experimental_generate` drains `stream` instead. + """ + ... + + +@runtime_checkable +class ModelLike(Protocol): + """What `ai.stream` and `ai.Agent.run` need from a model. + + An id and the provider that answers for it. `ai.Model` and its + `ai.Provider` have this shape. Anything else with the same shape goes + where a model goes. + """ + + @property + def id(self) -> str: ... + + @property + def provider(self) -> ProviderLike[Self]: ... + + class Model(pydantic.BaseModel): """Lightweight reference to a model on a specific provider. diff --git a/tests/external/__init__.py b/tests/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/external/claude_code/__init__.py b/tests/external/claude_code/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/external/claude_code/test_wire.py b/tests/external/claude_code/test_wire.py new file mode 100644 index 00000000..361a6d5f --- /dev/null +++ b/tests/external/claude_code/test_wire.py @@ -0,0 +1,711 @@ +"""The Claude Code external harness against a scripted SDK client, offline. + +The fake replaces `ClaudeSDKClient` only: the records it plays are the real +SDK message types, in the order a real `claude` 2.1.319 sends them. +""" + +from __future__ import annotations + +import asyncio +import copy +import json +import os +from collections.abc import AsyncIterable, AsyncIterator, Callable +from pathlib import Path +from typing import Any + +import pytest + +import ai +from ai import external +from ai.external import _process +from ai.types import messages as m + +from .. import fakes + +sdk = pytest.importorskip("claude_agent_sdk") + +from ai.external.claude_code import ClaudeCode # noqa: E402 + +SESSION = "11111111-1111-1111-1111-111111111111" + +OnQuery = Callable[[dict[str, Any]], list[Any]] + + +class FakeCLI: + """What the test scripts: the records each written message causes. + + `on_query(record)` returns what the CLI says when it reads a message + (default: echo it). `say()` adds records at any time. `on_interrupt` + is what an interrupt makes the CLI say. + """ + + def __init__(self) -> None: + self.clients: list[FakeClient] = [] + self.on_query: OnQuery = lambda record: [echo(record)] + self.on_interrupt: list[Any] = [ + sdk.UserMessage( + content=[ + sdk.TextBlock( + text="[Request interrupted by user for tool use]" + ) + ], + uuid="cli-own", + ), + result(subtype="error_during_execution", is_error=True), + ] + + @property + def client(self) -> FakeClient: + return self.clients[-1] + + def say(self, *records: Any) -> None: + for record in records: + self.client.queue.put_nowait(record) + + +class FakeClient: + def __init__(self, cli: FakeCLI, options: Any) -> None: + self.cli = cli + self.options = options + self.queue: asyncio.Queue[Any] = asyncio.Queue() + self.written: list[dict[str, Any]] = [] + self.interrupts = 0 + self.connected = False + self.disconnects = 0 + cli.clients.append(self) + + async def connect(self) -> None: + self.connected = True + + async def query(self, prompt: AsyncIterable[dict[str, Any]]) -> None: + async for record in prompt: + self.written.append(record) + # Said at once, before query() returns: the external harness must + # still put the user message first. + for item in self.cli.on_query(record): + self.queue.put_nowait(item) + + async def receive_messages(self) -> AsyncIterator[Any]: + while True: + item = await self.queue.get() + if item is None: + return + if isinstance(item, BaseException): + raise item + yield item + + async def interrupt(self) -> None: + self.interrupts += 1 + for item in self.cli.on_interrupt: + self.queue.put_nowait(item) + + async def disconnect(self) -> None: + self.disconnects += 1 + self.queue.put_nowait(None) + + +# -- records, as the CLI sends them ------------------------------------------- + + +def echo(record: dict[str, Any]) -> Any: + """The `--replay-user-messages` echo of a written message.""" + return sdk.UserMessage( + content=record["message"]["content"], uuid=record["uuid"] + ) + + +def stream(msg_id: str, *events: dict[str, Any]) -> list[Any]: + return [ + sdk.StreamEvent(uuid="e", session_id=SESSION, event=event) + for event in ( + {"type": "message_start", "message": {"id": msg_id}}, + *events, + ) + ] + + +def block(index: int, kind: str, chunk: str) -> list[dict[str, Any]]: + """Start, delta and stop of one text or thinking block.""" + key = "text" if kind == "text" else "thinking" + return [ + { + "type": "content_block_start", + "index": index, + "content_block": {"type": kind, key: ""}, + }, + { + "type": "content_block_delta", + "index": index, + "delta": {"type": f"{key}_delta", key: chunk}, + }, + {"type": "content_block_stop", "index": index}, + ] + + +def tool_use( + msg_id: str, index: int, call_id: str, name: str, args: dict[str, Any] +) -> list[Any]: + """A tool call: its stream start, the whole message, its stream stop.""" + return [ + sdk.StreamEvent( + uuid="e", + session_id=SESSION, + event={ + "type": "content_block_start", + "index": index, + "content_block": { + "type": "tool_use", + "id": call_id, + "name": name, + }, + }, + ), + sdk.AssistantMessage( + content=[sdk.ToolUseBlock(id=call_id, name=name, input=args)], + model="claude", + message_id=msg_id, + ), + sdk.StreamEvent( + uuid="e", + session_id=SESSION, + event={"type": "content_block_stop", "index": index}, + ), + ] + + +def tool_result(call_id: str, content: Any, *, is_error: bool = False) -> Any: + return sdk.UserMessage( + content=[ + sdk.ToolResultBlock( + tool_use_id=call_id, content=content, is_error=is_error + ) + ], + uuid=f"r-{call_id}", + ) + + +def result(**fields: Any) -> Any: + values: dict[str, Any] = { + "subtype": "success", + "duration_ms": 1, + "duration_api_ms": 1, + "is_error": False, + "num_turns": 1, + "session_id": SESSION, + "usage": { + "input_tokens": 3, + "cache_read_input_tokens": 7, + "cache_creation_input_tokens": 5, + "output_tokens": 4, + "output_tokens_details": {"thinking_tokens": 2}, + }, + "total_cost_usd": 0.01, + **fields, + } + return sdk.ResultMessage(**values) + + +# -- fixtures ----------------------------------------------------------------- + + +@pytest.fixture +def cwd(tmp_path: Path) -> str: + path = tmp_path / "project" + path.mkdir() + return os.path.realpath(path) + + +@pytest.fixture +def cli(monkeypatch: pytest.MonkeyPatch) -> FakeCLI: + cli = FakeCLI() + + async def probe(kind: str, executable: str, **kwargs: Any) -> str: + return "2.1.319 (Claude Code)" + + monkeypatch.setattr(_process, "probe", probe) + monkeypatch.setattr( + sdk, "ClaudeSDKClient", lambda options: FakeClient(cli, options) + ) + return cli + + +async def run( + harness: ClaudeCode, messages: list[m.Message] +) -> tuple[ai.Stream[str], ai.events.StreamEnd]: + ends: list[ai.events.StreamEnd] = [] + async with asyncio.timeout(5): + async with ai.stream(harness, messages) as s: + async for event in s: + if isinstance(event, ai.events.StreamEnd): + ends.append(event) + (end,) = ends + return s, end + + +# -- tests -------------------------------------------------------------------- + + +async def test_a_turn_streams_in_the_model_view(cli: FakeCLI, cwd: str) -> None: + cli.on_query = lambda record: [ + echo(record), + *stream("m1", *block(0, "thinking", "plan")), + *tool_use("m1", 1, "toolu_1", "Read", {"file_path": "secret.txt"}), + tool_result("toolu_1", "the word is HERON"), + *stream("m2", *block(0, "text", "HERON")), + sdk.AssistantMessage( + content=[sdk.TextBlock(text="HERON")], + model="claude", + message_id="m2", + ), + result(), + ] + user = ai.user_message("go") + harness = ClaudeCode(cwd=cwd) + async with harness: + s, end = await run(harness, [user]) + message = s.message + assert message.reasoning == "plan" + assert message.text == "HERON" # the whole message does not repeat it + (call,) = message.builtin_tool_calls + assert call.tool_name == "Read" + assert json.loads(call.tool_args) == {"file_path": "secret.txt"} + (returned,) = message.builtin_tool_returns + assert returned.result == "the word is HERON" + assert message.usage is not None + assert message.usage.input_tokens == 10 + assert message.usage.output_tokens == 4 + assert message.usage.reasoning_tokens == 2 + assert message.usage.cache_write_tokens == 5 + assert end.finish_reason == "stop" + # The echo came at once, but acceptance still comes first. + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + ] + assert harness.session.messages[0].id == user.id + # The uuid the echo matched is the one written. + (written,) = cli.client.written + assert written["message"]["content"] == [{"type": "text", "text": "go"}] + assert cli.client.disconnects == 1 + + +async def test_start_connects_with_the_bridge_and_no_approvals( + cli: FakeCLI, cwd: str +) -> None: + harness = ClaudeCode( + cwd=cwd, + provider=external.ClaudeCodeProvider( + executable="/custom/claude", env={"ANTHROPIC_API_KEY": "k"} + ), + model_name="haiku", + instructions="Be brief.", + allowed_tools=["Read"], + disallowed_tools=["Bash"], + max_turns=3, + ) + async with harness: + await harness.session.start() + options = cli.client.options + assert harness.session._run is not None + bridge = harness.session._run.bridge + assert options.mcp_servers == { + "ai_sdk": { + "type": "http", + "url": bridge.url, + "headers": {"Authorization": "Bearer ${AI_PYTHON_MCP_TOKEN}"}, + } + } + assert options.permission_mode == "bypassPermissions" + assert options.can_use_tool is None + assert options.allowed_tools == ["mcp__ai_sdk"] + assert options.extra_args == {"replay-user-messages": None} + assert options.include_partial_messages + assert options.system_prompt == { + "type": "preset", + "preset": "claude_code", + "append": "Be brief.", + } + assert options.cwd == cwd + assert str(options.cli_path) == "/custom/claude" + assert options.env == { + "MCP_TOOL_TIMEOUT": "86400000", + "ANTHROPIC_API_KEY": "k", + "AI_PYTHON_MCP_TOKEN": bridge.token, + } + assert options.model == "haiku" + assert options.tools == ["Read"] + assert options.disallowed_tools == ["Bash"] + assert options.max_turns == 3 + # A new session: the id we chose is the CLI's. + assert options.resume is None + assert options.session_id == harness.session.id + + +@pytest.mark.parametrize("config_kind", ["inherited", "override", "relative"]) +async def test_a_resumed_session_must_exist_in_the_effective_config( + cli: FakeCLI, + cwd: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + config_kind: str, +) -> None: + ambient = tmp_path / "ambient" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(ambient)) + config = ambient if config_kind == "inherited" else Path(cwd) / "config" + env = {} + if config_kind != "inherited": + env["CLAUDE_CONFIG_DIR"] = ( + "config" if config_kind == "relative" else str(config) + ) + provider = external.ClaudeCodeProvider(env=env) + key = sdk.project_key_for_directory(cwd) + project = config / "projects" / key + project.mkdir(parents=True) + record = '{"type":"custom-title","customTitle":"Known session"}\n' + (project / f"{SESSION}.jsonl").write_text(record) + + harness = ClaudeCode(cwd=cwd, provider=provider, resume=SESSION) + async with harness: + await harness.session.start() + assert cli.client.options.resume == SESSION + assert cli.client.options.session_id is None + assert harness.session.id == SESSION + + ghost_id = "22222222-2222-2222-2222-222222222222" + if config != ambient: + # A transcript in the parent's configuration is not in the CLI's. + wrong_project = ambient / "projects" / key + wrong_project.mkdir(parents=True) + (wrong_project / f"{ghost_id}.jsonl").write_text(record) + assert sdk.get_session_info(ghost_id, cwd) is not None + ghost = ClaudeCode(cwd=cwd, provider=provider, resume=ghost_id) + with pytest.raises(external.ExternalHarnessError, match="no session"): + async with ghost: + await ghost.session.start() + assert len(cli.clients) == 1 + + +async def test_a_new_session_is_seeded_from_the_past( + cli: FakeCLI, cwd: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ambient = tmp_path / "ambient" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(ambient)) + config = tmp_path / "config" + cli.on_query = lambda record: [ + echo(record), + *stream("m1", *block(0, "text", "ok")), + result(), + ] + history = [ + ai.user_message("one"), + m.Message( + role="assistant", + parts=[ + m.ReasoningPart(text="secret thoughts"), + m.TextPart(text="done"), + m.BuiltinToolCallPart( + tool_call_id="t1", tool_name="Bash", tool_args='{"a": 1}' + ), + m.BuiltinToolReturnPart( + tool_call_id="t1", tool_name="Bash", result="out" + ), + ], + ), + ai.user_message("two"), + ] + harness = ClaudeCode( + cwd=cwd, + provider=external.ClaudeCodeProvider( + env={"CLAUDE_CONFIG_DIR": str(config)} + ), + ) + with pytest.warns(UserWarning, match="1 reasoning part"): + async with harness: + await run(harness, history) + options = cli.client.options + assert options.resume == harness.session.id + key = sdk.project_key_for_directory(cwd) + path = config / "projects" / key / f"{harness.session.id}.jsonl" + assert path.stat().st_mode & 0o777 == 0o600 + assert path.parent.stat().st_mode & 0o777 == 0o700 + records = [json.loads(line) for line in path.read_text().splitlines()] + assert [(r["type"], r["message"]["content"]) for r in records] == [ + ("user", [{"type": "text", "text": "one"}]), + ( + "assistant", + [ + {"type": "text", "text": "done"}, + { + "type": "tool_use", + "id": "t1", + "name": "Bash", + "input": {"a": 1}, + }, + ], + ), + ( + "user", + [{"type": "tool_result", "tool_use_id": "t1", "content": "out"}], + ), + ] + assert records[1]["parentUuid"] == records[0]["uuid"] + assert {r["sessionId"] for r in records} == {harness.session.id} + # Only the new message went to the CLI. + (written,) = cli.client.written + assert written["message"]["content"] == [{"type": "text", "text": "two"}] + + +async def test_bridge_calls_are_ai_tool_calls_not_builtin( + cli: FakeCLI, cwd: str +) -> None: + @ai.tool + async def lookup(word: str) -> str: + """Look a word up.""" + return word.upper() + + answers: list[dict[str, Any]] = [] + + async def call_bridge() -> None: + # The CLI calls the bridge over MCP, as Claude Code does. + options = cli.client.options + config = options.mcp_servers["ai_sdk"] + reply = await fakes.mcp_call( + config["url"], + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "lookup", "arguments": {"word": "heron"}}, + }, + { + "Authorization": config["headers"]["Authorization"].replace( + "${AI_PYTHON_MCP_TOKEN}", options.env["AI_PYTHON_MCP_TOKEN"] + ) + }, + ) + answers.append(reply["result"]) + text = reply["result"]["content"][0]["text"] + cli.say( + tool_result("toolu_b", text), + *stream("m2", *block(0, "text", f"got {text}")), + result(), + ) + + tasks: list[asyncio.Task[None]] = [] + + def on_query(record: dict[str, Any]) -> list[Any]: + tasks.append(asyncio.ensure_future(call_bridge())) + return [ + echo(record), + *stream("m1"), + *tool_use( + "m1", 0, "toolu_b", "mcp__ai_sdk__lookup", {"word": "heron"} + ), + ] + + cli.on_query = on_query + harness = ClaudeCode(cwd=cwd) + async with harness: + async with asyncio.timeout(5): + async with ai.Agent(tools=[lookup]).run( + harness, [ai.user_message("go")] + ) as agent_run: + async for _ in agent_run: + pass + await asyncio.gather(*tasks) + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + "tool", + "assistant", + ] + (call,) = harness.session.messages[1].tool_calls + assert call.tool_name == "lookup" + assert all(not msg.builtin_tool_calls for msg in harness.session.messages) + assert all(not msg.builtin_tool_returns for msg in harness.session.messages) + assert harness.session.messages[3].text == "got HERON" + assert answers[0]["content"][0]["text"] == "HERON" + + +async def test_a_send_during_a_turn_steers_it(cli: FakeCLI, cwd: str) -> None: + first = [True] + + def on_query(record: dict[str, Any]) -> list[Any]: + if first: + first.clear() + return [ + echo(record), + *stream("m1", *block(0, "text", "Look")), + *tool_use("m1", 1, "toolu_1", "Bash", {"command": "sleep 4"}), + ] + return [] # the CLI holds a steer until the next step + + cli.on_query = on_query + harness = ClaudeCode(cwd=cwd) + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("fix")) + await harness.session.send(ai.user_message("skip slow ones")) + steer = cli.client.written[1] + # The step ends, then the CLI echoes the steer where it injects it. + cli.say( + tool_result("toolu_1", ""), + echo(steer), + *stream("m2", *block(0, "text", "OK")), + result(), + ) + async with asyncio.timeout(5): + while len(harness.session.messages) < 4: + await anext(feed) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "fix"), + ("assistant", "Look"), + ("user", "skip slow ones"), + ("assistant", "OK"), + ] + assert len(harness.session.messages[1].builtin_tool_returns) == 1 + + +async def test_interrupt_ends_the_turn_as_interrupted( + cli: FakeCLI, cwd: str +) -> None: + cli.on_query = lambda record: [ + echo(record), + *stream( + "m1", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "work"}, + }, + ), + ] + harness = ClaudeCode(cwd=cwd) + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("long job")) + async with asyncio.timeout(5): + async for event in feed: + if isinstance(event, ai.events.TextDelta): + break + await harness.session.interrupt() + assert cli.client.interrupts == 1 + assert (harness.session.messages[-1].provider_metadata or {}).get( + "external" + ) == {"stop_reason": "interrupted"} + assert harness.session.messages[-1].text == "work" + # The next turn is a normal one: the interrupt was used up. + cli.on_query = lambda record: [ + echo(record), + *stream("m2", *block(0, "text", "OK")), + result(), + ] + s, end = await run( + harness, [*harness.session.messages, ai.user_message("again")] + ) + assert s.text == "OK" + assert end.finish_reason == "stop" + + +async def test_a_failed_turn_without_a_key_is_not_authenticated( + cli: FakeCLI, cwd: str +) -> None: + detail = "Invalid API key · Fix external API key" + cli.on_query = lambda record: [ + echo(record), + # Measured: not streamed, from model "". + sdk.AssistantMessage( + content=[sdk.TextBlock(text=detail)], + model="", + error="authentication_failed", + ), + result(is_error=True, result=detail), + ] + harness = ClaudeCode(cwd=cwd) + async with harness: + with pytest.raises(external.NotAuthenticatedError): + await run(harness, [ai.user_message("go")]) + assert harness.session.messages[-1].text == detail + + +async def test_max_turns_ends_with_length(cli: FakeCLI, cwd: str) -> None: + cli.on_query = lambda record: [ + echo(record), + result(subtype="error_max_turns", is_error=True), + ] + harness = ClaudeCode(cwd=cwd, max_turns=1) + async with harness: + _, end = await run(harness, [ai.user_message("go")]) + assert end.finish_reason == "length" + + +async def test_a_crash_reaches_the_stream(cli: FakeCLI, cwd: str) -> None: + cli.on_query = lambda record: [ + echo(record), + sdk.ProcessError("boom", exit_code=1), + ] + harness = ClaudeCode(cwd=cwd) + async with harness: + with pytest.raises(external.ExternalHarnessError, match="crashed"): + await run(harness, [ai.user_message("go")]) + + +async def test_unsupported_parts_are_refused_before_writing( + cli: FakeCLI, cwd: str +) -> None: + harness = ClaudeCode(cwd=cwd) + async with harness: + with pytest.raises(external.UnsupportedError): + await harness.session.send( + ai.user_message( + m.FilePart(data=b"x", media_type="audio/wav"), + ) + ) + assert cli.client.written == [] + + +async def test_images_go_as_base64_blocks(cli: FakeCLI, cwd: str) -> None: + cli.on_query = lambda record: [echo(record), result()] + harness = ClaudeCode(cwd=cwd) + async with harness: + await run( + harness, + [ + ai.user_message( + "what is this", + m.FilePart(data=b"\x89PNG", media_type="image/png"), + ) + ], + ) + (written,) = cli.client.written + assert written["message"]["content"][1] == { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw==", + }, + } + + +async def test_close_is_idempotent(cli: FakeCLI, cwd: str) -> None: + async with ClaudeCode(cwd=cwd) as harness: + await harness.session.start() + assert harness.session._run is not None + wire = harness.session._run.wire + await wire.close() # the block closed it: a second close is safe + assert cli.client.disconnects == 1 + + +async def test_a_copy_keeps_the_settings(cli: FakeCLI, cwd: str) -> None: + harness = ClaudeCode(cwd=cwd, max_turns=2) + again = copy.copy(harness) + assert isinstance(again, ClaudeCode) + assert again.max_turns == 2 + assert again.harness_class_id == "claude-code" diff --git a/tests/external/codex/__init__.py b/tests/external/codex/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/external/codex/test_wire.py b/tests/external/codex/test_wire.py new file mode 100644 index 00000000..795f2457 --- /dev/null +++ b/tests/external/codex/test_wire.py @@ -0,0 +1,498 @@ +"""The Codex external harness against a scripted `codex app-server`, offline.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from typing import Any, cast + +import pytest + +import ai +from ai import external +from ai.external import _jsonrpc as jsonrpc +from ai.external import _process +from ai.types import messages as m + +from .. import fakes + +THREAD = "T" + + +class AppServer: + """Answers like `codex app-server`. + + `turns[i]` plays out after the i-th turn/start, BEFORE the response, as + the real server can: the external harness must still order acceptance first. + """ + + def __init__(self, cwd: str) -> None: + self.process = fakes.FakeProcess(self.answer) + self.cwd = cwd + self.turns: list[list[dict[str, Any]]] = [] + self.started = 0 + self.steer_error: dict[str, Any] | None = None + self.threads: dict[str, dict[str, Any]] = { + THREAD: {"id": THREAD, "cwd": cwd}, + "elsewhere": {"id": "elsewhere", "cwd": "/other"}, + } + self.on_request_answer: list[dict[str, Any]] = [] + self.spawned: list[tuple[list[str], dict[str, Any]]] = [] + + def says(self, method: str, **params: Any) -> None: + self.process.server_says({"method": method, "params": params}) + + def answer(self, message: dict[str, Any]) -> Any: + method = message.get("method") + params = message.get("params") or {} + if method == "turn/start": + self.started += 1 + turn = f"turn-{self.started}" + for record in self.turns[self.started - 1] if self.turns else []: + self.process.server_says(record) + return {"turn": {"id": turn}} + if method == "turn/steer": + if self.steer_error is not None: + self.process.server_says( + {"id": message["id"], "error": self.steer_error} + ) + return None + return {} + if method == "turn/interrupt": + self.says( + "turn/completed", + threadId=THREAD, + turn={"id": params["turnId"], "status": "interrupted"}, + ) + return {} + if method == "thread/read": + return {"thread": self.threads.get(str(params.get("threadId")), {})} + if method == "thread/start": + return {"thread": {"id": THREAD}} + if method == "thread/resume": + return {"thread": {"id": params["threadId"]}} + if "method" not in message: # our answer to a server request + self.on_request_answer.append(message) + return None + if "id" in message: + return {} + return None + + def sent(self, method: str) -> list[dict[str, Any]]: + return [w for w in self.process.written if w.get("method") == method] + + +def note(method: str, turn_id: str = "turn-1", **params: Any) -> dict[str, Any]: + return { + "method": method, + "params": {"threadId": THREAD, "turnId": turn_id, **params}, + } + + +COMMAND = { + "type": "commandExecution", + "id": "c1", + "command": "/bin/zsh -lc 'cat secret.txt'", + "aggregatedOutput": "the word is HERON\n", + "exitCode": 0, + "status": "completed", +} + +TURN = [ + note("item/reasoning/summaryTextDelta", itemId="r1", delta="plan"), + note("item/completed", item={"type": "reasoning", "id": "r1"}), + note("item/started", item={**COMMAND, "aggregatedOutput": None}), + note("item/completed", item=COMMAND), + note("item/agentMessage/delta", itemId="a1", delta="HER"), + note("item/agentMessage/delta", itemId="a1", delta="ON"), + note("item/completed", item={"type": "agentMessage", "id": "a1"}), + note( + "thread/tokenUsage/updated", + tokenUsage={"lastTurn": {"inputTokens": 10, "outputTokens": 4}}, + ), + note("turn/completed", turn={"id": "turn-1", "status": "completed"}), +] + + +@pytest.fixture +def cwd(tmp_path: Path) -> str: + return os.path.realpath(tmp_path) + + +@pytest.fixture +def server(cwd: str, monkeypatch: pytest.MonkeyPatch) -> AppServer: + server = AppServer(cwd) + + async def probe(kind: str, executable: str, **kwargs: Any) -> str: + return "codex 0.160.1" + + async def spawn(argv: list[str], **kwargs: Any) -> _process.Process: + server.spawned.append((argv, kwargs)) + if server.process.terminated: + server.process = fakes.FakeProcess(server.process.answer) + return cast("_process.Process", server.process) + + monkeypatch.setattr(_process, "probe", probe) + monkeypatch.setattr(_process, "spawn", spawn) + return server + + +async def run( + harness: external.Codex, messages: list[m.Message] +) -> ai.Stream[str]: + async with asyncio.timeout(5): + async with ai.stream(harness, messages) as s: + async for _ in s: + pass + return s + + +async def test_a_turn_streams_in_the_model_view( + server: AppServer, cwd: str +) -> None: + server.turns = [TURN] + user = ai.user_message("go") + harness = external.Codex(cwd=cwd) + async with harness: + s = await run(harness, [user]) + message = s.message + assert message.reasoning == "plan" + assert message.text == "HERON" + (call,) = message.builtin_tool_calls + assert call.tool_name == "commandExecution" + assert "secret.txt" in call.tool_args + (result,) = message.builtin_tool_returns + assert result.result == "the word is HERON\n" + assert message.usage is not None and message.usage.input_tokens == 10 + # Acceptance came first, although the notes came before the response. + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + ] + assert harness.session.messages[0].id == user.id + assert harness.session.id == THREAD + + +async def test_start_runs_app_server_with_the_bridge( + server: AppServer, cwd: str +) -> None: + harness = external.Codex( + cwd=cwd, + provider=external.CodexProvider( + executable="/custom/codex", env={"CODEX_HOME": "/h"} + ), + model_name="gpt-test", + instructions="Be brief.", + sandbox="workspace-write", + config={"model_reasoning_effort": "high"}, + ) + async with harness: + await harness.session.start() + (argv, kwargs) = server.spawned[0] + assert argv[:2] == ["/custom/codex", "app-server"] + assert harness.session._run is not None + bridge = harness.session._run.bridge + assert f'mcp_servers.ai_sdk.url="{bridge.url}"' in argv + assert ( + 'mcp_servers.ai_sdk.bearer_token_env_var="AI_PYTHON_MCP_TOKEN"' + in argv + ) + assert all(bridge.token not in arg for arg in argv) + assert "mcp_servers.ai_sdk.tool_timeout_sec=86400" in argv + assert 'model_reasoning_effort="high"' in argv + assert kwargs == { + "cwd": cwd, + "env": {"CODEX_HOME": "/h", "AI_PYTHON_MCP_TOKEN": bridge.token}, + } + (start,) = server.sent("thread/start") + assert start["params"] == { + "cwd": cwd, + "sandbox": "workspace-write", + "approvalPolicy": "never", + "model": "gpt-test", + "developerInstructions": "Be brief.", + } + assert server.process.terminated + + +async def test_a_new_thread_is_seeded_with_the_past( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [note("turn/completed", turn={"id": "turn-1", "status": "completed"})] + ] + history = [ + ai.user_message("one"), + ai.assistant_message("done"), + ai.user_message("two"), + ] + harness = external.Codex(cwd=cwd) + async with harness: # the first call starts the CLI, seeded + await run(harness, history) + (inject,) = server.sent("thread/inject_items") + texts = [item["content"][0]["text"] for item in inject["params"]["items"]] + assert texts == ["one", "done"] + (start,) = server.sent("turn/start") + assert start["params"]["input"] == [{"type": "text", "text": "two"}] + + +async def test_a_session_resumes_here_only(server: AppServer, cwd: str) -> None: + async with external.Codex(cwd=cwd, resume=THREAD) as harness: + await harness.session.start() + assert server.sent("thread/resume")[0]["params"]["threadId"] == THREAD + elsewhere = external.Codex(cwd=cwd, resume="elsewhere") + with pytest.raises( + external.ExternalHarnessError, match="belongs to /other" + ): + async with elsewhere: + await elsewhere.session.start() + + +async def test_another_turns_notifications_are_not_this_turns( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [ + note( + "turn/completed", + turn_id="old", + turn={"id": "old", "status": "completed"}, + ), + note("item/agentMessage/delta", itemId="a1", delta="mine"), + note("item/completed", item={"type": "agentMessage", "id": "a1"}), + note( + "turn/completed", turn={"id": "turn-1", "status": "completed"} + ), + ] + ] + harness = external.Codex(cwd=cwd) + async with harness: + s = await run(harness, [ai.user_message("go")]) + assert s.text == "mine" + + +async def test_a_message_that_was_never_streamed_still_arrives( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [ + note( + "item/completed", + item={"type": "agentMessage", "id": "a1", "text": "whole"}, + ), + note( + "turn/completed", turn={"id": "turn-1", "status": "completed"} + ), + ] + ] + harness = external.Codex(cwd=cwd) + async with harness: + s = await run(harness, [ai.user_message("go")]) + assert s.text == "whole" + + +async def test_a_failed_turn_without_a_key_is_not_authenticated( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [ + note( + "turn/completed", + turn={ + "id": "turn-1", + "status": "failed", + "error": {"message": "401 Unauthorized"}, + }, + ) + ] + ] + harness = external.Codex(cwd=cwd) + async with harness: + with pytest.raises(external.NotAuthenticatedError): + await run(harness, [ai.user_message("go")]) + + +async def test_interrupt_ends_the_turn_as_interrupted( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [note("item/agentMessage/delta", itemId="a1", delta="work")] + ] + harness = external.Codex(cwd=cwd) + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("long job")) + async with asyncio.timeout(5): + async for event in feed: + if isinstance(event, ai.events.TextDelta): + break + await harness.session.interrupt() + (interrupt,) = server.sent("turn/interrupt") + assert interrupt["params"] == {"threadId": THREAD, "turnId": "turn-1"} + assert (harness.session.messages[-1].provider_metadata or {}).get( + "external" + ) == {"stop_reason": "interrupted"} + assert harness.session.messages[-1].text == "work" + + +async def test_a_send_during_a_turn_steers_it( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [note("item/agentMessage/delta", itemId="a1", delta="Look")] + ] + harness = external.Codex(cwd=cwd) + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("fix")) + await harness.session.send(ai.user_message("skip slow ones")) + (steer,) = server.sent("turn/steer") + assert steer["params"]["expectedTurnId"] == "turn-1" + assert steer["params"]["input"] == [ + {"type": "text", "text": "skip slow ones"} + ] + server.process.server_says( + note("item/completed", item={"type": "agentMessage", "id": "a1"}) + ) + server.process.server_says( + note("item/agentMessage/delta", itemId="a2", delta="OK") + ) + server.process.server_says( + note("item/completed", item={"type": "agentMessage", "id": "a2"}) + ) + server.process.server_says( + note("turn/completed", turn={"id": "turn-1", "status": "completed"}) + ) + async with asyncio.timeout(5): + while len(harness.session.messages) < 4: + await anext(feed) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "fix"), + ("assistant", "Look"), + ("user", "skip slow ones"), + ("assistant", "OK"), + ] + + +async def test_a_steer_that_lost_the_race_starts_a_turn( + server: AppServer, cwd: str +) -> None: + server.turns = [ + [], + [ + note( + "turn/completed", + turn_id="turn-2", + turn={"id": "turn-2", "status": "completed"}, + ) + ], + ] + server.steer_error = {"code": -32600, "message": "no active turn to steer"} + harness = external.Codex(cwd=cwd) + async with harness: + await harness.session.send(ai.user_message("one")) + await harness.session.send(ai.user_message("two")) + assert len(server.sent("turn/start")) == 2 + + +async def test_other_steer_rejections_are_errors( + server: AppServer, cwd: str +) -> None: + server.turns = [[]] + server.steer_error = {"code": -32600, "message": "expected turn mismatch"} + harness = external.Codex(cwd=cwd) + async with harness: + await harness.session.send(ai.user_message("one")) + with pytest.raises( + external.ExternalHarnessError, match="steer rejected" + ): + await harness.session.send(ai.user_message("two")) + assert len(server.sent("turn/start")) == 1 + + +async def test_bridge_calls_are_ai_tool_calls_not_builtin( + server: AppServer, cwd: str +) -> None: + bridge_item = { + "type": "mcpToolCall", + "id": "m1", + "server": "ai_sdk", + "tool": "lookup", + } + server.turns = [ + [ + note("item/started", item=bridge_item), + note("item/completed", item={**bridge_item, "result": "x"}), + note("item/agentMessage/delta", itemId="a1", delta="done"), + note("item/completed", item={"type": "agentMessage", "id": "a1"}), + note( + "turn/completed", turn={"id": "turn-1", "status": "completed"} + ), + ] + ] + harness = external.Codex(cwd=cwd) + async with harness: + s = await run(harness, [ai.user_message("go")]) + assert s.message.builtin_tool_calls == [] + assert s.text == "done" + + +async def test_an_approval_request_is_answered( + server: AppServer, cwd: str +) -> None: + async with external.Codex(cwd=cwd) as harness: + await harness.session.start() + server.process.server_says( + { + "id": 99, + "method": "item/commandExecution/requestApproval", + "params": {}, + } + ) + async with asyncio.timeout(5): + while not server.on_request_answer: + await asyncio.sleep(0.01) + assert server.on_request_answer[0] == { + "jsonrpc": "2.0", + "id": 99, + "result": {"decision": "accept"}, + } + + +async def test_a_failed_handshake_closes_the_process( + server: AppServer, cwd: str +) -> None: + def refuse(message: dict[str, Any]) -> Any: + if message.get("method") == "initialize": + server.process.server_says( + {"id": message["id"], "error": {"code": 1, "message": "no"}} + ) + return None + return {} + + server.process.answer = refuse + with pytest.raises(jsonrpc.JsonRpcError): + async with external.Codex(cwd=cwd) as harness: + await harness.session.start() + assert server.process.terminated + + +async def test_a_crash_reaches_the_stream(server: AppServer, cwd: str) -> None: + server.turns = [ + [note("item/agentMessage/delta", itemId="a1", delta="half")] + ] + + async def die() -> None: + await asyncio.sleep(0.05) + await server.process.terminate() + + harness = external.Codex(cwd=cwd) + + async with harness: + task = asyncio.create_task(die()) + with pytest.raises( + external.ExternalHarnessError, match=r"crashed|stopped" + ): + await run(harness, [ai.user_message("go")]) + await task diff --git a/tests/external/fakes.py b/tests/external/fakes.py new file mode 100644 index 00000000..1d3b6719 --- /dev/null +++ b/tests/external/fakes.py @@ -0,0 +1,252 @@ +"""Test doubles for external harnesses: a scripted wire and a fake process.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import urllib.parse +from collections.abc import AsyncIterator, Callable, Iterable +from typing import Any, ClassVar + +from ai import external +from ai.external import _utils, _wire +from ai.types import events +from ai.types import messages as m + + +@dataclasses.dataclass +class CallTool: + """The CLI calls an `ai` tool through the MCP bridge, over HTTP. + + `then(text, is_error)` gives what the CLI says after the result. + """ + + name: str + arguments: dict[str, Any] = dataclasses.field(default_factory=dict) + then: Callable[[str, bool], Iterable[_wire.Item]] = lambda text_, err: [ + *text(f"got {text_}", block=f"after-{text_}"), + _wire.TurnEnd(), + ] + + +Step = _wire.Item | CallTool +Reply = Callable[[m.Message], Iterable[Step]] + + +def text(chunk: str, block: str = "t") -> list[_wire.Item]: + return [ + events.TextStart(block_id=block), + events.TextDelta(chunk=chunk, block_id=block), + events.TextEnd(block_id=block), + ] + + +def builtin( + call_id: str, name: str, args: str = "{}", result: Any = "ok" +) -> list[_wire.Item]: + """One of the CLI's own tool calls, from start to result.""" + return [ + *_utils.get_builtin_call_events( + call_id, name, args, started_call_ids=set() + ), + _utils.get_builtin_result_event(call_id, name, result, is_error=False), + ] + + +def echo(message: m.Message) -> list[_wire.Item]: + """Accept the message, answer with its text, end the turn.""" + return [ + _wire.Accepted(message), + *text(message.text, block=f"t-{message.id}"), + _wire.TurnEnd(), + ] + + +class FakeWire: + """A CLI that says what the test tells it to. + + `reply(message)` gives the items that a sent message causes (default: + accept it, nothing more). `push()` adds items at any time. + """ + + def __init__(self, harness: FakeHarness) -> None: + self.harness = harness + self.reply: Reply = harness._reply or ( + lambda msg: [_wire.Accepted(msg)] + ) + self.queue: asyncio.Queue[Any] = asyncio.Queue() + self.started: dict[str, Any] | None = None + self.sent: list[m.Message] = [] + self.interrupts = 0 + self.calls: list[asyncio.Task[tuple[str, bool]]] = [] + self.closed = False + # What `interrupt()` does. None: the CLI ignores interrupts. + self.on_interrupt: list[_wire.Item] | None = [ + _wire.TurnEnd(reason="interrupted") + ] + harness._wires.append(self) + + def push(self, *items: Step | BaseException) -> None: + for item in items: + self.queue.put_nowait(item) + + async def start( + self, + *, + session_id: str | None, + bridge_url: str, + bridge_token: str, + instructions: str | None, + seed: list[m.Message], + ) -> str: + self.started = { + "bridge_url": bridge_url, + "bridge_token": bridge_token, + "instructions": instructions, + "seed": seed, + "session_id": session_id, + } + return session_id or "s1" + + async def send(self, message: m.Message) -> None: + self.sent.append(message) + self.push(*self.reply(message)) + + async def events(self) -> AsyncIterator[_wire.Item]: + while True: + item = await self.queue.get() + if isinstance(item, BaseException): + raise item + if isinstance(item, CallTool): + task = asyncio.create_task(self._call(item)) + self.calls.append(task) + continue + yield item + + async def _call(self, call: CallTool) -> tuple[str, bool]: + assert self.started is not None + reply = await mcp_call( + self.started["bridge_url"], + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": call.name, "arguments": call.arguments}, + }, + {"Authorization": f"Bearer {self.started['bridge_token']}"}, + ) + result = reply["result"] + answer = (result["content"][0]["text"], bool(result["isError"])) + self.push(*call.then(*answer)) + return answer + + async def interrupt(self) -> None: + self.interrupts += 1 + if self.on_interrupt is not None: + self.push(*self.on_interrupt) + + async def close(self) -> None: + self.closed = True + + +class FakeProvider(external.ExternalHarnessProvider): + name: str = "fake" + + def _make_wire(self, harness: FakeHarness) -> FakeWire: + return FakeWire(harness) + + +@dataclasses.dataclass(kw_only=True, eq=False) +class FakeHarness(external.ExternalHarness): + harness_class_id: ClassVar[str] = "fake" + + provider: FakeProvider = dataclasses.field(default_factory=FakeProvider) + + # Test controls, not settings. + _reply: Reply | None = dataclasses.field( + default=None, init=False, repr=False + ) + _wires: list[FakeWire] = dataclasses.field( + default_factory=list, init=False, repr=False + ) + + @property + def wire(self) -> FakeWire: + return self._wires[-1] + + +def fake(reply: Reply | None = None, **fields: Any) -> FakeHarness: + harness = FakeHarness(**fields) + harness._reply = reply + return harness + + +class FakeProcess: + """In-memory duplex stdio, for the JSON-RPC peer and the wires. + + `answer(message)` is called for every message the client writes. For a + request, a non-None return is sent back as its result. `server_says()` + pushes a line from the "server": a notification or a request. + """ + + def __init__( + self, answer: Callable[[dict[str, Any]], Any] | None = None + ) -> None: + self._lines: asyncio.Queue[str] = asyncio.Queue() + self.answer = answer or (lambda message: {}) + self.written: list[dict[str, Any]] = [] + self.terminated = False + + def server_says(self, message: dict[str, Any]) -> None: + self._lines.put_nowait(json.dumps(message) + "\n") + + async def write(self, data: str) -> None: + message = json.loads(data) + self.written.append(message) + result = self.answer(message) + if "method" in message and "id" in message and result is not None: + self.server_says({"id": message["id"], "result": result}) + + async def readline(self) -> str: + return await self._lines.get() + + async def read_stderr(self) -> str: + return "" + + async def wait(self) -> int: + return 0 + + async def terminate(self) -> None: + self.terminated = True + self._lines.put_nowait("") + + async def death_note(self) -> str: + return "exit code 0; no stderr" + + +async def mcp_call( + url: str, payload: dict[str, Any], headers: dict[str, str] +) -> dict[str, Any]: + """POST one JSON-RPC message to the bridge; return the JSON reply.""" + parts = urllib.parse.urlsplit(url) + reader, writer = await asyncio.open_connection(parts.hostname, parts.port) + body = json.dumps(payload).encode() + extra = "".join(f"{key}: {value}\r\n" for key, value in headers.items()) + writer.write( + ( + f"POST {parts.path} HTTP/1.1\r\nHost: {parts.netloc}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\n" + f"{extra}\r\n" + ).encode() + + body + ) + await writer.drain() + head = await reader.readuntil(b"\r\n\r\n") + length = 0 + for line in head.decode().split("\r\n"): + if line.lower().startswith("content-length:"): + length = int(line.split(":", 1)[1]) + data = await reader.readexactly(length) + writer.close() + return json.loads(data) if data else {} diff --git a/tests/external/fx/__init__.py b/tests/external/fx/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/external/fx/test_wire.py b/tests/external/fx/test_wire.py new file mode 100644 index 00000000..75134259 --- /dev/null +++ b/tests/external/fx/test_wire.py @@ -0,0 +1,1992 @@ +"""The Fx external harness against a scripted `fx acp` 0.0.17, offline. + +Native fields and event order follow recorded ACP sessions. Prompt replies +are delayed until the turn ends. History replays before the load reply. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any, cast + +import pytest + +import ai +from ai import external +from ai.external import _process +from ai.external.fx import wire as wire_ +from ai.types import messages as m + +from .. import fakes + +SESSION = "HLBbbJxgqBxq" +PREFS = { + "provider": "gateway", + "model": "openai/gpt-test", + "effort": "auto", + "fast_mode": False, +} + + +def update(update_kind: str, **fields: Any) -> dict[str, Any]: + return { + "method": "session/update", + "params": { + "sessionId": SESSION, + "update": {"sessionUpdate": update_kind, **fields}, + }, + } + + +def text(value: str, message_id: str = "a1") -> dict[str, Any]: + return update( + "agent_message_chunk", + messageId=message_id, + content={"type": "text", "text": value}, + ) + + +CALL = update( + "tool_call", + toolCallId="c1", + name="shell", + title="Running", + kind="execute", + status="pending", + rawInput={"action": "run", "command": "cat secret.txt"}, + _meta={"fx": {"toolCall": {"internal": False}}}, +) +RESULT = update( + "tool_call_update", + toolCallId="c1", + status="completed", + content=[ + {"type": "content", "content": {"type": "text", "text": "HERON\n"}} + ], +) +END = {"stopReason": "end_turn"} + + +class YieldingProcess(fakes.FakeProcess): + """Let the peer read early records while a prompt write drains.""" + + yield_prompt_write = False + fail_next_prompt = False + + async def write(self, data: str) -> None: + if ( + self.fail_next_prompt + and json.loads(data).get("method") == "session/prompt" + ): + self.fail_next_prompt = False + raise OSError("scripted write failed") + await super().write(data) + if ( + self.yield_prompt_write + and json.loads(data).get("method") == "session/prompt" + ): + await asyncio.sleep(0) + await asyncio.sleep(0) + + +class AcpServer: + def __init__(self, cwd: str, home: Path) -> None: + self.cwd = cwd + self.home = home + self.process = YieldingProcess(self.answer) + self.spawned: list[tuple[list[str], dict[str, Any]]] = [] + self.turns: list[ + tuple[list[dict[str, Any]], dict[str, Any] | None] + ] = [] + self.prompts: list[dict[str, Any]] = [] + self.steers: list[dict[str, Any]] = [] + self.initialize_error: dict[str, Any] | None = None + self.load_error = False + self.hold_load = False + self.closed_folders: list[tuple[str, bool]] = [] + self.active_requests: set[int] = set() + self.rejected_prompts: list[dict[str, Any]] = [] + self.cancel_updates: list[dict[str, Any]] = [] + self.replay_users: int | None = None + self.loaded: Path | None = None + self.loaded_records: list[dict[str, Any]] = [] + self.pages: list[list[str]] = [[SESSION]] + self.replies: list[dict[str, Any]] = [] + self.cancel_result = END + self.active_session = SESSION + self.listed_cwd: str | None = None + self.cancel_request: dict[str, Any] | None = None + + def says(self, *records: dict[str, Any]) -> None: + for record in records: + if "id" in record and ("result" in record or "error" in record): + self.active_requests.discard(record["id"]) + if record.get("method") == "session/update": + record = { + **record, + "params": { + **record["params"], + "sessionId": self.active_session, + }, + } + self.process.server_says(record) + + def finish(self, request: dict[str, Any], result: dict[str, Any]) -> None: + self.says({"id": request["id"], "result": result}) + + def sent(self, method: str) -> list[dict[str, Any]]: + return [w for w in self.process.written if w.get("method") == method] + + async def wait_for(self, method: str, count: int = 1) -> None: + async with asyncio.timeout(5): + while len(self.sent(method)) < count: + await asyncio.sleep(0) + + def answer(self, request: dict[str, Any]) -> Any: + method = request.get("method") + params = request.get("params", {}) + options = [ + {"id": key, "type": "select", "currentValue": value} + for key, value in ( + ("provider", PREFS["provider"]), + ("model", PREFS["model"]), + ("effort", PREFS["effort"]), + ("fast", "false"), + ) + ] + if method == "initialize": + if self.initialize_error: + self.says({"id": request["id"], "error": self.initialize_error}) + return None + return { + "protocolVersion": 1, + "agentInfo": {"name": "fx", "version": "0.0.17"}, + "agentCapabilities": { + "loadSession": True, + "sessionCapabilities": { + "list": {}, + "resume": {}, + "close": {}, + "systemPrompt": {}, + }, + "_meta": {"fx": {"steering": True}}, + }, + "authMethods": [], + } + if method == "session/new": + return {"sessionId": SESSION, "configOptions": options} + if method == "session/list": + assert params["cwd"] == self.cwd + page = int(params.get("cursor", "0")) + result: dict[str, Any] = { + "sessions": [ + {"sessionId": sid, "cwd": self.listed_cwd or self.cwd} + for sid in self.pages[page] + ] + } + if page + 1 < len(self.pages): + result["nextCursor"] = str(page + 1) + return result + if method == "session/load": + self.active_session = params["sessionId"] + self.loaded = self.home / "sessions/v2" / params["sessionId"] + self.loaded_records = [ + json.loads(line) + for line in (self.loaded / "log.jsonl").read_text().splitlines() + ] + if self.hold_load: + return None + if self.load_error: + self.says( + { + "id": request["id"], + "error": { + "code": -32603, + "message": "Session could not be loaded", + }, + } + ) + return None + users = 0 + for index, record in enumerate(self.loaded_records): + kind = record.get("type") + if kind not in ("user", "assistant"): + continue + if kind == "user": + users += 1 + if ( + self.replay_users is not None + and users > self.replay_users + ): + continue + native = update( + f"{'user' if kind == 'user' else 'agent'}_message_chunk", + messageId=f"replay-{index}", + content={"type": "text", "text": record["data"]["text"]}, + ) + native["params"]["sessionId"] = params["sessionId"] + self.says(native) + # An image is a second chunk of the SAME user message. + for image in record["data"].get("images", []): + self.says( + { + **native, + "params": { + **native["params"], + "update": { + **native["params"]["update"], + "content": { + "type": "image", + "mimeType": image["media_type"], + "data": base64.b64encode( + ( + self.loaded + / "blobs" + / image["blob"] + ).read_bytes() + ).decode(), + }, + }, + }, + } + ) + return {"configOptions": options} + if method == "session/resume": + return {"configOptions": options} + if method == "session/prompt": + if params.get("_meta", {}).get("fx", {}).get("steer"): + self.active_requests.add(request["id"]) + self.steers.append(request) + return None + if self.active_requests: + self.rejected_prompts.append(request) + self.says( + { + "id": request["id"], + "error": { + "code": -32600, + "message": "Prompt already in progress", + }, + } + ) + return None + self.active_requests.add(request["id"]) + index = len(self.prompts) + self.prompts.append(request) + records, response = ( + self.turns[index] if index < len(self.turns) else ([], END) + ) + self.says(*records) + if response is not None: + self.finish(request, response) + return None + if method == "session/close": + sid = params["sessionId"] + folder = self.home / "sessions/v2" / sid + self.closed_folders.append((sid, folder.exists())) + return {} + if method == "session/cancel": + if self.prompts: + self.finish( + self.cancel_request or self.prompts[-1], self.cancel_result + ) + self.says(*self.cancel_updates) + return None + if method is None: + self.replies.append(request) + return None + return {} if "id" in request else None + + +@pytest.fixture +def cwd(tmp_path: Path) -> str: + project = tmp_path / "project" + project.mkdir() + return os.path.realpath(project) + + +@pytest.fixture +def server( + cwd: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> AcpServer: + server = AcpServer(cwd, tmp_path / "fx-home") + + async def probe(kind: str, executable: str, **kwargs: Any) -> str: + return "0.0.17" + + async def spawn(argv: list[str], **kwargs: Any) -> _process.Process: + server.spawned.append((argv, kwargs)) + return cast("_process.Process", server.process) + + monkeypatch.setattr(_process, "probe", probe) + monkeypatch.setattr(_process, "spawn", spawn) + return server + + +@pytest.fixture +def harness(server: AcpServer) -> external.Fx: + return external.Fx( + cwd=server.cwd, + provider=external.FxProvider(env={"FX_HOME": str(server.home)}), + ) + + +async def run( + harness: external.Fx, messages: list[m.Message] +) -> ai.Stream[str]: + async with asyncio.timeout(5), ai.stream(harness, messages) as stream: + async for _ in stream: + pass + return stream + + +async def test_a_turn_streams_in_the_model_view( + server: AcpServer, harness: external.Fx +) -> None: + server.turns = [ + ( + [ + update( + "agent_thought_chunk", + content={"type": "text", "text": "plan"}, + ), + CALL, + update( + "tool_call_update", toolCallId="c1", status="in_progress" + ), + RESULT, + RESULT, + text("HER"), + text("ON"), + update("usage_update", used=999, size=1000000), + ], + { + "stopReason": "end_turn", + "usage": { + "inputTokens": 10, + "outputTokens": 4, + "cacheReadTokens": 2, + "cacheWriteTokens": 1, + "reasoningTokens": 3, + }, + }, + ) + ] + user = ai.user_message("go") + async with harness: + stream = await run(harness, [user]) + assert harness.session.id == SESSION + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + ] + assert harness.session.messages[0].id == user.id + message = stream.message + assert message.text == "HERON" + assert message.reasoning == "plan" + (call,) = message.builtin_tool_calls + assert call.tool_name == "shell" + assert json.loads(call.tool_args) == CALL["params"]["update"]["rawInput"] + (result,) = message.builtin_tool_returns + assert result.result == "HERON\n" + assert message.usage is not None + assert message.usage.input_tokens == 10 + assert message.usage.output_tokens == 4 + assert message.usage.cache_read_tokens == 2 + assert message.usage.cache_write_tokens == 1 + assert message.usage.reasoning_tokens == 3 + metadata = message.provider_metadata or {} + assert metadata["model"] == PREFS["model"] + assert metadata["fx_version"] == "0.0.17" + + +async def test_start_gives_the_bridge_and_settings( + server: AcpServer, harness: external.Fx +) -> None: + harness.provider = external.FxProvider( + executable="/custom/fx", env={"FX_HOME": str(server.home)} + ) + harness.model_provider = "gateway" + harness.model_name = "openai/gpt-test" + harness.effort = "high" + harness.instructions = "Be brief." + async with harness: + await harness.session.start() + argv, kwargs = server.spawned[0] + assert argv == [ + "/custom/fx", + "acp", + "--permission-mode", + "full-access", + "--provider", + "gateway", + "--model", + "openai/gpt-test", + "--effort", + "high", + ] + assert kwargs == { + "cwd": server.cwd, + "env": {"FX_HOME": str(server.home)}, + } + (init,) = server.sent("initialize") + assert init["params"]["protocolVersion"] == 1 + assert init["params"]["clientCapabilities"]["session"]["notices"] == {} + (new,) = server.sent("session/new") + assert new["params"]["cwd"] == server.cwd + assert new["params"]["systemPrompt"] == [ + {"type": "text", "text": "Be brief."} + ] + (bridge,) = new["params"]["mcpServers"] + assert harness.session._run is not None + assert bridge == { + "type": "http", + "name": "ai_sdk", + "url": bridge["url"], + "headers": [ + { + "name": "Authorization", + "value": f"Bearer {harness.session._run.bridge.token}", + } + ], + } + assert bridge["url"].startswith("http://127.0.0.1:") + assert server.process.terminated + + +@pytest.mark.parametrize("instructions", [None, ""]) +async def test_default_start_omits_optional_flags_and_empty_instructions( + server: AcpServer, harness: external.Fx, instructions: str | None +) -> None: + harness.instructions = instructions + async with harness: + await harness.session.start() + assert server.spawned[0][0] == [ + "fx", + "acp", + "--permission-mode", + "full-access", + ] + assert "systemPrompt" not in server.sent("session/new")[0]["params"] + + +async def test_a_new_session_loads_the_written_past_without_streaming_replay( + server: AcpServer, harness: external.Fx +) -> None: + server.turns = [([text("fresh")], END)] + harness.instructions = "Keep the seed instructions." + history = [ + ai.user_message("one"), + ai.assistant_message("old reply"), + ai.user_message("two"), + ] + async with harness: + stream = await run(harness, history) + assert ( + harness.session.id + == server.sent("session/load")[0]["params"]["sessionId"] + ) + assert stream.text == "fresh" + assert [ + r["data"]["text"] + for r in server.loaded_records + if r.get("type") in ("user", "assistant") + ] == ["one", "old reply"] + prefs = next( + r["value"] for r in server.loaded_records if r.get("key") == "prefs" + ) + assert prefs == PREFS + assert ( + next( + r["value"] + for r in server.loaded_records + if r.get("key") == "client_prompt" + ) + == "Keep the seed instructions." + ) + assert server.loaded_records[0]["workspace"] == server.cwd + methods = [r.get("method") for r in server.process.written] + assert ( + methods.index("initialize") + < methods.index("session/new") + < methods.index("session/load") + < methods.index("session/prompt") + ) + assert not server.sent("session/list") + assert ( + server.sent("session/load")[0]["params"]["mcpServers"] + == server.sent("session/new")[0]["params"]["mcpServers"] + ) + assert server.prompts[0]["params"]["prompt"] == [ + {"type": "text", "text": "two"} + ] + + +@pytest.mark.parametrize("failure", ["load", "missing replay"]) +async def test_a_refused_seed_is_removed( + server: AcpServer, harness: external.Fx, failure: str +) -> None: + sibling = server.home / "sessions/v2/UserSession01" + sibling.mkdir(parents=True) + sibling_log = sibling / "log.jsonl" + sibling_log.write_bytes(b"user session must stay unchanged\n") + index = sibling.parent / "index.jsonl" + index.write_bytes(b"user index must stay unchanged\n") + server.load_error = failure == "load" + server.replay_users = 0 if failure == "missing replay" else None + with pytest.raises( + external.ExternalHarnessError, match=r"0\.0\.17" + ) as error: + async with harness: + await run( + harness, + [ + ai.user_message("old"), + ai.assistant_message("reply"), + ai.user_message("new"), + ], + ) + assert str(error.value).startswith( + "fx 0.0.17: this version of ai cannot seed an fx session: " + ) + assert sibling_log.read_bytes() == b"user session must stay unchanged\n" + assert index.read_bytes() == b"user index must stay unchanged\n" + assert server.loaded is not None and not server.loaded.exists() + assert server.closed_folders[0] == (server.loaded.name, True) + assert len(server.sent("session/new")) == 1 + assert not server.prompts + assert server.process.terminated + + +async def test_a_resume_reads_all_pages_and_opens_here_only( + server: AcpServer, harness: external.Fx +) -> None: + server.pages = [["other"], [SESSION]] + harness = external.Fx( + cwd=harness.cwd, provider=harness.provider, resume=SESSION + ) + async with harness: + await harness.session.start() + (resume,) = server.sent("session/resume") + assert resume["params"]["sessionId"] == SESSION + assert resume["params"]["cwd"] == server.cwd + assert resume["params"]["mcpServers"][0]["name"] == "ai_sdk" + assert len(server.sent("session/list")) == 2 + + +@pytest.mark.parametrize("session_id", ["elsewhere", "unknown"]) +async def test_a_resume_refuses_an_id_not_in_this_directory( + server: AcpServer, harness: external.Fx, session_id: str +) -> None: + with pytest.raises(external.ExternalHarnessError): + async with external.Fx( + cwd=harness.cwd, provider=harness.provider, resume=session_id + ) as harness: + await harness.session.start() + assert not server.sent("session/resume") + + +async def test_bridge_calls_are_ai_tool_calls_not_builtin( + server: AcpServer, harness: external.Fx +) -> None: + @ai.tool + async def lookup() -> str: + """Return the word.""" + return "PELICAN" + + async def call_bridge() -> None: + await server.wait_for("session/prompt") + bridge = server.sent("session/new")[0]["params"]["mcpServers"][0] + server.says( + update( + "tool_call", + toolCallId="bridge-1", + name="mcp_ai_sdk_lookup", + title="lookup", + kind="other", + status="pending", + rawInput={}, + _meta={ + "fx": { + "toolCall": { + "internal": False, + "mcp": {"server": "ai_sdk", "tool": "lookup"}, + } + } + }, + ) + ) + reply = await fakes.mcp_call( + bridge["url"], + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "lookup", "arguments": {}}, + }, + {h["name"]: h["value"] for h in bridge["headers"]}, + ) + assert "PELICAN" in reply["result"]["content"][0]["text"] + server.says( + update( + "tool_call_update", + toolCallId="bridge-1", + status="completed", + content=[ + { + "type": "content", + "content": {"type": "text", "text": "PELICAN"}, + } + ], + ), + text("done"), + ) + server.finish(server.prompts[0], END) + + server.turns = [([], None)] + task = asyncio.create_task(call_bridge()) + async with ( + asyncio.timeout(5), + harness, + ai.Agent(tools=[lookup]).run( + harness, [ai.user_message("go")] + ) as result, + ): + async for _ in result: + pass + await task + assert any( + msg.role == "tool" + and any("PELICAN" in str(part.result) for part in msg.tool_results) + for msg in result.messages + ) + assert not any( + msg.builtin_tool_calls or msg.builtin_tool_returns + for msg in result.messages + ) + + +async def test_a_send_during_a_turn_waits_for_its_correlated_echo( + server: AcpServer, harness: external.Fx +) -> None: + server.turns = [([text("Look")], None)] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + events = [] + await harness.session.send(ai.user_message("fix")) + async for event in feed: + events.append(event) + if isinstance(event, ai.events.TextDelta): + break + steer = asyncio.create_task( + harness.session.send(ai.user_message("skip slow ones")) + ) + await server.wait_for("session/prompt", 2) + request = server.steers[0] + assert request["params"]["_meta"] == {"fx": {"steer": True}} + assert request["params"]["prompt"] == [ + {"type": "text", "text": "skip slow ones"} + ] + server.says( + update( + "user_message_chunk", + messageId="unrelated", + content={"type": "text", "text": "skip slow ones"}, + _meta={"fx": {"steering": {"requestId": -1}}}, + ), + text(" before", "a1"), + ) + async for event in feed: + events.append(event) + if ( + isinstance(event, ai.events.TextDelta) + and event.chunk == " before" + ): + break + assert [ + msg.text for msg in harness.session.messages if msg.role == "user" + ] == ["fix"] + server.says( + update( + "user_message_chunk", + messageId="u2", + content={"type": "text", "text": "skip slow ones"}, + _meta={"fx": {"steering": {"requestId": request["id"]}}}, + ), + text("OK", "a2"), + ) + server.finish(server.prompts[0], END) + server.finish( + request, {**END, "_meta": {"fx": {"steering": "absorbed"}}} + ) + await steer + events += await through_turns(feed, 2) + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 2 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["other", "stop"] + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "fix"), + ("assistant", "Look before"), + ("user", "skip slow ones"), + ("assistant", "OK"), + ] + + +@pytest.mark.parametrize("outcome", ["dropped", "not delivered"]) +async def test_a_dropped_steer_starts_a_new_prompt( + server: AcpServer, harness: external.Fx, outcome: str +) -> None: + server.turns = [ + ([text("first", "a1")], None), + ([text("second", "a2")], END), + ] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("one")) + task = asyncio.create_task(harness.session.send(ai.user_message("two"))) + await server.wait_for("session/prompt", 2) + server.finish(server.prompts[0], END) + request = server.steers[0] + if outcome == "dropped": + server.finish( + request, + { + "stopReason": "cancelled", + "_meta": {"fx": {"steering": "dropped"}}, + }, + ) + else: + server.says( + { + "id": request["id"], + "error": { + "code": -32600, + "message": ( + "Steering prompt was not delivered because " + "the turn ended first; send it as a new prompt" + ), + }, + } + ) + await task + await server.wait_for("session/prompt", 3) + events = await through_turns(feed, 2) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "one"), + ("assistant", "first"), + ("user", "two"), + ("assistant", "second"), + ] + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 2 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["stop", "stop"] + assert server.prompts[1]["params"]["prompt"] == [ + {"type": "text", "text": "two"} + ] + assert ( + not server.prompts[1]["params"] + .get("_meta", {}) + .get("fx", {}) + .get("steer") + ) + + +@pytest.mark.parametrize("stop_reason", ["cancelled", "end_turn", "refused"]) +async def test_interrupt_ends_as_interrupted( + server: AcpServer, harness: external.Fx, stop_reason: str +) -> None: + server.turns = [([text("work")], None)] + server.cancel_result = {"stopReason": stop_reason} + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("long job")) + async for event in feed: + if isinstance(event, ai.events.TextDelta): + break + await harness.session.interrupt() + (cancel,) = server.sent("session/cancel") + assert "id" not in cancel + assert cancel["params"] == {"sessionId": SESSION} + assert (harness.session.messages[-1].provider_metadata or {})[ + "external" + ] == {"stop_reason": "interrupted"} + assert harness.session.messages[-1].text == "work" + + +async def test_a_failed_initialize_without_sign_in_is_not_authenticated( + server: AcpServer, harness: external.Fx +) -> None: + server.initialize_error = { + "code": -32600, + "message": ( + "fx needs access to Vercel AI Gateway. Run fx login to " + "sign in, fx login vercel --api-key to use an API key, " + "or set AI_GATEWAY_API_KEY." + ), + } + with pytest.raises(external.NotAuthenticatedError): + async with harness: + await run(harness, [ai.user_message("go")]) + assert server.process.terminated + + +@pytest.mark.parametrize("recovery", [True, False]) +async def test_a_refused_turn_raises_with_the_last_error_detail( + server: AcpServer, harness: external.Fx, recovery: bool +) -> None: + records = [text("first failure", "a1"), text("last failure", "a2")] + if recovery: + records += [ + update( + "session_info_update", + _meta={ + "fx": { + "modelResponseRecovery": { + "state": "waiting", + "attempt": 1, + "message": "old recovery", + } + } + }, + ), + update( + "session_info_update", + _meta={ + "fx": { + "modelResponseRecovery": { + "state": "waiting", + "attempt": 2, + "message": "final recovery", + } + } + }, + ), + ] + server.turns = [(records, {"stopReason": "refused"})] + with pytest.raises( + external.ExternalHarnessError, + ) as error: + async with harness: + await run(harness, [ai.user_message("go")]) + detail = "final recovery" if recovery else "last failure" + assert str(error.value) == f"fx turn failed: {detail}" + + +async def test_a_crash_reaches_the_stream( + server: AcpServer, harness: external.Fx +) -> None: + server.turns = [([text("half")], None)] + + async def die() -> None: + await server.wait_for("session/prompt") + await server.process.terminate() + + task = asyncio.create_task(die()) + with pytest.raises(external.ExternalHarnessError, match=r"crashed|stopped"): + async with harness: + await run(harness, [ai.user_message("go")]) + await task + + +async def test_close_is_safe_twice( + server: AcpServer, harness: external.Fx +) -> None: + async with harness: + await harness.session.start() + await harness.session._close() + await harness.session._close() + assert server.process.terminated + assert len(server.sent("session/close")) == 1 + + +async def test_a_failed_handshake_stops_the_process( + server: AcpServer, harness: external.Fx +) -> None: + server.initialize_error = {"code": -32602, "message": "unsupported effort"} + with pytest.raises( + external.ExternalHarnessError, match="unsupported effort" + ): + async with harness: + await harness.session.start() + assert server.process.terminated + + +async def test_text_and_reasoning_blocks_close_without_end_markers( + server: AcpServer, harness: external.Fx +) -> None: + server.turns = [ + ( + [ + text("one"), + text("two", "a2"), + update( + "agent_thought_chunk", + content={"type": "text", "text": "think"}, + ), + update( + "agent_thought_chunk", + content={"type": "text", "text": " more"}, + ), + text("three", "a3"), + ], + END, + ) + ] + events = [] + async with ( + asyncio.timeout(5), + harness, + ai.stream(harness, [ai.user_message("go")]) as stream, + ): + async for event in stream: + events.append(event) + blocks = [ + e + for e in events + if isinstance( + e, + ai.events.TextStart + | ai.events.TextEnd + | ai.events.ReasoningStart + | ai.events.ReasoningEnd, + ) + ] + assert [type(e) for e in blocks] == [ + ai.events.TextStart, + ai.events.TextEnd, + ai.events.TextStart, + ai.events.TextEnd, + ai.events.ReasoningStart, + ai.events.ReasoningEnd, + ai.events.TextStart, + ai.events.TextEnd, + ] + for start, end in zip(blocks[::2], blocks[1::2], strict=True): + assert start.block_id == end.block_id + assert len({e.block_id for e in blocks[::2]}) == 4 + assert sum(isinstance(e, ai.events.StreamEnd) for e in events) == 1 + + +async def test_seed_replay_counts_distinct_user_ids_not_image_chunks( + server: AcpServer, + harness: external.Fx, +) -> None: + image = m.FilePart(data=b"image bytes", media_type="image/png") + history = [ + m.Message(role="user", parts=[m.TextPart(text="past picture"), image]), + ai.assistant_message("past answer"), + ai.user_message("now"), + ] + server.turns = [([text("fresh")], END)] + async with harness: + stream = await run(harness, history) + assert stream.text == "fresh" + assert server.loaded is not None and server.loaded.exists() + + +@pytest.mark.parametrize( + "native, expected", + [ + ("end_turn", "stop"), + ("max_output_tokens", "length"), + ("max_model_turns", "length"), + ("cancelled", "other"), + ], +) +async def test_prompt_stop_reasons_have_one_stream_end( + server: AcpServer, + harness: external.Fx, + native: str, + expected: str, +) -> None: + server.turns = [([text("done")], {"stopReason": native})] + ends = [] + async with ( + asyncio.timeout(5), + harness, + ai.stream(harness, [ai.user_message("go")]) as stream, + ): + async for event in stream: + if isinstance(event, ai.events.StreamEnd): + ends.append(event) + assert len(ends) == 1 + assert ends[0].finish_reason == expected + + +async def test_an_unexpected_permission_request_gets_an_answer( + server: AcpServer, + harness: external.Fx, +) -> None: + async with asyncio.timeout(5), harness: + await harness.session.start() + server.says( + { + "id": 99, + "method": "session/request_permission", + "params": { + "sessionId": SESSION, + "toolCall": { + "toolCallId": "c", + "name": "shell", + "title": "Running", + "kind": "execute", + "status": "pending", + "rawInput": {}, + }, + "options": [ + { + "optionId": "allow_once", + "name": "Allow once", + "kind": "allow_once", + } + ], + }, + } + ) + while not server.replies: + await asyncio.sleep(0) + assert server.replies[0]["id"] == 99 + assert server.replies[0]["result"] == { + "outcome": {"outcome": "selected", "optionId": "allow_once"} + } + + +async def test_a_nontext_steer_waits_and_becomes_a_new_prompt( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([], None), ([text("seen")], END)] + image = m.FilePart(data=b"image bytes", media_type="image/png") + async with asyncio.timeout(5), harness: + await harness.session.send(ai.user_message("first")) + task = asyncio.create_task( + harness.session.send( + m.Message(role="user", parts=[m.TextPart(text="look"), image]) + ) + ) + await asyncio.sleep(0.01) + assert len(server.sent("session/prompt")) == 1 + server.finish(server.prompts[0], END) + await task + await server.wait_for("session/prompt", 2) + assert not server.steers + assert server.prompts[1]["params"]["prompt"] == [ + {"type": "text", "text": "look"}, + { + "type": "image", + "mimeType": "image/png", + "data": base64.b64encode(b"image bytes").decode(), + }, + ] + + +async def test_a_prompt_rpc_error_reaches_the_stream( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([], None)] + + async def reject() -> None: + await server.wait_for("session/prompt") + server.says( + { + "id": server.prompts[0]["id"], + "error": {"code": -32603, "message": "model request failed"}, + } + ) + + task = asyncio.create_task(reject()) + with pytest.raises( + external.ExternalHarnessError, match="model request failed" + ): + async with harness: + await run(harness, [ai.user_message("go")]) + await task + + +async def test_a_refused_turn_with_bad_credentials_is_not_authenticated( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [ + ( + [text("gateway authentication failed")], + {"stopReason": "refused"}, + ) + ] + with pytest.raises(external.NotAuthenticatedError): + async with harness: + await run(harness, [ai.user_message("go")]) + + +async def through_turns(feed: AsyncIterator[Any], count: int) -> list[Any]: + events = [] + async with asyncio.timeout(5): + async for event in feed: + events.append(event) + if isinstance(event, ai.events.StreamEnd): + count -= 1 + if count == 0: + break + try: + async with asyncio.timeout(0.02): + async for event in feed: + events.append(event) + except TimeoutError: + pass + return events + + +@pytest.mark.parametrize("has_content", [True, False]) +async def test_a_late_steer_runs_as_its_own_turn( + server: AcpServer, + harness: external.Fx, + has_content: bool, +) -> None: + # Native order from fx 0.0.17: first reply, second content, plain reply. + server.turns = [([text("ONE", "a1")], None)] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("ONE")) + await harness.session.send(ai.user_message("TWO")) + request = server.steers[0] + server.finish( + server.prompts[0], + {**END, "usage": {"inputTokens": 11, "outputTokens": 1}}, + ) + if has_content: + server.says(text("TWO", "a2")) + server.finish( + request, {**END, "usage": {"inputTokens": 22, "outputTokens": 2}} + ) + events = await through_turns(feed, 2) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE"), + ("user", "TWO"), + ("assistant", "TWO" if has_content else ""), + ] + boundaries = [ + e + for e in events + if isinstance(e, ai.events.StreamStart | ai.events.StreamEnd) + ] + assert [type(e) for e in boundaries] == [ + ai.events.StreamStart, + ai.events.StreamEnd, + ai.events.StreamStart, + ai.events.StreamEnd, + ] + ends = [e for e in events if isinstance(e, ai.events.StreamEnd)] + assert [e.finish_reason for e in ends] == ["stop", "stop"] + usages = [e.usage for e in ends] + assert all(usage is not None for usage in usages) + assert [ + usage.input_tokens for usage in usages if usage is not None + ] == [11, 22] + assert [ + usage.output_tokens for usage in usages if usage is not None + ] == [1, 2] + if has_content: + second_start = events.index(boundaries[2]) + second_delta = next( + i + for i, e in enumerate(events) + if isinstance(e, ai.events.TextDelta) and e.chunk == "TWO" + ) + assert ( + events.index(boundaries[1]) + < second_start + < second_delta + < events.index(boundaries[3]) + ) + assert ( + not server.sent("session/prompt")[1]["params"].get("prompt") == [] + ) + assert len(server.sent("session/prompt")) == 2 + + +async def test_interrupt_cancels_the_turn_of_a_late_steer( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([text("ONE", "a1")], None)] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("ONE")) + await harness.session.send(ai.user_message("TWO")) + request = server.steers[0] + server.cancel_request = request + server.finish(server.prompts[0], END) + server.says(text("TWO", "a2")) + events = [] + async for event in feed: + events.append(event) + if isinstance(event, ai.events.TextDelta) and event.chunk == "TWO": + break + await harness.session.interrupt() + (cancel,) = server.sent("session/cancel") + assert cancel["params"] == {"sessionId": SESSION} + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE"), + ("user", "TWO"), + ("assistant", "TWO"), + ] + assert (harness.session.messages[-1].provider_metadata or {})[ + "external" + ] == {"stop_reason": "interrupted"} + + +async def test_a_rejected_steer_waits_for_the_running_turn_then_retries( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([text("ONE", "a1")], None), ([text("TWO", "a2")], END)] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("ONE")) + await harness.session.send(ai.user_message("TWO")) + request = server.steers[0] + server.says( + { + "id": request["id"], + "error": { + "code": -32602, + "message": "Steering prompt too long", + }, + }, + text(" CONTINUES", "a1"), + ) + events = [] + async for event in feed: + events.append(event) + if ( + isinstance(event, ai.events.TextDelta) + and event.chunk == " CONTINUES" + ): + break + assert not any(isinstance(e, ai.events.StreamEnd) for e in events) + assert [ + msg.text for msg in harness.session.messages if msg.role == "user" + ] == ["ONE"] + assert len(server.sent("session/prompt")) == 2 + server.finish(server.prompts[0], END) + events += await through_turns(feed, 2) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE CONTINUES"), + ("user", "TWO"), + ("assistant", "TWO"), + ] + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 2 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["stop", "stop"] + assert len(server.prompts) == 2 and len(server.steers) == 1 + assert server.prompts[1]["params"]["prompt"] == [ + {"type": "text", "text": "TWO"} + ] + + +async def test_updates_after_a_prompt_response_do_not_join_the_ended_turn( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([CALL], None)] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("ONE")) + server.finish(server.prompts[0], END) + server.says(RESULT, text("late text", "late")) + events = await through_turns(feed, 1) + assert not any( + isinstance(e, ai.events.BuiltinToolResult | ai.events.TextDelta) + for e in events + ), ", ".join(type(e).__name__ for e in events) + assert harness.session.messages[-1].builtin_tool_returns == [] + assert harness.session.messages[-1].text == "" + + +@pytest.mark.parametrize( + "native", ["session_info_update", "notice", "usage_update"] +) +async def test_noncontent_updates_do_not_split_a_text_block( + server: AcpServer, + harness: external.Fx, + native: str, +) -> None: + fields_by_kind: dict[str, dict[str, Any]] = { + "session_info_update": {"title": "title"}, + "notice": {"title": "notice", "severity": "info"}, + "usage_update": {"used": 1, "size": 100}, + } + fields = fields_by_kind[native] + server.turns = [([text("HE"), update(native, **fields), text("RON")], END)] + events = [] + async with ( + harness, + ai.stream(harness, [ai.user_message("go")]) as stream, + ): + async for event in stream: + events.append(event) + assert stream.text == "HERON" + assert len([e for e in events if isinstance(e, ai.events.TextStart)]) == 1 + assert len([e for e in events if isinstance(e, ai.events.TextEnd)]) == 1 + assert len(stream.message.parts) == 1 + + +async def test_a_final_tool_update_without_a_call_is_ignored( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([RESULT, text("done")], END)] + events = [] + async with ( + harness, + ai.stream(harness, [ai.user_message("go")]) as stream, + ): + async for event in stream: + events.append(event) + assert not any(isinstance(e, ai.events.BuiltinToolResult) for e in events) + assert stream.message.builtin_tool_returns == [] + + +async def test_an_unknown_server_request_gets_an_error_reply( + server: AcpServer, + harness: external.Fx, +) -> None: + async with asyncio.timeout(5), harness: + await harness.session.start() + server.says({"id": 98, "method": "unknown/request", "params": {}}) + while not server.replies: + await asyncio.sleep(0) + reply = server.replies[0] + assert reply["id"] == 98 and "result" not in reply + assert isinstance(reply["error"], dict) + assert "code" in reply["error"] and "message" in reply["error"] + + +async def test_credential_refresh_failure_is_not_authenticated( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [ + ([text("credential refresh failed")], {"stopReason": "refused"}) + ] + with pytest.raises(external.NotAuthenticatedError): + async with harness: + await run(harness, [ai.user_message("go")]) + + +async def test_a_crash_while_idle_reaches_session_events( + server: AcpServer, + harness: external.Fx, +) -> None: + async with asyncio.timeout(5), harness: + await harness.session.start() + await server.process.terminate() + with pytest.raises( + external.ExternalHarnessError, match=r"^fx crashed:" + ): + await anext(harness.session.events()) + + +async def test_early_content_waits_for_acceptance_while_write_yields( + server: AcpServer, + harness: external.Fx, +) -> None: + server.process.yield_prompt_write = True + server.turns = [([text("reply")], END)] + async with harness: + stream = await run(harness, [ai.user_message("question")]) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "question"), + ("assistant", "reply"), + ] + assert stream.text == "reply" + + +async def test_a_resume_refuses_a_listed_session_with_a_different_cwd( + server: AcpServer, + harness: external.Fx, +) -> None: + server.listed_cwd = "/other" + with pytest.raises( + external.ExternalHarnessError, + match="unknown or belongs to another directory", + ): + async with external.Fx( + cwd=harness.cwd, provider=harness.provider, resume=SESSION + ) as harness: + await harness.session.start() + assert not server.sent("session/resume") + + +async def test_wire_start_omits_empty_instructions_and_close_is_safe_twice( + server: AcpServer, + harness: external.Fx, +) -> None: + # The close of the session has its own guard. Check the Wire contract + # separately. + wire = wire_.FxWire(harness) + await wire.start( + session_id=None, + bridge_url="http://127.0.0.1:1/mcp", + bridge_token="test", + instructions="", + seed=[], + ) + assert "systemPrompt" not in server.sent("session/new")[0]["params"] + await wire.close() + await wire.close() + assert server.process.terminated + assert len(server.sent("session/close")) == 1 + + +@pytest.mark.parametrize("outcome", ["dropped", "absorbed"]) +async def test_a_no_echo_steer_always_ends_and_the_next_send_is_normal( + server: AcpServer, + harness: external.Fx, + outcome: str, +) -> None: + server.turns = [([text("ONE", "a1")], None), ([text("THREE", "a3")], END)] + messages = [ai.user_message(value) for value in ("ONE", "TWO", "THREE")] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(messages[0]) + await harness.session.send(messages[1]) + request = server.steers[0] + server.finish(server.prompts[0], END) + server.says(text("TWO", "a2")) + server.finish( + request, + { + "stopReason": "cancelled" + if outcome == "dropped" + else "end_turn", + "_meta": {"fx": {"steering": outcome}}, + }, + ) + events = [] + async for event in feed: + events.append(event) + if sum(isinstance(e, ai.events.StreamEnd) for e in events) == 2: + break + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE"), + ("user", "TWO"), + ("assistant", "TWO"), + ] + await harness.session.send(messages[2]) + events += await through_turns(feed, 1) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE"), + ("user", "TWO"), + ("assistant", "TWO"), + ("user", "THREE"), + ("assistant", "THREE"), + ] + assert [ + msg.id for msg in harness.session.messages if msg.role == "user" + ] == [msg.id for msg in messages] + assert len(server.prompts) == 2 and len(server.steers) == 1 + assert not server.rejected_prompts + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 3 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["stop", "other" if outcome == "dropped" else "stop", "stop"] + + +async def test_a_send_in_the_gap_before_late_steer_content_keeps_send_order( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [([text("ONE", "a1")], None), ([text("THREE", "a3")], END)] + messages = [ai.user_message(value) for value in ("ONE", "TWO", "THREE")] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(messages[0]) + await harness.session.send(messages[1]) + second = server.steers[0] + server.finish(server.prompts[0], END) + events = [] + async for event in feed: + events.append(event) + if isinstance(event, ai.events.StreamEnd): + break + third_send = asyncio.create_task(harness.session.send(messages[2])) + # Let send() write, without requiring it to bypass a queue. + await asyncio.sleep(0.01) + assert not server.rejected_prompts + server.says(text("TWO", "a2")) + server.finish(second, END) + await server.wait_for("session/prompt", 3) + third = server.sent("session/prompt")[2] + if third in server.steers: + server.says(text("THREE", "a3")) + server.finish(third, END) + await third_send + events += await through_turns(feed, 2) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE"), + ("user", "TWO"), + ("assistant", "TWO"), + ("user", "THREE"), + ("assistant", "THREE"), + ] + assert [ + msg.id for msg in harness.session.messages if msg.role == "user" + ] == [msg.id for msg in messages] + assert not server.rejected_prompts + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 3 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["stop", "stop", "stop"] + + +@pytest.mark.parametrize("late", ["text", "tool result"]) +async def test_a_late_update_after_interrupt_does_not_accept_a_dropped_steer( + server: AcpServer, + harness: external.Fx, + late: str, +) -> None: + server.turns = [ + ([text("ONE", "a1"), CALL], None), + ([text("TWO", "a2")], END), + ] + server.cancel_result = {"stopReason": "cancelled"} + server.cancel_updates = [text("LATE", "late") if late == "text" else RESULT] + messages = [ai.user_message(value) for value in ("ONE", "TWO")] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + await harness.session.send(messages[0]) + await harness.session.send(messages[1]) + request = server.steers[0] + await harness.session.interrupt() + server.finish( + request, + { + "stopReason": "cancelled", + "_meta": {"fx": {"steering": "dropped"}}, + }, + ) + events = await through_turns(feed, 2) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "ONE"), + ("assistant", "ONE"), + ("user", "TWO"), + ("assistant", "TWO"), + ] + assert [ + msg.id for msg in harness.session.messages if msg.role == "user" + ] == [msg.id for msg in messages] + assert not any( + isinstance(e, ai.events.TextDelta) and e.chunk == "LATE" + for e in events + ) + assert not any( + isinstance(e, ai.events.BuiltinToolResult) for e in events + ) + assert len(server.prompts) == 2 and len(server.steers) == 1 + assert not server.rejected_prompts + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 2 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["other", "stop"] + + +async def test_a_cancelled_seed_load_closes_before_removing_its_folder( + server: AcpServer, + harness: external.Fx, +) -> None: + server.hold_load = True + sibling = server.home / "sessions/v2/UserSession01" + sibling.mkdir(parents=True) + (sibling / "log.jsonl").write_bytes(b"keep") + index = sibling.parent / "index.jsonl" + index.write_bytes(b"keep index") + history = [ai.user_message("old"), ai.assistant_message("old answer")] + harness = external.Fx( + cwd=harness.cwd, provider=harness.provider, history=history + ) + async with asyncio.timeout(5), harness: + task = asyncio.create_task(harness.session.start()) + await server.wait_for("session/load") + assert server.loaded is not None and server.loaded.exists() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert not server.loaded.exists() + assert server.closed_folders[0] == (server.loaded.name, True) + assert (sibling / "log.jsonl").read_bytes() == b"keep" + assert index.read_bytes() == b"keep index" + assert server.process.terminated + + +@pytest.mark.parametrize("ending", ["crash", "close"]) +async def test_an_image_send_waiting_for_a_turn_raises_when_the_process_stops( + server: AcpServer, + harness: external.Fx, + ending: str, +) -> None: + server.turns = [([text("working")], None)] + image_message = m.Message( + role="user", + parts=[ + m.TextPart(text="look"), + m.FilePart(data=b"image", media_type="image/png"), + ], + ) + async with asyncio.timeout(5), harness: + await harness.session.send(ai.user_message("first")) + pending = asyncio.create_task(harness.session.send(image_message)) + try: + await asyncio.sleep(0.01) + assert not pending.done() + assert len(server.sent("session/prompt")) == 1 + if ending == "crash": + await server.process.terminate() + else: + await harness.session._close() + with pytest.raises(external.ExternalHarnessError): + async with asyncio.timeout(1): + await pending + finally: + if not pending.done(): + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + + +async def test_a_newer_steer_echo_does_not_lose_an_older_refused_steer( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [ + ([text("T reply", "a1")], None), + ([text("OLDER reply", "a3")], END), + ] + initial, older, newer = [ + ai.user_message(value) for value in ("T", "OLDER", "NEWER") + ] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + for message in (initial, older, newer): + await harness.session.send(message) + old_request, new_request = server.steers + server.finish(server.prompts[0], END) + server.says( + update( + "user_message_chunk", + messageId="newer", + content={"type": "text", "text": "NEWER"}, + _meta={"fx": {"steering": {"requestId": new_request["id"]}}}, + ), + text("NEWER reply", "a2"), + ) + server.finish(new_request, END) + server.says( + { + "id": old_request["id"], + "error": { + "code": -32600, + "message": ( + "Steering prompt was not delivered because " + "the turn ended first; send it as a new prompt" + ), + }, + } + ) + events = await through_turns(feed, 3) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "T"), + ("assistant", "T reply"), + ("user", "NEWER"), + ("assistant", "NEWER reply"), + ("user", "OLDER"), + ("assistant", "OLDER reply"), + ] + assert [ + msg.id for msg in harness.session.messages if msg.role == "user" + ] == [ + initial.id, + newer.id, + older.id, + ] + assert [ + request["params"]["prompt"][0]["text"] for request in server.prompts + ] == ["T", "OLDER"] + assert not server.rejected_prompts + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 3 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == ["stop", "stop", "stop"] + + +@pytest.mark.parametrize( + ("accepted_outcome", "echo_before_end"), + [("absorbed", False), ("dropped", False), ("dropped", True)], +) +async def test_three_steers_keep_retry_order_and_accept_each_message_once( + server: AcpServer, + harness: external.Fx, + accepted_outcome: str, + echo_before_end: bool, +) -> None: + server.turns = [ + ([text("T reply", "a1")], None), + ([text("S1 reply", "a3")], None), + ([text("S3 reply", "a4")], None), + ([text("NEW reply", "a5")], END), + ] + initial, refused, accepted, dropped, queued = [ + ai.user_message(value) for value in ("T", "S1", "S2", "S3", "NEW") + ] + async with asyncio.timeout(5), harness: + feed = harness.session.events() + server.process.fail_next_prompt = True + failed = ai.user_message("not written") + with pytest.raises( + (OSError, external.ExternalHarnessError), + match="scripted write failed", + ): + await harness.session.send(failed) + await harness.session.send(initial) + assert len(server.prompts) == 1 and not server.steers + for message in (refused, accepted, dropped): + await harness.session.send(message) + s1, s2, s3 = server.steers + server.cancel_request = s2 + if not echo_before_end: + server.finish(server.prompts[0], END) + server.says( + update( + "user_message_chunk", + messageId="s2", + content={"type": "text", "text": "S2"}, + _meta={"fx": {"steering": {"requestId": s2["id"]}}}, + ), + text("S2 reply", "a2"), + ) + # Reverse reply order must not reverse the retry queue (S1, then S3). + server.finish( + s3, + { + "stopReason": "cancelled", + "_meta": {"fx": {"steering": "dropped"}}, + }, + ) + server.says( + { + "id": s1["id"], + "error": { + "code": -32600, + "message": ( + "Steering prompt was not delivered because " + "the turn ended first; send it as a new prompt" + ), + }, + }, + text(" READY", "a2"), + ) + events = [] + async for event in feed: + events.append(event) + if ( + isinstance(event, ai.events.TextDelta) + and event.chunk == " READY" + ): + break + new_send = asyncio.create_task(harness.session.send(queued)) + await asyncio.sleep(0.01) + assert [ + request["params"]["prompt"][0]["text"] + for request in server.sent("session/prompt") + ] == ["T", "S1", "S2", "S3"] + if echo_before_end: + server.finish(server.prompts[0], END) + server.finish( + s2, + { + "stopReason": "end_turn" + if accepted_outcome == "absorbed" + else "cancelled", + "_meta": {"fx": {"steering": accepted_outcome}}, + }, + ) + await server.wait_for("session/prompt", 5) + first_retry = server.prompts[1] + assert first_retry["params"]["prompt"] == [ + {"type": "text", "text": "S1"} + ] + assert not server.rejected_prompts + server.finish(first_retry, END) + await server.wait_for("session/prompt", 6) + second_retry = server.prompts[2] + assert second_retry["params"]["prompt"] == [ + {"type": "text", "text": "S3"} + ] + server.finish(second_retry, END) + await server.wait_for("session/prompt", 7) + new_request = server.sent("session/prompt")[6] + if new_request in server.steers: + server.says(text("NEW reply", "a5")) + server.finish(new_request, END) + await new_send + events += await through_turns(feed, 4) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "T"), + ("assistant", "T reply"), + ("user", "S2"), + ("assistant", "S2 reply READY"), + ("user", "S1"), + ("assistant", "S1 reply"), + ("user", "S3"), + ("assistant", "S3 reply"), + ("user", "NEW"), + ("assistant", "NEW reply"), + ] + assert [ + msg.id for msg in harness.session.messages if msg.role == "user" + ] == [ + initial.id, + accepted.id, + refused.id, + dropped.id, + queued.id, + ] + sent_texts = [ + request["params"]["prompt"][0]["text"] + for request in server.sent("session/prompt") + ] + assert sent_texts == ["T", "S1", "S2", "S3", "S1", "S3", "NEW"] + assert sent_texts.count("S2") == 1 + assert not server.rejected_prompts + assert ( + len([e for e in events if isinstance(e, ai.events.StreamStart)]) + == 5 + ) + assert [ + e.finish_reason + for e in events + if isinstance(e, ai.events.StreamEnd) + ] == [ + "other" if echo_before_end else "stop", + "stop" + if echo_before_end or accepted_outcome == "absorbed" + else "other", + "stop", + "stop", + "stop", + ] + + +async def test_seeding_warns_when_reasoning_is_not_carried_into_fx( + server: AcpServer, + harness: external.Fx, +) -> None: + history = [ + ai.user_message("old"), + m.Message( + role="assistant", + parts=[ + m.ReasoningPart(text="private thought"), + m.TextPart(text="old reply"), + ], + ), + ai.user_message("new"), + ] + server.turns = [([text("fresh")], END)] + with pytest.warns(UserWarning, match=r"1 reasoning part\(s\).*"): + async with harness: + stream = await run(harness, history) + assert stream.text == "fresh" + assert "private thought" not in json.dumps(server.loaded_records) + + +async def test_close_fails_a_send_waiting_behind_the_retry_queue( + server: AcpServer, + harness: external.Fx, +) -> None: + server.turns = [ + ([text("first", "a1")], None), + ([text("retry", "a2")], None), + ] + async with asyncio.timeout(5), harness: + await harness.session.send(ai.user_message("T")) + await harness.session.send(ai.user_message("S1")) + await harness.session.send(ai.user_message("S2")) + server.finish(server.prompts[0], END) + for request in server.steers: + server.finish( + request, + { + "stopReason": "cancelled", + "_meta": {"fx": {"steering": "dropped"}}, + }, + ) + await server.wait_for("session/prompt", 4) + pending = asyncio.create_task( + harness.session.send(ai.user_message("NEW")) + ) + try: + await asyncio.sleep(0.01) + assert not pending.done() + assert len(server.sent("session/prompt")) == 4 + await harness.session._close() + with pytest.raises(external.ExternalHarnessError): + async with asyncio.timeout(1): + await pending + assert all( + request["params"]["prompt"][0]["text"] != "NEW" + for request in server.sent("session/prompt") + ) + finally: + if not pending.done(): + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) diff --git a/tests/external/pi/__init__.py b/tests/external/pi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/external/pi/test_wire.py b/tests/external/pi/test_wire.py new file mode 100644 index 00000000..28bc22cb --- /dev/null +++ b/tests/external/pi/test_wire.py @@ -0,0 +1,802 @@ +"""The Pi external harness against a scripted `pi --mode rpc`, offline. + +The records are the shapes a real pi 1.5.0 sent when measured. The fake +plays a run's records BEFORE the `prompt` response unless a test says +otherwise: the external harness must still put the user message first. +""" + +from __future__ import annotations + +import asyncio +import json +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any, cast + +import pytest + +import ai +from ai import external +from ai.external import _process +from ai.external.pi import Pi +from ai.external.pi import wire as wire_ +from ai.types import messages as m + +from .. import fakes + + +def update(kind: str, index: int = 0, **fields: Any) -> dict[str, Any]: + return { + "type": "message_update", + "assistantMessageEvent": { + "type": kind, + "contentIndex": index, + **fields, + }, + } + + +def assistant(*content: dict[str, Any], stop: str = "stop") -> dict[str, Any]: + return { + "role": "assistant", + "content": list(content), + "api": "anthropic-messages", + "provider": "vercel-ai-gateway", + "model": "anthropic/claude-opus-5.5-fast", + "usage": { + "input": 10, + "output": 5, + "cacheRead": 100, + "cacheWrite": 0, + "cost": {"total": 0.01}, + }, + "stopReason": stop, + } + + +def user(text: str) -> list[dict[str, Any]]: + message = {"role": "user", "content": [{"type": "text", "text": text}]} + return [ + {"type": "message_start", "message": message}, + {"type": "message_end", "message": message}, + ] + + +def begin(text: str) -> list[dict[str, Any]]: + """The start of a run: pi's own system message, then the prompt.""" + system = {"role": "system", "content": "", "sections": {}} + return [ + {"type": "agent_start"}, + {"type": "turn_start"}, + {"type": "message_start", "message": system}, + {"type": "message_end", "message": system}, + *user(text), + ] + + +def said(text: str, index: int = 0) -> list[dict[str, Any]]: + """One assistant message that streams `text`.""" + return [ + {"type": "message_start", "message": assistant()}, + update("text_start", index), + update("text_delta", index, delta=text), + update("text_end", index, content=text), + { + "type": "message_end", + "message": assistant({"type": "text", "text": text}), + }, + ] + + +SETTLED: list[dict[str, Any]] = [ + {"type": "agent_end", "messages": []}, + {"type": "agent_settled", "aborted": False}, +] + +CALL = { + "type": "toolCall", + "id": "toolu_1", + "name": "bash", + "arguments": {"command": "echo hi"}, +} + +# thinking -> tool call -> tool result -> text: what pi sent for +# "run `echo hi`, then say done". +TOOL_TURN: list[dict[str, Any]] = [ + *begin("say done"), + {"type": "message_start", "message": assistant()}, + update("thinking_start"), + update("thinking_delta", delta="plan"), + update("thinking_end", content="plan"), + update("toolcall_start", 1, id="toolu_1", toolName="bash"), + update("toolcall_delta", 1, delta='{"command": "ec'), + update("toolcall_end", 1, toolCall=CALL), + { + "type": "message_end", + "message": assistant( + {"type": "thinking", "thinking": "plan"}, CALL, stop="toolUse" + ), + }, + { + "type": "tool_execution_start", + "toolCallId": "toolu_1", + "toolName": "bash", + "args": {"command": "echo hi"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "toolu_1", + "toolName": "bash", + "result": {"content": [{"type": "text", "text": "hi\n"}]}, + "isError": False, + }, + *said("done"), + *SETTLED, +] + + +class PiServer: + """Answers like `pi --mode rpc`. + + `turns[i]` plays out on the i-th prompt that starts a run, then the + response; `on_prompt`, if set, does it all instead. A prompt + while a run streams is queued, as a steer. `abort` settles the run as + aborted, then answers. + """ + + def __init__(self) -> None: + self.process = fakes.FakeProcess(self.answer) + self.turns: list[list[dict[str, Any]]] = [] + self.on_prompt: Callable[[dict[str, Any]], None] | None = None + self.started = 0 + self.streaming = False + self.loaded = True + self.prompt_error: str | None = None + self.spawned: list[tuple[list[str], dict[str, Any]]] = [] + + @property + def session_id(self) -> str: + argv = self.spawned[-1][0] + return argv[argv.index("--session-id") + 1] + + def says(self, *records: dict[str, Any]) -> None: + for record in records: + if record.get("type") == "agent_start": + self.streaming = True + elif record.get("type") == "agent_settled": + self.streaming = False + self.process.server_says(record) + + def respond(self, command: dict[str, Any], **fields: Any) -> None: + self.says( + { + "type": "response", + "id": command["id"], + "command": command["type"], + "success": True, + **fields, + } + ) + + def answer(self, command: dict[str, Any]) -> None: + kind = command.get("type") + if kind == "prompt": + if self.prompt_error is not None: + self.respond(command, success=False, error=self.prompt_error) + elif self.streaming: + self.says( + { + "type": "queue_update", + "steering": [command["message"]], + "followUp": [], + } + ) + self.respond(command, data={"disposition": "queued"}) + elif self.on_prompt is not None: + self.on_prompt(command) + else: + turn = self.turns[self.started] if self.turns else [] + self.started += 1 + self.says(*turn) + self.respond(command, data={"disposition": "started"}) + elif kind == "abort": + self.says( + { + "type": "message_end", + "message": assistant(stop="aborted"), + }, + {"type": "agent_end", "messages": []}, + {"type": "agent_settled", "aborted": True}, + ) + self.respond(command) + elif kind == "get_state": + self.respond( + command, + data={ + "sessionId": self.session_id, + "isStreaming": self.streaming, + }, + ) + elif kind == "get_commands": + names = [wire_.READY] if self.loaded else [] + self.respond( + command, data={"commands": [{"name": n} for n in names]} + ) + elif "id" in command: + self.respond(command) + + def sent(self, kind: str) -> list[dict[str, Any]]: + return [w for w in self.process.written if w.get("type") == kind] + + +@pytest.fixture +def cwd(tmp_path: Path) -> str: + project = tmp_path / "project" + project.mkdir() + return os.path.realpath(project) + + +@pytest.fixture +def agent_dir(tmp_path: Path) -> Path: + return tmp_path / "agent" + + +@pytest.fixture +def server(monkeypatch: pytest.MonkeyPatch) -> PiServer: + server = PiServer() + + async def probe(kind: str, executable: str, **kwargs: Any) -> str: + return "1.5.0" + + async def spawn(argv: list[str], **kwargs: Any) -> _process.Process: + server.spawned.append((argv, kwargs)) + if server.process.terminated: + server.process = fakes.FakeProcess(server.process.answer) + return cast("_process.Process", server.process) + + monkeypatch.setattr(_process, "probe", probe) + monkeypatch.setattr(_process, "spawn", spawn) + return server + + +def make(cwd: str, agent_dir: Path, **fields: Any) -> Pi: + provider = external.PiProvider( + executable=fields.pop("executable", None), + env={"PI_CODING_AGENT_DIR": str(agent_dir), **fields.pop("env", {})}, + ) + return Pi(cwd=cwd, provider=provider, **fields) + + +async def run(harness: Pi, messages: list[m.Message]) -> ai.Stream[str]: + async with asyncio.timeout(5): + async with ai.stream(harness, messages) as s: + async for _ in s: + pass + return s + + +def session_file(agent_dir: Path, session_id: str) -> Path: + (path,) = agent_dir.glob(f"sessions/*/*_{session_id}.jsonl") + return path + + +async def test_a_turn_streams_in_the_model_view( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [TOOL_TURN] + prompt = ai.user_message("say done") + harness = make(cwd, agent_dir) + async with harness: + s = await run(harness, [prompt]) + message = s.message + assert message.reasoning == "plan" + assert message.text == "done" + (call,) = message.builtin_tool_calls + assert call.tool_name == "bash" + # The arguments travel once, whole: pi's fragments are not what it + # settles on. + assert json.loads(call.tool_args) == {"command": "echo hi"} + (result,) = message.builtin_tool_returns + assert result.result == "hi\n" + # Two model calls: cached input counts as input, cost rides in raw. + usage = message.usage + assert usage is not None + assert (usage.input_tokens, usage.output_tokens) == (220, 10) + assert usage.cache_read_tokens == 200 + assert usage.raw == {"cost_usd": 0.02} + # Acceptance came first, although the run came before the response. + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + ] + assert harness.session.messages[0].id == prompt.id + assert harness.session.id == server.session_id + (sent,) = server.sent("prompt") + assert sent["message"] == "say done" + assert sent["streamingBehavior"] == "steer" + # Who wrote the reply, so a seeded pi continues on the same model. + assert message.provider_metadata == { + "pi": { + "api": "anthropic-messages", + "provider": "vercel-ai-gateway", + "model": "anthropic/claude-opus-5.5-fast", + } + } + + +async def test_start_runs_pi_with_the_extension_and_the_bridge( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + harness = make( + cwd, + agent_dir, + executable="/custom/pi", + env={"ANTHROPIC_API_KEY": "k"}, + model_name="claude-test", + model_provider="anthropic", + thinking="high", + instructions="Be brief.", + allowed_tools=[], + disallowed_tools=["write"], + args=["--no-skills"], + ) + async with harness: + await harness.session.start() + (argv, kwargs) = server.spawned[0] + assert argv[:3] == ["/custom/pi", "--mode", "rpc"] + assert argv[argv.index("-e") + 1] == wire_.EXTENSION + assert Path(wire_.EXTENSION).is_file() + assert argv[argv.index("--session-id") + 1] == harness.session.id + for flag, value in [ + ("--model", "claude-test"), + ("--provider", "anthropic"), + ("--thinking", "high"), + ("--append-system-prompt", "Be brief."), + ("--exclude-tools", "write"), + ]: + assert argv[argv.index(flag) + 1] == value + # No pi tools, but the bridge's (extension) tools stay. + assert "--no-builtin-tools" in argv + assert "--no-tools" not in argv + assert argv[-1] == "--no-skills" + assert kwargs["cwd"] == cwd + env = kwargs["env"] + assert env["ANTHROPIC_API_KEY"] == "k" + assert harness.session._run is not None + bridge = harness.session._run.bridge + assert json.loads(env[wire_.CONFIG_VAR]) == { + "mcp": { + "ai_sdk": { + "url": bridge.url, + "headers": {"Authorization": f"Bearer {bridge.token}"}, + "exposure": "direct", + "timeout": 86400, + } + } + } + assert server.process.terminated + + +async def test_a_pi_without_the_extension_is_refused( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.loaded = False + harness = make(cwd, agent_dir) + with pytest.raises(external.ExternalHarnessError, match="did not load"): + async with harness: + await harness.session.start() + assert server.process.terminated + + +async def test_close_is_idempotent( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + harness = make(cwd, agent_dir) + async with harness: + await harness.session.start() + await harness.session._close() # a second close does nothing + assert server.process.terminated + wire = harness.provider._make_wire(harness) + await wire.close() + await wire.close() + + +async def test_bridge_calls_are_ai_tool_calls_not_builtin( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + @ai.tool + async def lookup(word: str) -> str: + """Look a word up.""" + return word.upper() + + name = "mcp__ai_sdk__lookup" + bridge_call = { + "type": "toolCall", + "id": "toolu_b", + "name": name, + "arguments": {"word": "heron"}, + } + answers: list[dict[str, Any]] = [] + + async def call_bridge(command: dict[str, Any]) -> None: + # pi calls the MCP server the extension registered. + env = server.spawned[-1][1]["env"] + bridge = json.loads(env[wire_.CONFIG_VAR])["mcp"]["ai_sdk"] + reply = await fakes.mcp_call( + bridge["url"], + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "lookup", "arguments": {"word": "heron"}}, + }, + bridge["headers"], + ) + answers.append(reply) + text = reply["result"]["content"][0]["text"] + server.says( + { + "type": "tool_execution_end", + "toolCallId": "toolu_b", + "toolName": name, + "result": {"content": [{"type": "text", "text": text}]}, + "isError": False, + }, + *said(f"got {text}"), + *SETTLED, + ) + + tasks: list[asyncio.Task[None]] = [] + + def turn(command: dict[str, Any]) -> None: + server.says( + *begin("go"), + {"type": "message_start", "message": assistant()}, + update("toolcall_start", 0, id="toolu_b", toolName=name), + update("toolcall_end", 0, toolCall=bridge_call), + { + "type": "message_end", + "message": assistant(bridge_call, stop="toolUse"), + }, + { + "type": "tool_execution_start", + "toolCallId": "toolu_b", + "toolName": name, + "args": {"word": "heron"}, + }, + ) + server.respond(command, data={"disposition": "started"}) + tasks.append(asyncio.create_task(call_bridge(command))) + + server.on_prompt = turn + harness = make(cwd, agent_dir) + try: + async with harness: + async with asyncio.timeout(5): + async with ai.Agent(tools=[lookup]).run( + harness, [ai.user_message("go")] + ) as agent_run: + async for _ in agent_run: + pass + roles = [msg.role for msg in harness.session.messages] + assert roles == ["user", "assistant", "tool", "assistant"] + (call,) = harness.session.messages[1].tool_calls + assert call.tool_name == "lookup" + for message in harness.session.messages: + assert message.builtin_tool_calls == [] + assert message.builtin_tool_returns == [] + assert harness.session.messages[3].text == "got HERON" + assert answers[0]["result"]["isError"] is False + finally: + for task in tasks: + await task + + +async def test_a_send_during_a_run_steers_it( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [ + [ + *begin("fix"), + {"type": "message_start", "message": assistant()}, + update("text_start"), + update("text_delta", delta="Look"), + ] + ] + harness = make(cwd, agent_dir) + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("fix")) + await harness.session.send(ai.user_message("skip slow ones")) + assert len(server.sent("prompt")) == 2 + steer = server.sent("prompt")[-1] + assert steer["message"] == "skip slow ones" + assert steer["streamingBehavior"] == "steer" + # Queued, not delivered: pi did not put it in the conversation yet. + assert "skip slow ones" not in [ + msg.text for msg in harness.session.messages + ] + server.says( + update("text_end", content="Look"), + { + "type": "message_end", + "message": assistant({"type": "text", "text": "Look"}), + }, + {"type": "turn_end"}, + {"type": "turn_start"}, + {"type": "queue_update", "steering": [], "followUp": []}, + *user("skip slow ones"), + *said("OK"), + *SETTLED, + ) + async with asyncio.timeout(5): + while len(harness.session.messages) < 4: + await anext(feed) + assert [(msg.role, msg.text) for msg in harness.session.messages] == [ + ("user", "fix"), + ("assistant", "Look"), + ("user", "skip slow ones"), + ("assistant", "OK"), + ] + + +async def test_interrupt_ends_the_run_as_interrupted( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [ + [ + *begin("long job"), + {"type": "message_start", "message": assistant()}, + update("text_start"), + update("text_delta", delta="work"), + ] + ] + harness = make(cwd, agent_dir) + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("long job")) + async with asyncio.timeout(5): + async for event in feed: + if isinstance(event, ai.events.TextDelta): + break + await harness.session.interrupt() + assert len(server.sent("abort")) == 1 + assert (harness.session.messages[-1].provider_metadata or {}).get( + "external" + ) == {"stop_reason": "interrupted"} + assert harness.session.messages[-1].text == "work" + + +async def test_a_rejected_prompt_without_a_key_is_not_authenticated( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + """Measured: pi refuses the prompt itself when it has no credentials.""" + server.prompt_error = "No API key found for anthropic.\n\nUse /login" + harness = make(cwd, agent_dir) + async with harness: + with pytest.raises(external.NotAuthenticatedError): + await run(harness, [ai.user_message("go")]) + + +async def test_a_failed_run_without_a_key_is_not_authenticated( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [ + [ + *begin("go"), + {"type": "message_start", "message": assistant()}, + { + "type": "message_end", + "message": { + **assistant(stop="error"), + "errorMessage": "401 Unauthorized: invalid api key", + }, + }, + *SETTLED, + ] + ] + harness = make(cwd, agent_dir) + async with harness: + with pytest.raises(external.NotAuthenticatedError): + await run(harness, [ai.user_message("go")]) + assert "error" in ( + harness.session.messages[-1].provider_metadata or {} + ).get("external", {}) + + +async def test_a_new_session_is_seeded_with_the_past( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [[*begin("two"), *said("fine"), *SETTLED]] + history = [ + ai.user_message("one"), + m.Message( + role="assistant", + parts=[ + m.ReasoningPart(text="hmm"), + m.TextPart(text="Reading."), + m.BuiltinToolCallPart( + tool_call_id="c1", + tool_name="read", + tool_args='{"path": "a.txt"}', + ), + m.BuiltinToolReturnPart( + tool_call_id="c1", tool_name="read", result="milk" + ), + ], + ), + ai.user_message("two"), + ] + harness = make(cwd, agent_dir) + with pytest.warns(UserWarning, match="reasoning"): + async with harness: # the first call starts pi, seeded + s = await run(harness, history) + assert s.text == "fine" + session_id = harness.session.id + assert session_id is not None + assert session_id == server.session_id + path = session_file(agent_dir, session_id) + assert path.stat().st_mode & 0o777 == 0o600 + assert path.parent.stat().st_mode & 0o777 == 0o700 + assert path.parent.name.startswith("--") + entries = [json.loads(line) for line in path.read_text().splitlines()] + header = entries[0] + assert (header["type"], header["id"], header["cwd"]) == ( + "session", + session_id, + cwd, + ) + stored = [e["message"] for e in entries[1:]] + assert [msg["role"] for msg in stored] == [ + "user", + "assistant", + "toolResult", + ] + assert stored[0]["content"] == [{"type": "text", "text": "one"}] + # The reasoning was not carried; the tool call was, natively. + assert [b["type"] for b in stored[1]["content"]] == ["text", "toolCall"] + assert stored[2]["content"][0]["text"] == "milk" + # A chain, root first: pi reads it as one branch. + assert [e["parentId"] for e in entries[1:]] == [ + None, + *[e["id"] for e in entries[1:-1]], + ] + (prompt,) = server.sent("prompt") + assert prompt["message"] == "two" + + +async def test_an_existing_session_opens_here_only( + server: PiServer, cwd: str, agent_dir: Path, tmp_path: Path +) -> None: + server.turns = [[*begin("one"), *said("ok"), *SETTLED]] + first = make(cwd, agent_dir) + past = [ai.user_message("zero"), ai.assistant_message("fine")] + async with first: + await run(first, [*past, ai.user_message("one")]) + session_id = first.session.id + assert session_id is not None + assert server.process.terminated + # The seed made the session file (a real pi also writes it itself). + path = session_file(agent_dir, session_id) + + again = make(cwd, agent_dir, resume=session_id) + + async with again: + await again.session.start() + assert server.session_id == session_id + assert again.session.id == session_id + + with pytest.raises(external.ExternalHarnessError, match="no session"): + async with make(cwd, agent_dir, resume="0000-unknown") as unknown: + await unknown.session.start() + with pytest.raises( + external.ExternalHarnessError, match="not a pi session id" + ): + async with make(cwd, agent_dir, resume="../escape") as escape: + await escape.session.start() + # A session that names another directory is not this one's. + entries = path.read_text().splitlines() + header = json.loads(entries[0]) + other = path.with_name(f"x_{session_id}-moved.jsonl") + other.write_text( + json.dumps( + {**header, "id": f"{session_id}-moved", "cwd": str(tmp_path)} + ) + + "\n" + ) + with pytest.raises(external.ExternalHarnessError, match="belongs to"): + moved = make(cwd, agent_dir, resume=f"{session_id}-moved") + async with moved: + await moved.session.start() + assert len(server.spawned) == 2 + + +async def test_a_dialog_is_answered_without_a_person( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + async with make(cwd, agent_dir) as harness: + await harness.session.start() + server.says( + { + "type": "extension_ui_request", + "id": "ui-1", + "method": "confirm", + "title": "Allow?", + }, + { + "type": "extension_ui_request", + "id": "ui-2", + "method": "notify", + "message": "hello", + }, + ) + async with asyncio.timeout(5): + while not server.sent("extension_ui_response"): + await asyncio.sleep(0.01) + await asyncio.sleep(0.05) + assert server.sent("extension_ui_response") == [ + {"type": "extension_ui_response", "id": "ui-1", "cancelled": True} + ] + + +async def test_images_travel_as_base64( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [[*begin("look"), *said("a dot"), *SETTLED]] + png = m.FilePart(data=b"\x89PNG", media_type="image/png") + message = m.Message(role="user", parts=[m.TextPart(text="look"), png]) + harness = make(cwd, agent_dir) + async with harness: + await run(harness, [message]) + with pytest.raises(external.UnsupportedError): + pdf = m.FilePart(data=b"%PDF", media_type="application/pdf") + await harness.session.send(m.Message(role="user", parts=[pdf])) + (prompt,) = server.sent("prompt") + assert prompt["images"] == [ + {"type": "image", "data": "iVBORw==", "mimeType": "image/png"} + ] + + +async def test_a_command_that_starts_nothing_ends_the_turn_empty( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + def turn(command: dict[str, Any]) -> None: + server.respond(command, data={"disposition": "handled"}) + + server.on_prompt = turn + harness = make(cwd, agent_dir) + async with harness: + s = await run(harness, [ai.user_message("/noop")]) + assert s.message.parts == [] + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + ] + + +async def test_a_crash_reaches_the_stream( + server: PiServer, cwd: str, agent_dir: Path +) -> None: + server.turns = [ + [ + *begin("go"), + {"type": "message_start", "message": assistant()}, + update("text_start"), + update("text_delta", delta="half"), + ] + ] + + async def die() -> None: + await asyncio.sleep(0.05) + await server.process.terminate() + + harness = make(cwd, agent_dir) + + async with harness: + task = asyncio.create_task(die()) + with pytest.raises( + external.ExternalHarnessError, match=r"crashed|stopped" + ): + await run(harness, [ai.user_message("go")]) + await task diff --git a/tests/external/test_bridge.py b/tests/external/test_bridge.py new file mode 100644 index 00000000..496b6dab --- /dev/null +++ b/tests/external/test_bridge.py @@ -0,0 +1,38 @@ +import urllib.parse +from typing import Any + +import httpx2 + +from ai.external import _bridge + + +async def test_bridge_requires_auth_and_rejects_browser_origins() -> None: + async def on_call(name: str, arguments: dict[str, Any]) -> tuple[Any, bool]: + raise AssertionError("ping must not invoke tools") + + bridge = _bridge.Bridge(on_call) + other = _bridge.Bridge(on_call) + await bridge.start() + try: + authorization = {"Authorization": f"Bearer {bridge.token}"} + payload = {"jsonrpc": "2.0", "id": 1, "method": "ping"} + authority = urllib.parse.urlsplit(bridge.url).netloc + async with httpx2.AsyncClient(trust_env=False) as client: + for headers, status in ( + ({}, 401), + ({"Authorization": f"Bearer {other.token}"}, 401), + ({**authorization, "Host": "example.test"}, 403), + ({**authorization, "Origin": f"http://{authority}"}, 403), + ({**authorization, "Origin": "null"}, 403), + ): + response = await client.post( + bridge.url, json=payload, headers=headers + ) + assert response.status_code == status + assert not response.content + response = await client.post( + bridge.url, json=payload, headers=authorization + ) + assert response.json() == {"jsonrpc": "2.0", "id": 1, "result": {}} + finally: + await bridge.close() diff --git a/tests/external/test_handoff.py b/tests/external/test_handoff.py new file mode 100644 index 00000000..9af88e50 --- /dev/null +++ b/tests/external/test_handoff.py @@ -0,0 +1,852 @@ +"""The seeding rule, pinned without a CLI. + +Text verbatim; tool calls as native items with the ORIGINAL name and arguments; +tool results as strings bound to their call; reasoning as a user-role note after +the turn it belonged to (codex), or not at all (claude, fx, pi). An external +harness holds a tool an `ai.Agent` ran as a call and a `tool` message, and a +tool the CLI ran as built-in parts in one message: each renderer takes both, and +both seed the same. +""" + +from __future__ import annotations + +import base64 +import datetime +import hashlib +import json +import os +import stat +from pathlib import Path +from typing import Any + +import pydantic +import pytest + +from ai.external import _utils +from ai.external.claude_code import handoff as claude_handoff +from ai.external.codex import handoff as codex_handoff +from ai.external.fx import handoff as fx_handoff +from ai.external.pi import handoff as pi_handoff +from ai.types import messages as m +from ai.types.messages import ( + BuiltinToolCallPart, + BuiltinToolReturnPart, + Message, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, +) + +# Write as a tool an `ai.Agent` ran: its result in a `tool` message. +AGENT_TOOL = [ + Message(role="user", parts=[TextPart(text="Create recipe.txt")]), + Message( + role="assistant", + parts=[ + ReasoningPart(text="The user wants a file; Write is the tool."), + TextPart(text="Creating it."), + ToolCallPart( + tool_call_id="call_1", + tool_name="Write", + tool_args=json.dumps( + {"file_path": "/w/recipe.txt", "content": "two eggs"} + ), + ), + ], + ), + Message( + role="tool", + parts=[ + ToolResultPart( + tool_call_id="call_1", + tool_name="Write", + result={"ok": True}, + result_kind="json", + ) + ], + ), + Message(role="assistant", parts=[TextPart(text="Done.")]), +] + +# The same conversation, Write as a tool the CLI ran: built-in parts. +CLI_TOOL = [ + AGENT_TOOL[0], + Message( + role="assistant", + parts=[ + ReasoningPart(text="The user wants a file; Write is the tool."), + TextPart(text="Creating it."), + BuiltinToolCallPart( + tool_call_id="call_1", + tool_name="Write", + tool_args=json.dumps( + {"file_path": "/w/recipe.txt", "content": "two eggs"} + ), + ), + BuiltinToolReturnPart( + tool_call_id="call_1", tool_name="Write", result={"ok": True} + ), + ], + ), + AGENT_TOOL[3], +] + + +def test_codex_items_follow_the_rule() -> None: + items = codex_handoff.to_codex_items(AGENT_TOOL) + kinds = [(i["type"], i.get("role")) for i in items] + assert kinds == [ + ("message", "user"), + ("message", "assistant"), # "Creating it." + ("function_call", None), # Write, name untouched + ("message", "user"), # the reasoning note, after its turn + ("function_call_output", None), + ("message", "assistant"), # "Done." + ] + call = items[2] + assert call["name"] == "Write" + assert json.loads(call["arguments"])["content"] == "two eggs" + assert call["call_id"] == "call_1" + out = items[4] + assert out["call_id"] == "call_1" + assert out["output"] == json.dumps({"ok": True}) + note = items[3]["content"][0]["text"] + assert note.startswith(codex_handoff.NOTE_PREFIX) + assert "Write is the tool" in note + + +async def test_both_shapes_seed_the_same() -> None: + assert codex_handoff.to_codex_items(CLI_TOOL) == ( + codex_handoff.to_codex_items(AGENT_TOOL) + ) + assert pi_handoff.to_pi_messages(CLI_TOOL, timestamp=1) == ( + pi_handoff.to_pi_messages(AGENT_TOOL, timestamp=1) + ) + cli = await claude_handoff.to_claude_records(CLI_TOOL, "s", "/w") + agent = await claude_handoff.to_claude_records(AGENT_TOOL, "s", "/w") + assert [(r["type"], r["message"]) for r in cli] == [ + (r["type"], r["message"]) for r in agent + ] + + +async def test_claude_records_follow_the_rule() -> None: + recs = await claude_handoff.to_claude_records(CLI_TOOL, "sid-1", "/w") + # user prompt, assistant (text + tool_use), tool result, assistant; NO + # reasoning note: Claude's safeguards refuse replayed reasoning. + assert [r["type"] for r in recs] == [ + "user", + "assistant", + "user", + "assistant", + ] + assistant = recs[1]["message"]["content"] + assert [b["type"] for b in assistant] == ["text", "tool_use"] + assert assistant[1]["name"] == "Write" + assert assistant[1]["input"]["content"] == "two eggs" + assert assistant[1]["id"] == "call_1" + result = recs[2]["message"]["content"][0] + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "call_1" + assert _utils.count_reasoning_parts(CLI_TOOL) == 1 # what is warned about + # a real chain, in the CLI's own store shape + assert recs[0]["parentUuid"] is None + assert all( + recs[i]["parentUuid"] == recs[i - 1]["uuid"] + for i in range(1, len(recs)) + ) + assert all(r["sessionId"] == "sid-1" and r["cwd"] == "/w" for r in recs) + + +def test_pi_merges_assistant_messages_and_keeps_tool_results() -> None: + rendered = pi_handoff.to_pi_messages( + [ + Message(role="user", parts=[TextPart(text="hi")]), + Message(role="assistant", parts=[TextPart(text="one")]), + Message( + role="assistant", + parts=[ + BuiltinToolCallPart( + tool_call_id="c1", tool_name="Bash", tool_args="{}" + ), + BuiltinToolReturnPart( + tool_call_id="c1", + tool_name="Bash", + result={"ok": True}, + is_error=True, + ), + ], + ), + ], + timestamp=1, + ) + assert [r["role"] for r in rendered] == ["user", "assistant", "toolResult"] + assert [b["type"] for b in rendered[1]["content"]] == ["text", "toolCall"] + assert rendered[1]["stopReason"] == "toolUse" + assert rendered[2]["content"] == [{"type": "text", "text": '{"ok": true}'}] + assert rendered[2]["isError"] is True + + +def test_text_after_a_cli_tool_stays_after_its_result() -> None: + one = Message( + role="assistant", + parts=[ + BuiltinToolCallPart(tool_call_id="c", tool_name="Read"), + BuiltinToolReturnPart( + tool_call_id="c", tool_name="Read", result="x" + ), + TextPart(text="It says x."), + ], + ) + items = codex_handoff.to_codex_items([one]) + assert [i["type"] for i in items] == [ + "function_call", + "function_call_output", + "message", + ] + rendered = pi_handoff.to_pi_messages([one], timestamp=1) + assert [r["role"] for r in rendered] == [ + "assistant", + "toolResult", + "assistant", + ] + + +def test_reasoning_is_never_the_agents_own_thought() -> None: + """No reasoning item: measured to be ignored. It travels as a note.""" + items = codex_handoff.to_codex_items(AGENT_TOOL) + assert not any(i["type"] == "reasoning" for i in items) + + +def test_empty_parts_produce_nothing() -> None: + blank = [Message(role="assistant", parts=[TextPart(text=" ")])] + assert codex_handoff.to_codex_items(blank) == [] + + +FX_PREFS = { + "provider": "gateway", + "model": "openai/gpt-5.4-nano", + "effort": "auto", + "fast_mode": False, +} + + +def read_fx_log(home: Path, session_id: str) -> list[dict[str, Any]]: + """Check the byte frame independently of the writer's checksum code.""" + folder = home / "sessions/v2" / session_id + assert (folder / "lock").read_bytes() == b"" + assert not (home / "sessions/v2/index.jsonl").exists() + records = [] + for number, line in enumerate( + (folder / "log.jsonl").read_bytes().splitlines(keepends=True), start=1 + ): + assert line.endswith(b"}\n") + body, checksum = line.rsplit(b',"crc":"', 1) + record = json.loads(line) + assert list(record)[:4] == ["v", "seq", "ts", "kind"] + assert list(record)[-1] == "crc" + assert record["v"] == 1 and record["seq"] == number + # Bitwise reference implementation: no table from the source. + crc = 0xFFFFFFFF + for byte in body: + crc ^= byte + for _ in range(8): + crc = (crc >> 1) ^ (0x82F63B78 if crc & 1 else 0) + assert checksum == f"{crc ^ 0xFFFFFFFF:08x}".encode() + b'"}\n' + records.append(record) + return records + + +class ProjectedResult(pydantic.BaseModel): + published_on: datetime.date + hidden: str = pydantic.Field(default="do not replay", exclude=True) + + +class WrappedBundle(pydantic.BaseModel): + bundles: list[m.MessageBundle] + + +PROJECTED_BUNDLE = m.MessageBundle( + messages=( + Message( + role="tool", + parts=[ + ToolResultPart( + tool_call_id="inner", + tool_name="Read", + result="private nested result", + model_input="nested model input", + ), + ], + ), + Message(role="assistant", parts=[TextPart(text="Public reply")]), + ) +) + + +@pytest.mark.parametrize( + ("model_input", "expected"), + [ + pytest.param("public", "public", id="text"), + pytest.param({"public": True}, '{"public": true}', id="json"), + pytest.param(None, "null", id="explicit-null"), + pytest.param( + ProjectedResult(published_on=datetime.date(2026, 11, 17)), + '{"published_on": "2026-11-17"}', + id="pydantic-model", + ), + pytest.param( + m.ContentOutput( + value=[ + TextPart(text="Visible description"), + m.FilePart(data=b"not inlined", media_type="image/png"), + ] + ), + "Visible description\n[file: image/png]", + id="multipart", + ), + pytest.param( + PROJECTED_BUNDLE, + "Public reply", + id="message-bundle", + ), + pytest.param( + WrappedBundle(bundles=[PROJECTED_BUNDLE]), + '"[unsupported structured output omitted]"', + id="wrapped-message-bundle", + ), + ], +) +async def test_handoffs_use_model_input_for_tool_results( + tmp_path: Path, model_input: object, expected: str +) -> None: + raw_result = {"private": "not for the model"} + part = ToolResultPart( + tool_call_id="call_1", + tool_name="Write", + result=raw_result, + model_input=model_input, + ) + messages = [ + *AGENT_TOOL[:2], + Message(role="tool", parts=[part]), + AGENT_TOOL[-1], + ] + + claude = await claude_handoff.to_claude_records(messages, "sid", "/w") + claude_result = claude[2]["message"]["content"][0] + assert claude_result["tool_use_id"] == "call_1" + assert claude_result["content"] == expected + + codex = codex_handoff.to_codex_items(messages) + codex_result = next(i for i in codex if i["type"] == "function_call_output") + assert codex_result["call_id"] == "call_1" + assert codex_result["output"] == expected + + pi = pi_handoff.to_pi_messages(messages, timestamp=1) + pi_result = next(i for i in pi if i["role"] == "toolResult") + assert pi_result["toolCallId"] == "call_1" + assert pi_result["content"] == [{"type": "text", "text": expected}] + + session_id, _ = fx_handoff.write_session( + messages, + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + fx_result = next( + r + for r in read_fx_log(tmp_path, session_id) + if r.get("type") == "tool_result" + ) + assert fx_result["data"]["call_id"] == "call_1" + assert fx_result["data"]["preview"] == expected + (digest,) = fx_result["blobs"] + blobs = tmp_path / "sessions/v2" / session_id / "blobs" + assert (blobs / digest).read_bytes() == expected.encode() + assert part.result == raw_result + + +def test_fx_crc32c_has_the_standard_check_value() -> None: + assert fx_handoff.crc32c(b"123456789") == 0xE3069283 + assert fx_handoff.crc32c(b"") == 0 + + +def test_fx_log_matches_the_recorded_text_session(tmp_path: Path) -> None: + # These records are the first turn in the real ExampleSeed01 fixture. + instructions = ( + "Seeded session. Be brief. If asked for the pass phrase " + "of the instructions, it is ORCHID-22." + ) + session_id, turns = fx_handoff.write_session( + [ + Message( + role="user", + parts=[ + TextPart(text="My secret word is PLUMBAGO-7. Remember it.") + ], + ), + Message(role="assistant", parts=[TextPart(text="Noted.")]), + ], + fx_home=str(tmp_path), + cwd="/work/project", + prefs=FX_PREFS, + instructions=instructions, + session_id="ExampleSeed01", + now_ms=1794857000000, + ) + assert session_id == "ExampleSeed01" and turns == 1 + records = read_fx_log(tmp_path, session_id) + assert [r["kind"] for r in records] == [ + "session_created", + "set", + "set", + "set", + "set", + "turn_started", + "item", + "item", + "item", + "turn_committed", + "closed", + ] + created = { + k: v + for k, v in records[0].items() + if k not in ("v", "seq", "ts", "crc") + } + assert created == { + "kind": "session_created", + "id": session_id, + "workspace": "/work/project", + "role": "root", + "host": "acp", + } + assert [(r["key"], r["value"]) for r in records if r["kind"] == "set"] == [ + ("prefs", FX_PREFS), + ( + "permissions", + {"schema_version": 2, "next_generation": 1, "rules": []}, + ), + ("language", "und"), + ("client_prompt", instructions), + ] + assert [(r["type"], r["data"]) for r in records if r["kind"] == "item"] == [ + ( + "user", + { + "text": "My secret word is PLUMBAGO-7. Remember it.", + "images": [], + "work_id": None, + }, + ), + ( + "assistant", + { + "text": "Noted.", + "provider_replay": None, + "standalone_response": False, + }, + ), + ("turn_end", {"files": [], "turn_summary": None}), + ] + + +def test_fx_both_tool_shapes_keep_names_arguments_and_results( + tmp_path: Path, +) -> None: + rendered = [] + for index, messages in enumerate((AGENT_TOOL, CLI_TOOL)): + home = tmp_path / str(index) + session_id, turns = fx_handoff.write_session( + messages, + fx_home=str(home), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + now_ms=1000, + ) + assert turns == 1 + records = read_fx_log(home, session_id) + items = [r for r in records if r["kind"] == "item"] + assert [r["type"] for r in items] == [ + "user", + "assistant", + "tool_call", + "tool_result", + "assistant", + "turn_end", + ] + call, result = items[2]["data"], items[3]["data"] + assert call == { + "call_id": "call_1", + "tool_name": "Write", + "arguments_json": call["arguments_json"], + "argument_integrity": "valid", + "provisional_id": None, + "provider_result": None, + "final_identity": "valid", + "provenance": "fx_local", + } + assert json.loads(call["arguments_json"]) == { + "file_path": "/w/recipe.txt", + "content": "two eggs", + } + assert result["call_id"] == call["call_id"] + assert result["tool_name"] == call["tool_name"] + assert json.loads(result["preview"]) == {"ok": True} + assert result["status"] == "success" + assert set(result) == { + "call_id", + "tool_name", + "status", + "artifact_ref", + "tool_image_handle", + "output_bytes", + "stored_bytes", + "completeness", + "preview", + "provider_native", + "created_at_ms", + "permission_feedback", + "committed_file_presentation", + "command_replay_ref", + "command_replay_bytes", + "command_process_presentation", + "terminal_action_presentation", + } + assert "Write is the tool" not in json.dumps(items) + rendered.append( + [ + ( + r["type"], + { + k: v + for k, v in r["data"].items() + if k != "created_at_ms" + }, + ) + for r in items + ] + ) + assert rendered[0] == rendered[1] + + +def test_fx_long_tool_results_are_complete_in_a_blob(tmp_path: Path) -> None: + body = "é" * 3000 + messages = [ + Message(role="user", parts=[TextPart(text="read")]), + Message( + role="assistant", + parts=[ + BuiltinToolCallPart( + tool_call_id="c", tool_name="Read", tool_args="{}" + ), + BuiltinToolReturnPart( + tool_call_id="c", + tool_name="Read", + result=body, + is_error=True, + ), + TextPart(text="After the result."), + ], + ), + ] + session_id, _ = fx_handoff.write_session( + messages, + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + records = read_fx_log(tmp_path, session_id) + items = [r for r in records if r["kind"] == "item"] + assert [r["type"] for r in items] == [ + "user", + "assistant", + "tool_call", + "tool_result", + "assistant", + "turn_end", + ] + result = next(r for r in items if r["type"] == "tool_result") + digest = hashlib.sha256(body.encode()).hexdigest() + assert result["blobs"] == [digest] + assert ( + tmp_path / "sessions/v2" / session_id / "blobs" / digest + ).read_bytes() == body.encode() + data = result["data"] + assert data["artifact_ref"] == f"result-{digest}.txt" + assert data["status"] == "failure" and data["completeness"] == "complete" + assert data["stored_bytes"] == len(body.encode()) + assert 0 < len(data["preview"].encode()) <= 4096 + assert body.startswith(data["preview"]) + assert items[-2]["data"]["text"] == "After the result." + + +def test_fx_images_use_the_recorded_blob_shape(tmp_path: Path) -> None: + picture = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NsQ0AAAzCMP5/un0CNkuZ41wybXsHAAAAAAAAAAAAxR4yw/wuPL6QkAAAAABJRU5ErkJggg==" + ) + messages = [ + Message( + role="user", + parts=[ + TextPart(text="Here is a picture."), + m.FilePart(data=picture, media_type="image/png"), + ], + ) + ] + session_id, turns = fx_handoff.write_session( + messages, + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + assert turns == 1 + records = read_fx_log(tmp_path, session_id) + user = next(r for r in records if r.get("type") == "user") + digest = hashlib.sha256(picture).hexdigest() + assert ( + digest + == "becf0cf30d6978325d9ccdea315923e2dbbf7e591c2857b92b5f82e41957bb98" + ) + (image,) = user["data"]["images"] + assert set(image) == {"id", "path", "media_type", "blob"} + assert ( + image["id"] == 1 + and image["media_type"] == "image/png" + and image["blob"] == digest + ) + assert user["data"]["text"] == "Here is a picture.\n[Image #1]" + assert user["blobs"] == [digest] + assert ( + tmp_path / "sessions/v2" / session_id / "blobs" / digest + ).read_bytes() == picture + + +def test_fx_assistant_first_past_gets_a_nonempty_user(tmp_path: Path) -> None: + session_id, turns = fx_handoff.write_session( + [Message(role="assistant", parts=[TextPart(text="earlier answer")])], + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + assert turns == 1 + items = [ + r for r in read_fx_log(tmp_path, session_id) if r["kind"] == "item" + ] + assert [r["type"] for r in items] == ["user", "assistant", "turn_end"] + assert items[0]["data"]["text"].strip() + assert items[1]["data"]["text"] == "earlier answer" + + +def test_fx_consecutive_users_merge_into_one_turn(tmp_path: Path) -> None: + session_id, turns = fx_handoff.write_session( + [ + Message(role="user", parts=[TextPart(text="one")]), + Message(role="user", parts=[TextPart(text="two")]), + ], + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + assert turns == 1 + items = [ + r for r in read_fx_log(tmp_path, session_id) if r["kind"] == "item" + ] + assert [r["type"] for r in items] == ["user", "turn_end"] + assert ( + "one" in items[0]["data"]["text"] and "two" in items[0]["data"]["text"] + ) + assert items[0]["data"]["text"].index("one") < items[0]["data"][ + "text" + ].index("two") + + +def test_fx_unanswered_calls_get_a_failed_result(tmp_path: Path) -> None: + session_id, _ = fx_handoff.write_session( + [ + Message(role="user", parts=[TextPart(text="read")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + tool_call_id="missing", + tool_name="foreign_tool", + tool_args='{"x":1}', + ) + ], + ), + ], + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + items = [ + r for r in read_fx_log(tmp_path, session_id) if r["kind"] == "item" + ] + call = next(r["data"] for r in items if r["type"] == "tool_call") + result = next(r["data"] for r in items if r["type"] == "tool_result") + assert call["call_id"] == result["call_id"] == "missing" + assert call["tool_name"] == result["tool_name"] == "foreign_tool" + assert result["status"] == "failure" and result["preview"].strip() + + +def test_fx_does_not_replace_an_existing_session(tmp_path: Path) -> None: + kwargs: dict[str, Any] = { + "fx_home": str(tmp_path), + "cwd": "/w", + "prefs": FX_PREFS, + "instructions": None, + "session_id": "existing", + } + messages = [Message(role="user", parts=[TextPart(text="keep")])] + fx_handoff.write_session(messages, **kwargs) + log = tmp_path / "sessions/v2/existing/log.jsonl" + before = log.read_bytes() + with pytest.raises(FileExistsError): + fx_handoff.write_session(messages, **kwargs) + assert log.read_bytes() == before + + +def test_fx_parallel_calls_precede_their_paired_results(tmp_path: Path) -> None: + # The two-call step follows the recorded ExampleSeed01 second turn. + messages = [ + Message( + role="user", + parts=[ + TextPart( + text=( + "Look up the release code in the docs, " + "and read notes.txt." + ) + ) + ], + ), + Message( + role="assistant", + parts=[ + TextPart(text="I will search the docs."), + ToolCallPart( + tool_call_id="call_seed_1", + tool_name="search_docs", + tool_args='{"query":"release code"}', + ), + ToolCallPart( + tool_call_id="call_seed_2", + tool_name="read_file", + tool_args='{"path":"notes.txt"}', + ), + ], + ), + Message( + role="tool", + parts=[ + ToolResultPart( + tool_call_id="call_seed_1", + tool_name="search_docs", + result="The release code is ZX-4417.", + ), + ToolResultPart( + tool_call_id="call_seed_2", + tool_name="read_file", + result="notes.txt: the build colour is teal.", + ), + ], + ), + Message(role="assistant", parts=[TextPart(text="Done.")]), + Message(role="user", parts=[TextPart(text="Next turn")]), + ] + session_id, turns = fx_handoff.write_session( + messages, + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + assert turns == 2 + records = read_fx_log(tmp_path, session_id) + items = [r for r in records if r["kind"] == "item" and r["turn"] == 1] + assert [r["type"] for r in items] == [ + "user", + "assistant", + "tool_call", + "tool_call", + "tool_result", + "tool_result", + "assistant", + "turn_end", + ] + for call, result in zip(items[2:4], items[4:6], strict=True): + assert call["data"]["call_id"] == result["data"]["call_id"] + assert call["data"]["tool_name"] == result["data"]["tool_name"] + assert [ + (r["kind"], r["turn"]) + for r in records + if r["kind"] in ("turn_started", "turn_committed") + ] == [ + ("turn_started", 1), + ("turn_committed", 1), + ("turn_started", 2), + ("turn_committed", 2), + ] + + +def test_fx_new_store_directories_are_private_with_a_permissive_umask( + tmp_path: Path, +) -> None: + home = tmp_path / "new-home" + previous = os.umask(0o022) + try: + session_id, _ = fx_handoff.write_session( + [Message(role="user", parts=[TextPart(text="private")])], + fx_home=str(home), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + ) + finally: + os.umask(previous) + for directory in [ + home, + home / "sessions", + home / "sessions/v2", + home / "sessions/v2/.tmp", + home / "sessions/v2" / session_id, + ]: + assert stat.S_IMODE(directory.stat().st_mode) == 0o700, directory + folder = home / "sessions/v2" / session_id + assert stat.S_IMODE((folder / "log.jsonl").stat().st_mode) == 0o600 + assert stat.S_IMODE((folder / "lock").stat().st_mode) == 0o600 + + +def test_fx_failed_publish_removes_only_its_staging_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sibling = tmp_path / "sessions/v2/existing" + sibling.mkdir(parents=True) + (sibling / "log.jsonl").write_bytes(b"keep") + + def fail_publish(src: str, dst: str) -> None: + raise OSError("publish failed") + + monkeypatch.setattr(os, "rename", fail_publish) + with pytest.raises(OSError, match=r"^publish failed$"): + fx_handoff.write_session( + [Message(role="user", parts=[TextPart(text="private")])], + fx_home=str(tmp_path), + cwd="/w", + prefs=FX_PREFS, + instructions=None, + session_id="failed-publish", + ) + assert not (tmp_path / "sessions/v2/.tmp/failed-publish").exists() + assert not (tmp_path / "sessions/v2/failed-publish").exists() + assert (sibling / "log.jsonl").read_bytes() == b"keep" diff --git a/tests/external/test_harness.py b/tests/external/test_harness.py new file mode 100644 index 00000000..3dd4f20c --- /dev/null +++ b/tests/external/test_harness.py @@ -0,0 +1,1012 @@ +"""The external harness core against a scripted wire: calls and sessions.""" + +from __future__ import annotations + +import asyncio +import contextlib +import copy +import dataclasses +import pickle +from collections.abc import Callable +from typing import Any + +import pydantic +import pydantic_core +import pytest + +import ai +from ai import external +from ai.external import _session, _wire +from ai.types import events +from ai.types import messages as m + +from . import fakes + + +async def drain(feed: Any, until: type = events.StreamEnd) -> list[Any]: + """Read events up to and including the first `until`.""" + seen: list[Any] = [] + async for event in feed: + seen.append(event) + if isinstance(event, until): + break + return seen + + +# -- the external harness as a model ------------------------------------------ + + +async def test_a_harness_streams_like_a_model() -> None: + harness = fakes.fake(fakes.echo) + user = ai.user_message("hello") + async with harness: + async with ai.stream(harness, [user]) as s: + kinds = [type(e).__name__ async for e in s] + assert s.text == "hello" + assert kinds[0] == "StreamStart" and kinds[-1] == "StreamEnd" + # The session's state and the stream agree, ids too. + assert [msg.id for msg in harness.session.messages] == [ + user.id, + s.message.id, + ] + assert harness.session.messages[1].text == "hello" + assert harness.session.id == "s1" + + +async def test_the_next_call_sends_only_the_new_messages() -> None: + harness = fakes.fake(fakes.echo) + first = ai.user_message("one") + async with harness: + async with ai.stream(harness, [first]) as s: + async for _ in s: + pass + second = ai.user_message("two") + async with ai.stream(harness, [first, s.message, second]) as s2: + async for _ in s2: + pass + assert [msg.text for msg in harness.wire.sent] == ["one", "two"] + assert s2.text == "two" + assert len(harness.session.messages) == 4 + + +async def test_a_rewritten_history_is_refused() -> None: + harness = fakes.fake(fakes.echo) + async with harness: + async with ai.stream(harness, [ai.user_message("one")]) as s: + async for _ in s: + pass + with pytest.raises(external.TranscriptMismatchError): + async with ai.stream( + harness, [ai.user_message("other"), ai.user_message("two")] + ) as s: + async for _ in s: + pass + + +async def test_a_fresh_harness_seeds_the_session_with_the_past() -> None: + harness = fakes.fake(fakes.echo, instructions="Be brief.") + history = [ + ai.system_message("You fix tests."), + ai.user_message("one"), + ai.assistant_message("done"), + ai.user_message("two"), + ] + async with harness: + assert harness._wires == [] # the CLI starts with the first use + async with ai.stream(harness, history, tools=[_tool("a")]) as s: + async for _ in s: + pass + assert len(harness._wires) == 1 # it had the tools and the past + started = harness.wire.started + assert started is not None + assert started["instructions"] == "Be brief.\n\nYou fix tests." + assert [msg.text for msg in started["seed"]] == ["one", "done"] + assert [msg.text for msg in harness.wire.sent] == ["two"] + assert [msg.id for msg in harness.session.messages[:4]] == [ + x.id for x in history + ] + + +async def test_a_resumed_session_does_not_resend_the_past() -> None: + harness = fakes.fake(fakes.echo, resume="old") + history = [ + ai.user_message("one"), + ai.assistant_message("done"), + ai.user_message("two"), + ] + async with harness: + async with ai.stream(harness, history) as s: + async for _ in s: + pass + assert harness.wire.started is not None + assert harness.wire.started["seed"] == [] # the CLI has it + assert harness.wire.started["session_id"] == "old" + assert [msg.text for msg in harness.wire.sent] == ["two"] + + +async def test_a_started_empty_session_is_replaced_by_a_seeded_one() -> None: + harness = fakes.fake(fakes.echo) + history = [ai.system_message("s"), ai.user_message("x")] + async with harness: + await harness.session.start() + async with ai.stream(harness, history) as st: + async for _ in st: + pass + first, second = harness._wires + assert first.closed and first.sent == [] + assert second.started is not None + assert second.started["session_id"] is None # a new session + assert second.started["instructions"] == "s" + assert [msg.text for msg in second.sent] == ["x"] + + +async def test_new_tools_restart_an_idle_cli_on_the_same_session() -> None: + harness = fakes.fake(fakes.echo) + async with harness: + await harness.session.start() + first = ai.user_message("one") + async with ai.stream(harness, [first], tools=[_tool("a")]) as st: + async for _ in st: + pass + assert len(harness._wires) == 2 # the first CLI saw no tools + assert harness.wire.started is not None + assert harness.wire.started["session_id"] is None # it was unused + again = [first, st.message, ai.user_message("two")] + async with ai.stream(harness, again, tools=[_tool("a")]) as st: + async for _ in st: + pass + assert len(harness._wires) == 2 # same tools: no restart + more = [*harness.session.messages, ai.user_message("three")] + async with ai.stream(harness, more, tools=[_tool("b")]) as st: + async for _ in st: + pass + assert len(harness._wires) == 3 + assert harness.wire.started is not None + assert harness.wire.started["session_id"] == "s1" # a used session + + +async def test_a_started_resumed_session_takes_its_known_past() -> None: + harness = fakes.fake(fakes.echo, resume="old") + history = [ + ai.user_message("one"), + ai.assistant_message("done"), + ai.user_message("two"), + ] + async with harness: + await harness.session.start() + async with ai.stream(harness, history) as s: + async for _ in s: + pass + assert [msg.text for msg in harness.wire.sent] == ["two"] + + +async def test_a_harness_cannot_continue_tool_calls_it_did_not_make() -> None: + harness = fakes.fake(fakes.echo) + call = m.ToolCallPart(tool_call_id="c", tool_name="t", tool_args="{}") + history = [ + ai.user_message("go"), + m.Message(role="assistant", parts=[call]), + ai.tool_message(tool_call_id="c", result="r"), + ] + async with harness: + with pytest.raises(external.TranscriptMismatchError): + async with ai.stream(harness, history) as s: + async for _ in s: + pass + assert harness._wires == [] # refused before the CLI starts + + +async def test_a_model_call_needs_an_open_harness() -> None: + harness = fakes.fake(fakes.echo) + first = ai.user_message("one") + with pytest.raises( + external.ExternalHarnessError, match="open external harness" + ): + async with ai.stream(harness, [first]) as s: + async for _ in s: + pass + assert harness._wires == [] # refused before the CLI starts + async with harness: + async with ai.stream(harness, [first]) as s: + async for _ in s: + pass + assert harness.wire.closed + # A closed external harness is not an open one. + more = [*harness.session.messages, ai.user_message("two")] + with pytest.raises( + external.ExternalHarnessError, match="open external harness" + ): + async with ai.stream(harness, more) as s: + async for _ in s: + pass + assert len(harness._wires) == 1 + + +async def test_a_saved_session_id_continues_the_session() -> None: + harness = fakes.fake(fakes.echo) + first = ai.user_message("one") + async with harness: + async with ai.stream(harness, [first]) as s: + async for _ in s: + pass + assert harness.session.id == "s1" + # The id is plain data: a new external harness continues with it. + again = fakes.fake(fakes.echo, resume=harness.session.id) + async with again: + async with ai.stream( + again, [first, s.message, ai.user_message("two")] + ) as s: + async for _ in s: + pass + assert s.text == "two" + assert again.wire.started is not None + assert again.wire.started["session_id"] == "s1" + assert again.wire.started["seed"] == [] + assert [msg.text for msg in again.wire.sent] == ["two"] + assert again.wire.closed + # An external harness that opens again continues its session. + async with harness: + assert harness.session.id == "s1" + more = [*harness.session.messages, ai.user_message("other")] + async with ai.stream(harness, more) as s: + async for _ in s: + pass + assert s.text == "other" + assert harness.wire.started is not None + assert harness.wire.started["session_id"] == "s1" + + +async def test_in_a_block_the_model_calls_use_the_live_cli() -> None: + harness = fakes.fake(fakes.echo, resume="abc") + first = ai.user_message("one") + async with harness: + assert harness.session.id == "abc" + async with ai.stream(harness, [first]) as s: + async for _ in s: + pass + more = [first, s.message, ai.user_message("two")] + async with ai.stream(harness, more) as s: + async for _ in s: + pass + assert len(harness._wires) == 1 and not harness.wire.closed + assert harness.wire.started is not None + assert harness.wire.started["session_id"] == "abc" + assert [msg.text for msg in harness.wire.sent] == ["one", "two"] + # A copy is another external harness, and it is closed. + twin = copy.copy(harness) + with pytest.raises( + external.ExternalHarnessError, match="open external harness" + ): + async with ai.stream(twin, [ai.user_message("x")]) as s: + async for _ in s: + pass + assert harness.wire.closed + + +async def test_two_conversations_are_two_harnesses() -> None: + async def chat(text: str) -> tuple[str, fakes.FakeHarness]: + async with fakes.fake(fakes.echo) as harness: + async with ai.stream(harness, [ai.user_message(text)]) as s: + async for _ in s: + pass + return s.text, harness + + (one, a), (two, b) = await asyncio.gather(chat("one"), chat("two")) + assert (one, two) == ("one", "two") + assert len(a._wires) == len(b._wires) == 1 # each had its own live CLI + assert [msg.text for msg in a.session.messages] == ["one", "one"] + assert [msg.text for msg in b.session.messages] == ["two", "two"] + + +async def test_the_cli_stays_for_a_tool_loop() -> None: + @ai.tool + async def lookup(word: str) -> str: + """Look a word up.""" + return word.upper() + + def reply(msg: m.Message) -> list[Any]: + return [ + _wire.Accepted(msg), + fakes.CallTool("lookup", {"word": "heron"}), + ] + + harness = fakes.fake(reply) + async with harness: + async with ai.Agent(tools=[lookup]).run( + harness, [ai.user_message("go")] + ) as run: + async for _ in run: + pass + assert [msg.role for msg in run.messages] == [ + "user", + "assistant", + "tool", + "assistant", + ] + assert run.messages[3].text == "got HERON" + assert len(harness._wires) == 1 # one CLI for the whole turn + assert not harness.wire.closed + assert harness.wire.closed # it stops when the block ends + + +async def test_output_type_is_not_supported_yet() -> None: + class Out(pydantic.BaseModel): + answer: str + + harness = fakes.fake(fakes.echo) + with pytest.raises(external.UnsupportedError): + async with ai.stream( + harness, [ai.user_message("x")], output_type=Out + ) as s: + async for _ in s: + pass + + +async def test_cli_tools_are_builtin_parts() -> None: + def reply(msg: m.Message) -> list[Any]: + return [ + _wire.Accepted(msg), + *fakes.builtin( + "b1", "commandExecution", '{"command": "ls"}', "a\nb" + ), + *fakes.text("two files"), + _wire.TurnEnd(), + ] + + harness = fakes.fake(reply) + async with harness: + async with ai.stream(harness, [ai.user_message("ls")]) as s: + async for _ in s: + pass + (call,) = s.message.builtin_tool_calls + (result,) = s.message.builtin_tool_returns + assert call.tool_name == "commandExecution" + assert call.tool_args == '{"command": "ls"}' + assert result.result == "a\nb" + assert s.message.text == "two files" + + +async def test_turn_metadata_goes_on_the_last_reply() -> None: + def reply(msg: m.Message) -> list[Any]: + return [ + _wire.Accepted(msg), + *fakes.text("hi"), + _wire.TurnEnd(metadata={"fake": {"model": "m1"}}), + ] + + harness = fakes.fake(reply) + async with harness: + async with ai.stream(harness, [ai.user_message("x")]) as s: + async for _ in s: + pass + assert s.message.provider_metadata == {"fake": {"model": "m1"}} + assert harness.session.messages[-1].provider_metadata == { + "fake": {"model": "m1"} + } + + +async def test_a_failed_turn_raises_from_stream() -> None: + error = external.NotAuthenticatedError("fake", "401", "log in") + + def reply(msg: m.Message) -> list[Any]: + return [_wire.Accepted(msg), _wire.TurnEnd(reason="error", error=error)] + + harness = fakes.fake(reply) + history = [ai.user_message("x")] + async with harness: + with pytest.raises(external.NotAuthenticatedError): + async with ai.stream(harness, history) as s: + async for _ in s: + pass + # The state keeps the failed message, marked. + assert harness.session.messages[-1].provider_metadata == { + "external": {"error": str(error)} + } + # The caller does not have it; the next call still works. + harness.wire.reply = fakes.echo + async with ai.stream( + harness, [*history, ai.user_message("again")] + ) as s: + async for _ in s: + pass + assert s.text == "again" + # Nothing new is refused in an open external harness. + with pytest.raises( + external.TranscriptMismatchError, match="nothing new" + ): + async with ai.stream(harness, list(harness.session.messages)) as s: + async for _ in s: + pass + + +async def test_one_stream_at_a_time() -> None: + harness = fakes.fake() # accepts, never answers + async with harness: + first = harness.provider.stream(harness, [ai.user_message("one")]) + task = asyncio.create_task(anext(first)) + await asyncio.sleep(0.01) + with pytest.raises(external.ExternalHarnessError, match="one stream"): + await anext( + harness.provider.stream(harness, [ai.user_message("two")]) + ) + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + await first.aclose() + + +# -- ai tools through the bridge ---------------------------------------------- + + +async def test_an_agent_runs_ai_tools_for_the_cli() -> None: + @ai.tool + async def lookup(word: str) -> str: + """Look a word up.""" + return word.upper() + + def reply(msg: m.Message) -> list[Any]: + return [ + _wire.Accepted(msg), + fakes.CallTool("lookup", {"word": "heron"}), + ] + + harness = fakes.fake(reply) + async with harness: + async with ai.Agent(tools=[lookup]).run( + harness, [ai.user_message("go")] + ) as run: + async for _ in run: + pass + roles = [msg.role for msg in harness.session.messages] + assert roles == ["user", "assistant", "tool", "assistant"] + (call,) = harness.session.messages[1].tool_calls + assert call.tool_name == "lookup" + assert harness.session.messages[1].provider_metadata is None + assert harness.session.messages[3].text == "got HERON" + # The CLI got the result through MCP. + assert await harness.wire.calls[0] == ("HERON", False) + assert [msg.id for msg in run.messages] == [ + msg.id for msg in harness.session.messages + ] + assert harness.tools[0].name == "lookup" + + +async def test_parallel_calls_are_one_message() -> None: + def reply(msg: m.Message) -> list[Any]: + return [ + _wire.Accepted(msg), + fakes.CallTool("a", then=lambda t, e: []), + fakes.CallTool( + "b", then=lambda t, e: [*fakes.text("both"), _wire.TurnEnd()] + ), + ] + + harness = fakes.fake(reply, tools=[_tool("a"), _tool("b")]) + async with harness: + async with ai.stream(harness, [ai.user_message("go")]) as s: + async for _ in s: + pass + assert s.message.tool_calls and len(s.message.tool_calls) == 2 + assert s.message.provider_metadata is None + results = ai.tool_message( + *[ + m.ToolResultPart( + tool_call_id=c.tool_call_id, + tool_name=c.tool_name, + result="ok", + ) + for c in s.message.tool_calls + ] + ) + async with ai.stream( + harness, [*harness.session.messages, results] + ) as s2: + async for _ in s2: + pass + assert s2.text == "both" + + +async def test_cli_work_during_a_tool_call_comes_after_the_result() -> None: + harness = fakes.fake( + lambda msg: [ + _wire.Accepted(msg), + fakes.CallTool("a", then=lambda t, e: [_wire.TurnEnd()]), + ], + tools=[_tool("a")], + ) + async with harness: + async with ai.stream(harness, [ai.user_message("go")]) as s: + async for _ in s: + pass + # The CLI keeps working on its own tools while it waits. + harness.wire.push(*fakes.builtin("b1", "read")) + await asyncio.sleep(0.05) + assert [msg.role for msg in harness.session.messages] == [ + "user", + "assistant", + ] + (call,) = s.message.tool_calls + await harness.session.send( + ai.tool_message( + tool_call_id=call.tool_call_id, tool_name="a", result="r" + ) + ) + await asyncio.sleep(0.05) + roles = [msg.role for msg in harness.session.messages] + assert roles == ["user", "assistant", "tool", "assistant"] + assert ( + harness.session.messages[3].builtin_tool_calls[0].tool_name + == "read" + ) + + +async def test_a_new_message_fails_waiting_calls_with_a_tool_message() -> None: + harness = fakes.fake( + lambda msg: [ + _wire.Accepted(msg), + fakes.CallTool("a", then=lambda t, e: []), + ] + if msg.text == "go" + else fakes.echo(msg), + tools=[_tool("a")], + ) + async with harness: + async with ai.stream(harness, [ai.user_message("go")]) as s: + async for _ in s: + pass + # An agent was cancelled while its tool ran; the next run comes. + async with ai.stream( + harness, [*harness.session.messages, ai.user_message("next")] + ) as s2: + async for _ in s2: + pass + roles = [msg.role for msg in harness.session.messages] + assert roles == ["user", "assistant", "tool", "user", "assistant"] + synthetic = harness.session.messages[2] + assert synthetic.provider_metadata == {"external": {"synthetic": True}} + assert synthetic.tool_results[0].result_kind == "error" + assert await harness.wire.calls[0] == (_session.NEW_MESSAGE, True) + + +async def test_a_result_for_no_call_is_refused() -> None: + harness = fakes.fake() + async with harness: + with pytest.raises(ValueError, match="no tool call waits"): + await harness.session.send( + ai.tool_message(tool_call_id="x", result="r") + ) + + +# -- the session: direct control ---------------------------------------------- + + +async def test_events_after_send_lose_nothing() -> None: + harness = fakes.fake(fakes.echo) + async with harness: + await harness.session.send(ai.user_message("hi")) + await asyncio.sleep(0.01) # the turn ends before anyone reads + seen = await drain(harness.session.events()) + assert isinstance(seen[0], external.MessageEvent) + assert seen[0].message.text == "hi" + assert isinstance(seen[1], events.StreamStart) + # A reader that stopped needs no cleanup: the next one continues. + rest = await drain( + harness.session.events(), until=external.MessageEvent + ) + assert [type(e) for e in rest] == [external.MessageEvent] + assert rest[0].message.role == "assistant" + + +async def test_events_that_nobody_reads_do_not_pile_up() -> None: + harness = fakes.fake(fakes.echo) + async with harness: + history: list[m.Message] = [] + sizes = [] + for text in ("one", "two", "three"): + history = [*harness.session.messages, ai.user_message(text)] + async with ai.stream(harness, history) as s: + async for _ in s: + pass + sizes.append(len(harness.session._pending_events)) + assert sizes[0] == sizes[1] == sizes[2] # the latest turn only + # A reader that reads keeps everything. + seen = await drain(harness.session.events()) + await harness.session.send(ai.user_message("four")) + rest = await drain( + harness.session.events(), until=external.MessageEvent + ) + assert isinstance(seen[0], external.MessageEvent) + assert rest[0].message.text == "three" # the reply before "four" + + +async def test_a_steer_splits_the_reply_at_a_clean_point() -> None: + harness = fakes.fake() + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("fix the tests")) + harness.wire.push( + events.TextStart(block_id="t1"), + events.TextDelta(chunk="Look", block_id="t1"), + ) + steer = ai.user_message("skip the slow ones") + await harness.session.send(steer) + # The text block is open: the split waits for its end. + harness.wire.push( + events.TextDelta(chunk="ing.", block_id="t1"), + events.TextEnd(block_id="t1"), + ) + harness.wire.push(*fakes.text("Skipped.", block="t2"), _wire.TurnEnd()) + await drain(feed) # the first part ends + await drain(feed) # the second part ends + texts = [(msg.role, msg.text) for msg in harness.session.messages] + assert texts == [ + ("user", "fix the tests"), + ("assistant", "Looking."), + ("user", "skip the slow ones"), + ("assistant", "Skipped."), + ] + + +async def test_interrupt_stops_the_turn_and_marks_the_reply() -> None: + harness = fakes.fake() + async with harness: + await harness.session.send(ai.user_message("long job")) + harness.wire.push(*fakes.text("Starting")) + await asyncio.sleep(0.01) + await harness.session.interrupt() + assert harness.wire.interrupts == 1 + reply = harness.session.messages[-1] + assert reply.text == "Starting" + # Not a finish reason of the vocabulary: the raw value is here. + assert reply.provider_metadata == { + "external": {"stop_reason": "interrupted"} + } + await harness.session.interrupt() # no turn: nothing happens + assert harness.wire.interrupts == 1 + + +async def test_an_ignored_interrupt_stops_the_cli( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_session, "INTERRUPT_TIMEOUT", 0.05) + harness = fakes.fake() + async with harness: + await harness.session.start() + harness.wire.on_interrupt = None + feed = harness.session.events() + await harness.session.send(ai.user_message("long job")) + harness.wire.push(*fakes.text("Starting")) + await asyncio.sleep(0.01) + await harness.session.interrupt() + assert harness.session._run is None + assert harness.wire.closed + assert harness.session.messages[-1].provider_metadata == { + "external": {"stop_reason": "interrupted"} + } + assert (await drain(feed))[-1].finish_reason == "other" + # The session stays open: the next use continues the CLI session. + harness._reply = fakes.echo + await harness.session.send(ai.user_message("again")) + assert len(harness._wires) == 2 + assert harness.wire.started is not None + assert harness.wire.started["session_id"] == "s1" + + +async def test_a_cancelled_stream_interrupts_the_turn() -> None: + harness = fakes.fake(lambda msg: [_wire.Accepted(msg), *fakes.text("work")]) + history = [ai.user_message("long job")] + async with harness: + async with ai.stream(harness, history) as s: + async for event in s: + if isinstance(event, events.TextEnd): + break # stop reading: the turn must stop too + assert harness.wire.interrupts == 1 + assert harness.session.messages[-1].provider_metadata == { + "external": {"stop_reason": "interrupted"} + } + # ai.Agent keeps no cancelled message; the next call still works. + harness.wire.reply = fakes.echo + async with ai.stream( + harness, [*history, ai.user_message("again")] + ) as s: + async for _ in s: + pass + assert s.text == "again" + + +async def test_a_cancelled_task_interrupts_the_turn() -> None: + harness = fakes.fake(lambda msg: [_wire.Accepted(msg), *fakes.text("work")]) + + async def consume() -> None: + async with ai.stream(harness, [ai.user_message("long job")]) as s: + async for _ in s: + pass + + async with harness: + task = asyncio.create_task(consume()) + await asyncio.sleep(0.02) + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + assert harness.wire.interrupts == 1 + assert harness.session.messages[-1].provider_metadata == { + "external": {"stop_reason": "interrupted"} + } + + +async def test_a_crash_reaches_the_reader_and_start_recovers() -> None: + harness = fakes.fake() + async with harness: + feed = harness.session.events() + await harness.session.send(ai.user_message("go")) + harness.wire.push(*fakes.text("half"), RuntimeError("boom")) + with pytest.raises(external.ExternalHarnessError, match="boom"): + await drain(feed, until=type(None)) + with pytest.raises(external.ExternalHarnessError, match="stopped"): + await harness.session.send(ai.user_message("again")) + assert harness.session.messages[-1].provider_metadata is not None + await harness.session.start() + assert len(harness._wires) == 2 + + +async def test_close_ends_readers_and_keeps_the_state() -> None: + harness = fakes.fake(fakes.echo) + session = harness.session + async with harness: + feed = session.events() + await session.send(ai.user_message("hi")) + await drain(feed) + # The reply's MessageEvent follows its StreamEnd; then the feed ends. + rest = [e async for e in feed] + assert [type(e) for e in rest] == [external.MessageEvent] + assert harness.wire.closed + assert session.id == "s1" + assert [msg.text for msg in session.messages] == ["hi", "hi"] + with pytest.raises(external.ExternalHarnessError, match="not open"): + await session.send(ai.user_message("x")) + with pytest.raises(external.ExternalHarnessError, match="not open"): + await anext(session.events()) + + +async def test_start_needs_an_open_harness() -> None: + harness = fakes.fake(fakes.echo) + session = harness.session + with pytest.raises(external.ExternalHarnessError, match="not open"): + await session.start() # the block is the only way to open + assert harness._wires == [] + async with harness: + assert harness._wires == [] + await session.start() # the CLI starts now, not with the first use + assert len(harness._wires) == 1 + await session.send(ai.user_message("hi")) + await drain(session.events()) + assert harness.wire.closed + async with harness: # open again: the same CLI session + await session.start() + assert harness.wire.started is not None + assert harness.wire.started["session_id"] == "s1" + + +async def test_a_session_is_open_once() -> None: + one = fakes.fake(fakes.echo, resume="abc") + two = fakes.fake(fakes.echo, resume="abc") + async with one: + # Two external harnesses, one CLI session: two writers. + with pytest.raises(external.ExternalHarnessError, match="open already"): + async with two: + pass + async with two: # the first one closed + await two.session.send(ai.user_message("hi")) + + +async def test_a_harness_does_not_open_twice() -> None: + harness = fakes.fake(fakes.echo) + async with harness: + with pytest.raises(external.ExternalHarnessError, match="open already"): + async with harness: + pass + # It is still open. + async with ai.stream(harness, [ai.user_message("hi")]) as s: + async for _ in s: + pass + assert [msg.text for msg in harness.session.messages] == ["hi", "hi"] + async with harness: # it closed: it opens again + pass + + +async def test_the_feed_stays_when_new_tools_restart_the_cli() -> None: + harness = fakes.fake(fakes.echo) + async with harness: + await harness.session.start() + feed = harness.session.events() + async with ai.stream( + harness, [ai.user_message("one")], tools=[_tool("a")] + ) as st: + async for _ in st: + pass + assert len(harness._wires) == 2 + seen = await drain(feed) + assert isinstance(seen[0], external.MessageEvent) + assert seen[0].message.text == "one" + + +# -- state --------------------------------------------------------------------- + + +async def test_a_harness_is_settings_and_a_session() -> None: + harness = external.Codex( + cwd="/w", + model_name="gpt-5.5", + sandbox="read-only", + provider=external.CodexProvider( + executable="/bin/codex", env={"K": "v"} + ), + ) + # The fields are the settings: the session has the id and the + # conversation, and no field keeps `resume` or `history`. + fields = {field.name for field in dataclasses.fields(harness)} + assert not {"resume", "history", "session_id", "messages"} & fields + assert harness.id == "gpt-5.5" + assert harness.session.harness is harness + assert harness.session.id is None + assert harness.session.messages == [] + history = [ai.user_message("one")] + resumed = external.Codex(cwd="/w", resume="abc", history=history) + assert resumed.session.id == "abc" + assert [msg.id for msg in resumed.session.messages] == [history[0].id] + with pytest.raises(TypeError): + external.ExternalHarness(provider=external.CodexProvider()) + with pytest.raises(TypeError): + external.Codex(cwdd="/w") # type: ignore[call-arg] # ty: ignore[unknown-argument] + + +@pytest.mark.parametrize( + "harness_class", + [external.ClaudeCode, external.Codex, external.Fx, external.Pi], +) +def test_a_copy_or_a_pickle_has_the_settings_and_the_conversation( + harness_class: Callable[..., external.ExternalHarness], +) -> None: + history = [ai.user_message("one")] + harness = harness_class( + cwd="/w", model_name="m", resume="abc", history=history + ) + for again in ( + copy.copy(harness), + copy.deepcopy(harness), + pickle.loads(pickle.dumps(harness)), + ): + assert type(again) is harness_class + assert (again.cwd, again.model_name) == ("/w", "m") + assert again.provider == harness.provider + assert again.session is not harness.session + assert again.session.harness is again + assert again.session.id == "abc" + assert [msg.id for msg in again.session.messages] == [history[0].id] + + +async def test_an_open_harness_copies_and_pickles_as_a_closed_one() -> None: + async with external.Codex(cwd="/w") as harness: # no CLI starts + for again in (copy.copy(harness), pickle.loads(pickle.dumps(harness))): + with pytest.raises(external.ExternalHarnessError, match="not open"): + await again.session.start() + + +async def test_a_seeded_session_starts_with_its_past() -> None: + history = [ai.user_message("one"), ai.assistant_message("done")] + harness = fakes.fake(fakes.echo, history=history) + async with harness: + await harness.session.send(ai.user_message("two")) + await drain(harness.session.events()) + assert harness.wire.started is not None + assert harness.wire.started["session_id"] is None + assert [msg.text for msg in harness.wire.started["seed"]] == [ + "one", + "done", + ] + assert len(history) == 2 # the caller's list stays as it was + assert harness.session.id == "s1" + assert [msg.id for msg in harness.session.messages[:2]] == [ + msg.id for msg in history + ] + + +async def test_an_agent_context_keeps_the_harness() -> None: + harness = external.Codex(cwd="/w") + context = ai.Context(model=harness, messages=[], tools=[]) + assert context.model is harness + # An external harness is not plain data: a context with one does not + # dump. + with pytest.raises(pydantic_core.PydanticSerializationError): + context.model_dump(mode="json") + + +# -- telemetry ---------------------------------------------------------------- + + +async def test_stream_reply_is_in_history_while_telemetry_is_pending() -> None: + telemetry_started = asyncio.Event() + release_telemetry = asyncio.Event() + + class BlockingSink: + async def on_push(self, span: ai.experimental_telemetry.Span) -> None: + if ( + isinstance(span.data, external.ExternalHarnessMessageSpanData) + and span.ended_at is not None + ): + telemetry_started.set() + await release_telemetry.wait() + + harness = fakes.fake(fakes.echo) + user = ai.user_message("one") + + async def reply() -> m.Message: + async with ai.stream(harness, [user]) as stream: + async for _ in stream: + pass + return stream.message + + async with ai.experimental_telemetry.use_sink(BlockingSink()), harness: + task = asyncio.create_task(reply()) + try: + async with asyncio.timeout(5): + message = await asyncio.shield(task) + assert telemetry_started.is_set() + assert [msg.id for msg in harness.session.messages] == [ + user.id, + message.id, + ] + finally: + release_telemetry.set() + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +async def test_sessions_messages_and_cli_tools_have_spans() -> None: + def reply(msg: m.Message) -> list[Any]: + return [ + _wire.Accepted(msg), + *fakes.builtin("b1", "read", '{"path": "x"}', "data"), + _wire.TurnEnd(), + ] + + sink = ai.experimental_telemetry.DictSink() + async with ai.experimental_telemetry.use_sink(sink): + harness = fakes.fake(reply) + async with harness: + async with ai.stream(harness, [ai.user_message("go")]) as s: + async for _ in s: + pass + by_kind = {span.data.kind: span for span in sink.finished_spans} + assert { + "external_session", + "external_message", + "tool_execution", + "ai_stream", + } <= set(by_kind) + session, message, tool = ( + by_kind["external_session"], + by_kind["external_message"], + by_kind["tool_execution"], + ) + assert isinstance(session.data, external.ExternalHarnessSessionSpanData) + assert isinstance(message.data, external.ExternalHarnessMessageSpanData) + assert isinstance( + tool.data, ai.experimental_telemetry.ToolExecutionSpanData + ) + assert session.data.session_id == "s1" + assert message.parent_id == session.id + assert message.data.finish_reason == "stop" + assert tool.parent_id == message.id + assert tool.data.args == {"path": "x"} + assert tool.data.result == "data" + + +def _tool(name: str) -> ai.tools.Tool: + return ai.tools.Tool( + kind="function", + name=name, + spec=ai.tools.ToolSpec(params={"type": "object"}), + ) diff --git a/tests/external/test_jsonrpc.py b/tests/external/test_jsonrpc.py new file mode 100644 index 00000000..ba1bdfcc --- /dev/null +++ b/tests/external/test_jsonrpc.py @@ -0,0 +1,176 @@ +"""The JSON-RPC peer for CLI external harnesses, offline.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any, cast + +import pytest + +from ai import external +from ai.external import _jsonrpc as jsonrpc + +from . import fakes + + +def _peer( + process: fakes.FakeProcess, + on_request: Any = None, + on_notify: Any = None, +) -> jsonrpc.JsonRpcPeer: + async def no_notify(method: str, params: dict[str, Any]) -> None: + pass + + async def no_request( + request_id: Any, method: str, params: dict[str, Any] + ) -> None: + pass + + peer = jsonrpc.JsonRpcPeer(cast("Any", process), kind="test") + peer.start(on_notify or no_notify, on_request or no_request) + return peer + + +async def test_a_call_gets_its_answer() -> None: + process = fakes.FakeProcess(lambda message: {"echo": message["method"]}) + peer = _peer(process) + assert await peer.call("ping", timeout=1) == {"echo": "ping"} + await peer.close() + + +async def test_a_call_made_while_answering_a_request_is_answered() -> None: + """Answering a server request must not stop the peer reading.""" + process = fakes.FakeProcess() + took: list[float] = [] + done = asyncio.Event() + peer: jsonrpc.JsonRpcPeer + + async def on_request( + request_id: Any, method: str, params: dict[str, Any] + ) -> None: + started = time.monotonic() + await peer.respond(request_id, {"decision": "decline"}) + await peer.call("turn/interrupt", {}, timeout=2) + took.append(time.monotonic() - started) + done.set() + + peer = _peer(process, on_request=on_request) + process.server_says({"id": 99, "method": "approval", "params": {}}) + await asyncio.wait_for(done.wait(), 3) + assert took[0] < 0.5 + await peer.close() + + +async def test_notifications_flow_while_a_request_is_pending() -> None: + """A slow answer (a person approving) does not freeze the stream.""" + process = fakes.FakeProcess() + release = asyncio.Event() + seen: list[str] = [] + + async def on_request( + request_id: Any, method: str, params: dict[str, Any] + ) -> None: + await release.wait() + + async def on_notify(method: str, params: dict[str, Any]) -> None: + seen.append(method) + + peer = _peer(process, on_request=on_request, on_notify=on_notify) + process.server_says({"id": 1, "method": "approval", "params": {}}) + process.server_says({"method": "item/agentMessage/delta", "params": {}}) + assert await peer.call("thread/read", timeout=1) == {} + for _ in range(50): + if seen: + break + await asyncio.sleep(0.01) + assert seen == ["item/agentMessage/delta"] + release.set() + await peer.close() + + +async def test_notifications_keep_their_order() -> None: + process = fakes.FakeProcess() + seen: list[int] = [] + + async def on_notify(method: str, params: dict[str, Any]) -> None: + await asyncio.sleep(0) + seen.append(params["n"]) + + peer = _peer(process, on_notify=on_notify) + for n in range(20): + process.server_says({"method": "n", "params": {"n": n}}) + await peer.call("ping", timeout=1) + assert seen == list(range(20)) + await peer.close() + + +async def test_close_cancels_requests_still_being_answered() -> None: + process = fakes.FakeProcess() + cancelled = asyncio.Event() + started = asyncio.Event() + + async def on_request( + request_id: Any, method: str, params: dict[str, Any] + ) -> None: + started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancelled.set() + raise + + peer = _peer(process, on_request=on_request) + process.server_says({"id": 1, "method": "approval", "params": {}}) + await asyncio.wait_for(started.wait(), 1) + await peer.close() + assert cancelled.is_set() + + +async def test_a_failing_request_handler_does_not_kill_the_peer() -> None: + process = fakes.FakeProcess() + + async def on_request( + request_id: Any, method: str, params: dict[str, Any] + ) -> None: + raise ValueError("bug in the handler") + + peer = _peer(process, on_request=on_request) + process.server_says({"id": 1, "method": "approval", "params": {}}) + assert await peer.call("ping", timeout=1) == {} + await peer.close() + with pytest.raises(external.ExternalHarnessError): + await peer.call("ping", timeout=1) + + +async def test_a_dead_process_fails_its_callers() -> None: + process = fakes.FakeProcess(lambda message: None) # never answers + crashed: list[BaseException] = [] + peer = jsonrpc.JsonRpcPeer(cast("Any", process), kind="test") + + async def nothing(*args: Any) -> None: + pass + + peer.start(nothing, nothing, on_crash=crashed.append) + call = asyncio.ensure_future(peer.call("ping", timeout=2)) + await asyncio.sleep(0) + await process.terminate() # EOF + with pytest.raises(external.ExternalHarnessError, match="crashed"): + await call + assert len(crashed) == 1 + await peer.close() + + +async def test_a_failed_write_removes_its_waiter() -> None: + class BrokenProcess(fakes.FakeProcess): + async def write(self, data: str) -> None: + raise BrokenPipeError("stdin closed") + + peer = _peer(BrokenProcess()) + try: + with pytest.raises(external.ExternalHarnessError, match="stdin closed"): + await peer.call("ping", timeout=1) + assert peer._waiters == {} + assert peer._completions == {} + finally: + await peer.close() diff --git a/tests/external/test_live.py b/tests/external/test_live.py new file mode 100644 index 00000000..68f3b03d --- /dev/null +++ b/tests/external/test_live.py @@ -0,0 +1,175 @@ +"""Live tests against the real CLIs. + +Opt-in: `AI_EXTERNAL_LIVE=1 uv run pytest tests/external/test_live.py`. + +Each test skips when its CLI is not installed. They spend tokens on the +local login of each CLI, so the prompts are small. +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +import ai +from ai import external + +pytestmark = pytest.mark.skipif( + not os.environ.get("AI_EXTERNAL_LIVE"), + reason="drives real CLIs: set AI_EXTERNAL_LIVE=1 to run", +) + +KINDS: dict[str, tuple[type[external.ExternalHarness], str]] = { + "claude-code": (external.ClaudeCode, "claude"), + "codex": (external.Codex, "codex"), + "fx": (external.Fx, "fx"), + "pi": (external.Pi, "pi"), +} + + +@pytest.fixture(params=sorted(KINDS)) +def make( + request: pytest.FixtureRequest, tmp_path: Path +) -> Callable[..., external.ExternalHarness]: + harness_class, executable = KINDS[request.param] + if shutil.which(executable) is None: + pytest.skip(f"{executable} is not installed") + (tmp_path / "secret.txt").write_text("the word is HERON\n") + cwd = os.path.realpath(tmp_path) + + def build(**fields: Any) -> external.ExternalHarness: + return harness_class(cwd=cwd, **fields) + + return build + + +async def test_a_turn_with_a_cli_tool_and_a_resume( + make: Callable[..., external.ExternalHarness], +) -> None: + harness = make() + async with asyncio.timeout(240), harness: + async with ai.stream( + harness, + [ + ai.user_message( + "Read secret.txt with a tool. Reply with the word only." + ) + ], + ) as s: + async for _ in s: + pass + assert "HERON" in s.text.upper() + assert s.message.builtin_tool_calls + assert harness.session.id is not None + # The session id is all that a later process needs, with the settings. + again = make(resume=harness.session.id) + async with asyncio.timeout(240), again: + reply = await ai.experimental_generate( + again, [ai.user_message("Which word did you read? One word.")] + ) + assert "HERON" in reply.text.upper() + assert again.session.id == harness.session.id + + +async def test_an_agent_gives_its_tools_to_the_cli( + make: Callable[..., external.ExternalHarness], +) -> None: + @ai.tool + async def secret_word(topic: str) -> str: + """Return the secret word for a topic.""" + return "PELICAN" if topic == "birds" else "UNKNOWN" + + harness = make() + async with asyncio.timeout(240), harness: + async with ai.Agent(tools=[secret_word]).run( + harness, + [ + ai.user_message( + "Call the secret_word tool with topic 'birds'. " + "Reply with the word only." + ) + ], + ) as run: + async for _ in run: + pass + roles = [message.role for message in run.messages] + assert "tool" in roles + assert "PELICAN" in run.messages[-1].text.upper() + + +async def test_cancel_interrupts_and_the_session_continues( + make: Callable[..., external.ExternalHarness], +) -> None: + harness = make() + history = [ai.user_message("Write the numbers 1 to 300, one per line.")] + async with asyncio.timeout(240), harness: + async with ai.stream(harness, history) as s: + async for event in s: + if isinstance(event, ai.events.TextDelta): + break # stop reading: the turn stops + cut = harness.session.messages[-1].provider_metadata or {} + assert cut.get("external") == {"stop_reason": "interrupted"} + reply = await ai.experimental_generate( + harness, [*history, ai.user_message("Reply with the word OK only.")] + ) + assert "OK" in reply.text.upper() + + +async def test_a_steer_lands_in_the_conversation( + make: Callable[..., external.ExternalHarness], +) -> None: + harness = make() + async with asyncio.timeout(240), harness: + await harness.session.send( + ai.user_message( + "Run `sleep 5` with a shell tool, " + "then reply with the word DONE." + ) + ) + steered = False + async for event in harness.session.events(): + if not steered and isinstance(event, ai.events.BuiltinToolStart): + steered = True + await harness.session.send( + ai.user_message("After that, also say the word STEERED.") + ) + if ( + isinstance(event, ai.events.StreamEnd) + and event.finish_reason == "stop" + ): + if any( + m.role == "user" and "STEERED" in m.text + for m in harness.session.messages + ): + break + user_texts = [m.text for m in harness.session.messages if m.role == "user"] + assert any("STEERED" in t for t in user_texts) + assert ( + "STEERED" + in "".join( + m.text for m in harness.session.messages if m.role == "assistant" + ).upper() + ) + + +async def test_a_new_session_uses_the_given_past( + make: Callable[..., external.ExternalHarness], +) -> None: + harness = make() + history = [ + ai.user_message("The secret word is GENTIAN-31. Remember it."), + ai.assistant_message("Noted."), + ai.user_message( + "What is the secret word from our conversation? Reply with it only." + ), + ] + async with asyncio.timeout(240), harness: + reply = await ai.experimental_generate(harness, history) + assert "GENTIAN-31" in reply.text.upper() + assert harness.session.id is not None diff --git a/tests/models/core/test_model.py b/tests/models/core/test_model.py new file mode 100644 index 00000000..0027dc76 --- /dev/null +++ b/tests/models/core/test_model.py @@ -0,0 +1,88 @@ +"""`ModelLike`: an id and a provider go where a model goes.""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator, Sequence +from typing import Any + +import pydantic + +import ai +from ai.types import events +from ai.types import messages as m +from ai.types import tools as tools_ + + +class EchoProvider: + name = "echo" + + async def stream( + self, + model: Echo, + messages: list[m.Message], + *, + tools: Sequence[tools_.Tool] | None = None, + output_type: type[pydantic.BaseModel] | None = None, + params: Any = None, + ) -> AsyncGenerator[events.Event]: + model.calls.append(list(messages)) + text = next(x.text for x in reversed(messages) if x.role == "user") + yield events.StreamStart() + yield events.TextStart(block_id="t") + yield events.TextDelta(chunk=text, block_id="t") + yield events.TextEnd(block_id="t") + yield events.StreamEnd(finish_reason="stop") + + async def generate( + self, + model: Echo, + messages: list[m.Message], + *, + tools: Sequence[tools_.Tool] | None = None, + output_type: type[pydantic.BaseModel] | None = None, + params: Any = None, + ) -> m.Message: + raise NotImplementedError + + +class Echo: + """Not an `ai.Model`: it answers with the last user message's text.""" + + id = "echo" + + def __init__(self) -> None: + self.calls: list[list[m.Message]] = [] + self.provider = EchoProvider() + + +def test_a_model_and_a_stand_in_are_model_like() -> None: + assert isinstance(Echo(), ai.models.ModelLike) + assert isinstance(ai.get_model("openai/gpt-5"), ai.models.ModelLike) + + +async def test_stream_takes_a_model_like() -> None: + async with ai.stream(Echo(), [ai.user_message("hello")]) as s: + async for _ in s: + pass + assert s.text == "hello" + + +async def test_generate_takes_a_model_like() -> None: + message = await ai.experimental_generate(Echo(), [ai.user_message("hi")]) + assert message.text == "hi" + + +async def test_an_agent_runs_on_a_model_like() -> None: + model = Echo() + async with ai.Agent().run(model, [ai.user_message("ping")]) as run: + async for _ in run: + pass + assert [x.role for x in run.messages] == ["user", "assistant"] + assert run.messages[-1].text == "ping" + assert len(model.calls) == 1 + + +def test_a_context_keeps_a_model_like_as_it_is() -> None: + model = Echo() + context = ai.Context(model=model, messages=[], tools=[]) + assert context.model is model diff --git a/uv.lock b/uv.lock index d371b1aa..eae93425 100644 --- a/uv.lock +++ b/uv.lock @@ -25,6 +25,9 @@ dependencies = [ anthropic = [ { name = "anthropic" }, ] +claude-code = [ + { name = "claude-agent-sdk" }, +] mcp = [ { name = "mcp" }, ] @@ -46,6 +49,7 @@ vercel = [ dev = [ { name = "anthropic" }, { name = "async-solipsism" }, + { name = "claude-agent-sdk" }, { name = "mcp" }, { name = "mypy" }, { name = "openai" }, @@ -64,6 +68,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.83.0,<2.0.0" }, + { name = "claude-agent-sdk", marker = "extra == 'claude-code'", specifier = ">=0.2.179,<0.3" }, { name = "httpx2", specifier = ">=2.12.0" }, { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.0.0,<3.0.0" }, { name = "modelsdotdev", specifier = "==0.*" }, @@ -75,12 +80,13 @@ requires-dist = [ { name = "typing-extensions", specifier = ">=4.15.0" }, { name = "vercel-oidc", marker = "extra == 'vercel'", specifier = ">=0.8.0" }, ] -provides-extras = ["anthropic", "mcp", "openai", "otel", "typesafe", "vercel"] +provides-extras = ["anthropic", "claude-code", "mcp", "openai", "otel", "typesafe", "vercel"] [package.metadata.requires-dev] dev = [ { name = "anthropic", specifier = ">=0.83.0" }, { name = "async-solipsism", specifier = ">=0.9" }, + { name = "claude-agent-sdk", specifier = ">=0.2.179,<0.3" }, { name = "mcp", specifier = ">=2.0.0,<3.0.0" }, { name = "mypy", specifier = "~=2.1.0" }, { name = "openai", specifier = ">=2.14.0" }, @@ -333,6 +339,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, ] +[[package]] +name = "claude-agent-sdk" +version = "0.2.179" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "jsonschema" }, + { name = "mcp" }, + { name = "sniffio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/cf/4e6cb9c949abd667c077cd07909bfbf9b6f1c19dff326b9f6dafc716c185/claude_agent_sdk-0.2.179.tar.gz", hash = "sha256:97b29d785bb9c9e868c3d7e7458fb68ea7ceaec8c830273d6ad316da014a974d", size = 400216, upload-time = "2026-11-06T17:56:01.69Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/e4/da356f282abbdfc348864a424ef39d892d4681516150785d8402008940a2/claude_agent_sdk-0.2.179-py3-none-macosx_11_0_arm64.whl", hash = "sha256:28b40c646eb696316ca9d0531bb5f1179d1df4cd76fa630440170c91fbb788f0", size = 139333276, upload-time = "2026-11-06T17:56:09.546Z" }, + { url = "https://files.pythonhosted.org/packages/03/79/8a37f953154916f81cae43873685afa2a11296fcff2a5191998126d26709/claude_agent_sdk-0.2.179-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:ce95a42db94fa023f97a51982beeb95230d73be1e6c5babc77f2b635469c4759", size = 146623781, upload-time = "2026-11-06T17:56:20.409Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e7/5429f36dd8a83c2835b6f50e7921b7cf05680007b19f520ed32a458d237f/claude_agent_sdk-0.2.179-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:26b1fb00ac0c2a19af08071bdd20fde579410cb7dbf0d8281feaa834ba5b91db", size = 150981993, upload-time = "2026-11-06T17:56:33.426Z" }, + { url = "https://files.pythonhosted.org/packages/00/16/972f8bbe1d471bc38b93bdea1744b6931b3e6e5bc7f3f647f07c98346622/claude_agent_sdk-0.2.179-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:834159329beca83e1a1bd63815a5461b5ef5e8e039a8719fcc77df7c2a83296d", size = 151361548, upload-time = "2026-11-06T17:56:40.202Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5b/25020ae125ea7f58b14f9cbf1d0c1f46c61628c3947ccfbd8c8032946b24/claude_agent_sdk-0.2.179-py3-none-win_amd64.whl", hash = "sha256:f9ac74aec7847f2e0474d710ec5ddcc8c0e1898b1d5f54f60a96c7a39bb6d7cc", size = 153067067, upload-time = "2026-11-06T17:56:56.516Z" }, +] + [[package]] name = "click" version = "8.3.1"