From a118c9ebfa3538f0d42fe8c30a60eaba1d44b44d Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Mon, 16 Nov 2026 13:24:11 -0800 Subject: [PATCH 1/2] Preserve scripted usage and isolate FakeModel responses Carry scripted usage through stream completion and return deep copies of scripted responses so callers cannot mutate the fake's script through them. Keep direct provider stream selection eager. Add regression tests and document usage accounting and the limits of scripted model tests. --- .../ai-python/content/docs/basics/testing.mdx | 18 ++++++++ .../content/docs/reference/testing.mdx | 14 +++++++ src/ai/testing.py | 11 ++++- tests/test_testing.py | 41 +++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/docs/ai-python/content/docs/basics/testing.mdx b/docs/ai-python/content/docs/basics/testing.mdx index aa1cba77..1875f4f4 100644 --- a/docs/ai-python/content/docs/basics/testing.mdx +++ b/docs/ai-python/content/docs/basics/testing.mdx @@ -139,3 +139,21 @@ assert message.get_output(Weather).conditions == "sunny" The fake does not validate scripted text against the schema; parsing happens only when the test calls `get_output`. + +## Limits + +- `FakeModel` scripts model conversations for `ai.stream`, + `ai.experimental_generate`, and `Agent.run`. It does not implement `ai.ops.*` + operations or model capability probing with `ai.probe`. +- Streaming emits one text delta per nonempty scripted text part. It does not + simulate token-by-token delivery or network timing. +- To test usage accounting, set `usage` on the scripted assistant message. With + `ai.stream`, read the completed stream's `stream.usage`. With `Agent.run`, read + `usage` on each assistant message in `stream.messages`; the agent stream has no + `usage` property. Usage is not calculated from the text. +- Every scripted tool call needs a result before the next assistant message can + play, even when tool messages are omitted from the script. The fake does not + repair incomplete tool histories. +- The fake does not check whether a real provider supports the supplied tools, + schema, or inference parameters. Use separate provider integration tests for + those checks. diff --git a/docs/ai-python/content/docs/reference/testing.mdx b/docs/ai-python/content/docs/reference/testing.mdx index cccb038d..8da8cc56 100644 --- a/docs/ai-python/content/docs/reference/testing.mdx +++ b/docs/ai-python/content/docs/reference/testing.mdx @@ -43,6 +43,20 @@ Properties: `FakeModel` works with both `ai.stream` and `Agent.run`. +Responses are deep copies of scripted messages. Scripted `usage` is preserved in +both generated messages and completed streams. + +### Limits + +- `FakeModel` does not implement `ai.ops.*` or `ai.probe`. +- Each nonempty text part produces one text delta, not token-sized chunks. +- Usage comes from the scripted message; the fake does not count tokens. +- Matching requires a result for every tool call. It does not repair incomplete + tool histories. +- The fake does not validate provider capabilities or inference parameters. + Structured output is parsed when `Message.get_output` is called, not when the + script is played. + ## tool_call ```python diff --git a/src/ai/testing.py b/src/ai/testing.py index 4e8bf134..df6d71cc 100644 --- a/src/ai/testing.py +++ b/src/ai/testing.py @@ -247,7 +247,14 @@ def stream( params: params_.InferenceRequestParams | None = None, ) -> AsyncGenerator[events.Event]: response = self._next_response(messages) - return events._replay_message_events(response) + + async def replay() -> AsyncGenerator[events.Event]: + async for event in events._replay_message_events(response): + if isinstance(event, events.StreamEnd): + event = event.model_copy(update={"usage": response.usage}) + yield event + + return replay() async def generate( self, @@ -272,7 +279,7 @@ def _next_response( result = _walk(script, messages, fingerprints) if isinstance(result, int): script.played.add(result) - return script.messages[result] + return script.messages[result].model_copy(deep=True) failures.append((script, result)) raise AssertionError(self._no_match_error(fingerprints, failures)) diff --git a/tests/test_testing.py b/tests/test_testing.py index fb3325c1..264ffd4d 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -15,6 +15,7 @@ import ai from ai.types import events as events_ from ai.types import messages as messages_ +from ai.types import usage as usage_ @ai.tool @@ -280,3 +281,43 @@ async def test_foreign_tool_results_fail() -> None: pass assert "call_bogus" in str(err.value) + + +async def test_fake_model_stream_reports_scripted_usage() -> None: + """A completed stream reports the usage on the scripted response.""" + scripted = ai.assistant_message("hello") + scripted.usage = usage_.Usage(input_tokens=3, output_tokens=5) + model = ai.testing.FakeModel([ai.user_message("go"), scripted]) + + async with ai.stream(model, [ai.user_message("go")]) as stream: + async for _ in stream: + pass + + assert stream.usage == scripted.usage + + +async def test_fake_model_generate_does_not_hand_out_the_script() -> None: + """A generated response is equal to, but distinct from, its script.""" + scripted = ai.assistant_message("hello") + model = ai.testing.FakeModel([ai.user_message("go"), scripted]) + + message = await ai.generate(model, [ai.user_message("go")]) + + assert message == scripted + assert message is not scripted + + +def test_fake_provider_stream_matches_before_iteration() -> None: + scripted = ai.assistant_message("hello") + model = ai.testing.FakeModel([ai.user_message("go"), scripted]) + messages = [ai.user_message("go")] + + model.provider.stream(model, messages) + + assert len(model.calls) == 1 + assert model.calls[0] == messages + assert not model.unused + + with pytest.raises(AssertionError, match="no script continues"): + model.provider.stream(model, [ai.user_message("unexpected")]) + assert len(model.calls) == 2 From 1f46a900ab6024d8c402f107f5f4ac3dfe24e874 Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Thu, 19 Nov 2026 16:01:32 -0800 Subject: [PATCH 2/2] Address review --- docs/ai-python/content/docs/basics/testing.mdx | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/docs/ai-python/content/docs/basics/testing.mdx b/docs/ai-python/content/docs/basics/testing.mdx index 1875f4f4..aa1cba77 100644 --- a/docs/ai-python/content/docs/basics/testing.mdx +++ b/docs/ai-python/content/docs/basics/testing.mdx @@ -139,21 +139,3 @@ assert message.get_output(Weather).conditions == "sunny" The fake does not validate scripted text against the schema; parsing happens only when the test calls `get_output`. - -## Limits - -- `FakeModel` scripts model conversations for `ai.stream`, - `ai.experimental_generate`, and `Agent.run`. It does not implement `ai.ops.*` - operations or model capability probing with `ai.probe`. -- Streaming emits one text delta per nonempty scripted text part. It does not - simulate token-by-token delivery or network timing. -- To test usage accounting, set `usage` on the scripted assistant message. With - `ai.stream`, read the completed stream's `stream.usage`. With `Agent.run`, read - `usage` on each assistant message in `stream.messages`; the agent stream has no - `usage` property. Usage is not calculated from the text. -- Every scripted tool call needs a result before the next assistant message can - play, even when tool messages are omitted from the script. The fake does not - repair incomplete tool histories. -- The fake does not check whether a real provider supports the supplied tools, - schema, or inference parameters. Use separate provider integration tests for - those checks.