diff --git a/docs/ai-python/content/docs/reference/testing.mdx b/docs/ai-python/content/docs/reference/testing.mdx index cccb038d..8da8cc56 100644 --- a/docs/ai-python/content/docs/reference/testing.mdx +++ b/docs/ai-python/content/docs/reference/testing.mdx @@ -43,6 +43,20 @@ Properties: `FakeModel` works with both `ai.stream` and `Agent.run`. +Responses are deep copies of scripted messages. Scripted `usage` is preserved in +both generated messages and completed streams. + +### Limits + +- `FakeModel` does not implement `ai.ops.*` or `ai.probe`. +- Each nonempty text part produces one text delta, not token-sized chunks. +- Usage comes from the scripted message; the fake does not count tokens. +- Matching requires a result for every tool call. It does not repair incomplete + tool histories. +- The fake does not validate provider capabilities or inference parameters. + Structured output is parsed when `Message.get_output` is called, not when the + script is played. + ## tool_call ```python diff --git a/src/ai/testing.py b/src/ai/testing.py index 4e8bf134..df6d71cc 100644 --- a/src/ai/testing.py +++ b/src/ai/testing.py @@ -247,7 +247,14 @@ def stream( params: params_.InferenceRequestParams | None = None, ) -> AsyncGenerator[events.Event]: response = self._next_response(messages) - return events._replay_message_events(response) + + async def replay() -> AsyncGenerator[events.Event]: + async for event in events._replay_message_events(response): + if isinstance(event, events.StreamEnd): + event = event.model_copy(update={"usage": response.usage}) + yield event + + return replay() async def generate( self, @@ -272,7 +279,7 @@ def _next_response( result = _walk(script, messages, fingerprints) if isinstance(result, int): script.played.add(result) - return script.messages[result] + return script.messages[result].model_copy(deep=True) failures.append((script, result)) raise AssertionError(self._no_match_error(fingerprints, failures)) diff --git a/tests/test_testing.py b/tests/test_testing.py index fb3325c1..264ffd4d 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -15,6 +15,7 @@ import ai from ai.types import events as events_ from ai.types import messages as messages_ +from ai.types import usage as usage_ @ai.tool @@ -280,3 +281,43 @@ async def test_foreign_tool_results_fail() -> None: pass assert "call_bogus" in str(err.value) + + +async def test_fake_model_stream_reports_scripted_usage() -> None: + """A completed stream reports the usage on the scripted response.""" + scripted = ai.assistant_message("hello") + scripted.usage = usage_.Usage(input_tokens=3, output_tokens=5) + model = ai.testing.FakeModel([ai.user_message("go"), scripted]) + + async with ai.stream(model, [ai.user_message("go")]) as stream: + async for _ in stream: + pass + + assert stream.usage == scripted.usage + + +async def test_fake_model_generate_does_not_hand_out_the_script() -> None: + """A generated response is equal to, but distinct from, its script.""" + scripted = ai.assistant_message("hello") + model = ai.testing.FakeModel([ai.user_message("go"), scripted]) + + message = await ai.generate(model, [ai.user_message("go")]) + + assert message == scripted + assert message is not scripted + + +def test_fake_provider_stream_matches_before_iteration() -> None: + scripted = ai.assistant_message("hello") + model = ai.testing.FakeModel([ai.user_message("go"), scripted]) + messages = [ai.user_message("go")] + + model.provider.stream(model, messages) + + assert len(model.calls) == 1 + assert model.calls[0] == messages + assert not model.unused + + with pytest.raises(AssertionError, match="no script continues"): + model.provider.stream(model, [ai.user_message("unexpected")]) + assert len(model.calls) == 2