Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/ai-python/content/docs/reference/testing.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,20 @@ Properties:

`FakeModel` works with both `ai.stream` and `Agent.run`.

Responses are deep copies of scripted messages. Scripted `usage` is preserved in
both generated messages and completed streams.

### Limits

- `FakeModel` does not implement `ai.ops.*` or `ai.probe`.
- Each nonempty text part produces one text delta, not token-sized chunks.
- Usage comes from the scripted message; the fake does not count tokens.
- Matching requires a result for every tool call. It does not repair incomplete
tool histories.
- The fake does not validate provider capabilities or inference parameters.
Structured output is parsed when `Message.get_output` is called, not when the
script is played.

## tool_call

```python
Expand Down
11 changes: 9 additions & 2 deletions src/ai/testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,14 @@ def stream(
params: params_.InferenceRequestParams | None = None,
) -> AsyncGenerator[events.Event]:
response = self._next_response(messages)
return events._replay_message_events(response)

async def replay() -> AsyncGenerator[events.Event]:
async for event in events._replay_message_events(response):
if isinstance(event, events.StreamEnd):
event = event.model_copy(update={"usage": response.usage})
yield event

return replay()

async def generate(
self,
Expand All @@ -272,7 +279,7 @@ def _next_response(
result = _walk(script, messages, fingerprints)
if isinstance(result, int):
script.played.add(result)
return script.messages[result]
return script.messages[result].model_copy(deep=True)
failures.append((script, result))
raise AssertionError(self._no_match_error(fingerprints, failures))

Expand Down
41 changes: 41 additions & 0 deletions tests/test_testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import ai
from ai.types import events as events_
from ai.types import messages as messages_
from ai.types import usage as usage_


@ai.tool
Expand Down Expand Up @@ -280,3 +281,43 @@ async def test_foreign_tool_results_fail() -> None:
pass

assert "call_bogus" in str(err.value)


async def test_fake_model_stream_reports_scripted_usage() -> None:
"""A completed stream reports the usage on the scripted response."""
scripted = ai.assistant_message("hello")
scripted.usage = usage_.Usage(input_tokens=3, output_tokens=5)
model = ai.testing.FakeModel([ai.user_message("go"), scripted])

async with ai.stream(model, [ai.user_message("go")]) as stream:
async for _ in stream:
pass

assert stream.usage == scripted.usage


async def test_fake_model_generate_does_not_hand_out_the_script() -> None:
"""A generated response is equal to, but distinct from, its script."""
scripted = ai.assistant_message("hello")
model = ai.testing.FakeModel([ai.user_message("go"), scripted])

message = await ai.generate(model, [ai.user_message("go")])

assert message == scripted
assert message is not scripted


def test_fake_provider_stream_matches_before_iteration() -> None:
scripted = ai.assistant_message("hello")
model = ai.testing.FakeModel([ai.user_message("go"), scripted])
messages = [ai.user_message("go")]

model.provider.stream(model, messages)

assert len(model.calls) == 1
assert model.calls[0] == messages
assert not model.unused

with pytest.raises(AssertionError, match="no script continues"):
model.provider.stream(model, [ai.user_message("unexpected")])
assert len(model.calls) == 2
Loading