diff --git a/docs/ai-python/content/docs/basics/streaming.mdx b/docs/ai-python/content/docs/basics/streaming.mdx index aaef16a6..c96fabed 100644 --- a/docs/ai-python/content/docs/basics/streaming.mdx +++ b/docs/ai-python/content/docs/basics/streaming.mdx @@ -63,15 +63,15 @@ message. ## Generate without streamed events -Use `ai.experimental_generate` when you only need the complete message: +Use `ai.generate` when you only need the complete message: ```python -message = await ai.experimental_generate(model, messages) +message = await ai.generate(model, messages) print(message.text) ``` The function accepts the same tools, output type, and request parameters as -`ai.stream`. It is experimental and may change or be removed. +`ai.stream`. ## Use structured output diff --git a/docs/ai-python/content/docs/basics/telemetry.mdx b/docs/ai-python/content/docs/basics/telemetry.mdx index b40fe701..1fa509e4 100644 --- a/docs/ai-python/content/docs/basics/telemetry.mdx +++ b/docs/ai-python/content/docs/basics/telemetry.mdx @@ -10,8 +10,7 @@ and agent loop turn — as a tree of *spans*. You decide where the spans go: export them to OpenTelemetry, send them to an observability vendor, or print them to the terminal. -Telemetry lives in the `ai.experimental_telemetry` module. It is experimental -and is not part of the stable API. It may change or be removed. +Telemetry lives in the `ai.telemetry` module. ## Turn on tracing @@ -34,7 +33,7 @@ exporter before running your agent: ```python import ai -from ai.experimental_telemetry import otel +from ai.telemetry import otel from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -45,7 +44,7 @@ provider.add_span_processor( OTLPSpanExporter(endpoint="http://127.0.0.1:4318/v1/traces") ) ) -ai.experimental_telemetry.register(otel.OtelAdapter(tracer_provider=provider)) +ai.telemetry.register(otel.OtelAdapter(tracer_provider=provider)) ``` Point the exporter at your collector. At application shutdown, call @@ -62,7 +61,7 @@ with adapters. ```python import ai -async with ai.experimental_telemetry.span("retrieval") as sp: +async with ai.telemetry.span("retrieval") as sp: sp.set_attrs(query=query) docs = await search(query) sp.set_attrs(count=len(docs)) @@ -93,7 +92,7 @@ to pytest fixtures or FastAPI lifecycles. ```python import ai -@ai.experimental_telemetry.adapter +@ai.telemetry.adapter async def vendor(span): with sdk.start_span(span.name) as v: # create a span before the loop while (ev := (yield)) is not None: # process events coming from span.add_event() @@ -102,7 +101,7 @@ async def vendor(span): v.set_error(span.error.message) v.update(output=span.data.model_dump(mode="json")) -ai.experimental_telemetry.register(vendor) # register custom adapter +ai.telemetry.register(vendor) # register custom adapter ``` The function runs once per span: @@ -127,5 +126,5 @@ independently. The module also has a low-level API, that can be useful for instrumenting complicated cases such as durable execution. It provides more control over adapter functionality and spans' lifecycles. See the -[`ai.experimental_telemetry` reference](/docs/reference/telemetry) for the +[`ai.telemetry` reference](/docs/reference/telemetry) for the full API, the span data types, and the durable execution patterns. diff --git a/docs/ai-python/content/docs/basics/testing.mdx b/docs/ai-python/content/docs/basics/testing.mdx index 2937d6af..aa1cba77 100644 --- a/docs/ai-python/content/docs/basics/testing.mdx +++ b/docs/ai-python/content/docs/basics/testing.mdx @@ -129,7 +129,7 @@ model = ai.testing.FakeModel( ] ) -message = await ai.experimental_generate( +message = await ai.generate( model, [ai.user_message("Weather in San Francisco as JSON.")], output_type=Weather, diff --git a/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx b/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx index fbfa1b4b..6d962318 100644 --- a/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx +++ b/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx @@ -545,7 +545,7 @@ See [`ai.errors`](/docs/reference/errors) for the complete error hierarchy. The Python provider currently supports: - Language model streaming. -- Buffered language model responses through `ai.experimental_generate`. +- Buffered language model responses through `ai.generate`. - Structured output. - Function and provider-executed tools. - Model listing and connection checks. diff --git a/docs/ai-python/content/docs/reference/ai.providers/index.mdx b/docs/ai-python/content/docs/reference/ai.providers/index.mdx index 9ecd7e04..017355ca 100644 --- a/docs/ai-python/content/docs/reference/ai.providers/index.mdx +++ b/docs/ai-python/content/docs/reference/ai.providers/index.mdx @@ -33,7 +33,7 @@ provider-specific namespaces. - `rerank`: Rerank documents against a query. - `decide`: Decide typed questions about shared state. -Application code normally uses `ai.stream`, `ai.experimental_generate`, or +Application code normally uses `ai.stream`, `ai.generate`, or [`ai.ops`](/docs/reference/ops) instead of calling provider methods directly. A protocol only needs to implement the operations that it supports. diff --git a/docs/ai-python/content/docs/reference/ai/experimental-generate.mdx b/docs/ai-python/content/docs/reference/ai/experimental-generate.mdx index a279e30a..84e6ec7c 100644 --- a/docs/ai-python/content/docs/reference/ai/experimental-generate.mdx +++ b/docs/ai-python/content/docs/reference/ai/experimental-generate.mdx @@ -1,24 +1,24 @@ --- -title: "experimental_generate" +title: "generate" description: Generate a buffered language-model response. type: reference -summary: Reference for ai.experimental_generate. +summary: Reference for ai.generate. --- -`ai.experimental_generate` makes one language-model call and returns the +`ai.generate` makes one language-model call and returns the complete assistant `Message` without exposing streamed events. -This API is experimental and may change or be removed. +`ai.experimental_generate` is a deprecated alias for `ai.generate`. ```python -message = await ai.experimental_generate(model, messages) +message = await ai.generate(model, messages) print(message.text) ``` ## Function ```python -await ai.experimental_generate( +await ai.generate( model, messages, *, @@ -31,7 +31,7 @@ await ai.experimental_generate( You can also pass an agent-loop context: ```python -message = await ai.experimental_generate(context=context) +message = await ai.generate(context=context) ``` Pass either `model` and `messages`, or `context=`, not both. @@ -63,7 +63,7 @@ class Summary(pydantic.BaseModel): points: list[str] -message = await ai.experimental_generate( +message = await ai.generate( model, [ai.user_message("Summarize the mission as JSON.")], output_type=Summary, diff --git a/docs/ai-python/content/docs/reference/ai/index.mdx b/docs/ai-python/content/docs/reference/ai/index.mdx index 1663df48..3b336cee 100644 --- a/docs/ai-python/content/docs/reference/ai/index.mdx +++ b/docs/ai-python/content/docs/reference/ai/index.mdx @@ -21,7 +21,7 @@ These top-level exports have their own pages under `ai`. - [`stream`](/docs/reference/ai/stream): Call a model and iterate response events. The same page documents `Stream`. -- [`experimental_generate`](/docs/reference/ai/experimental-generate): Call a +- [`generate`](/docs/reference/ai/experimental-generate): Call a model and return a buffered `Message`. - [`get_model`](/docs/reference/ai/get-model): Resolve a model id into a `Model`. @@ -47,7 +47,7 @@ These module aliases are also re-exported from `ai`. - [`tools`](/docs/reference/tools): Model-facing tool schema types. - [`ui`](/docs/reference/ai.ui/ai-sdk): UI adapter namespaces. - [`util`](/docs/reference/util): Async utility helpers. -- [`experimental_telemetry`](/docs/reference/telemetry): Tracing and +- [`telemetry`](/docs/reference/telemetry): Tracing and observability APIs. ## Models diff --git a/docs/ai-python/content/docs/reference/ai/stream.mdx b/docs/ai-python/content/docs/reference/ai/stream.mdx index 670fdfae..97ed1548 100644 --- a/docs/ai-python/content/docs/reference/ai/stream.mdx +++ b/docs/ai-python/content/docs/reference/ai/stream.mdx @@ -68,6 +68,10 @@ tools use built-in tool events. Generated files arrive as `FileEvent`. Each yielded event has the current aggregated `event.message`. +`stream.telemetry_span` is the [telemetry span](/docs/reference/telemetry) +for this model call, including replay. It is `None` on streams not created +by `ai.stream`. + ## Replay If the last input message has `replay=True`, `ai.stream` does not call the @@ -89,5 +93,5 @@ If the provider stream ends before a finish event, iteration raises `stream.message`. Use -[`ai.experimental_generate`](/docs/reference/ai/experimental-generate) when +[`ai.generate`](/docs/reference/ai/experimental-generate) when you need the complete message without consuming streamed events. diff --git a/docs/ai-python/content/docs/reference/telemetry.mdx b/docs/ai-python/content/docs/reference/telemetry.mdx index 38141bf7..3464e0ae 100644 --- a/docs/ai-python/content/docs/reference/telemetry.mdx +++ b/docs/ai-python/content/docs/reference/telemetry.mdx @@ -1,16 +1,16 @@ --- -title: "ai.experimental_telemetry" +title: "ai.telemetry" description: Reference for spans, adapters, sinks, and the OpenTelemetry exporter. type: reference -summary: Reference for ai.experimental_telemetry. +summary: Reference for ai.telemetry. --- -The module is experimental: it is not part of the stable API -and may change or be removed. - -`ai.experimental_telemetry` contains instrumentation for tracing agent runs, model calls, +`ai.telemetry` contains instrumentation for tracing agent runs, model calls, tool executions, and other work in a provider-agnostic way. +`ai.experimental_telemetry` is a deprecated alias for `ai.telemetry`. +Use `Stream.telemetry_span` instead of the deprecated `Stream.experimental_span`. + The high-level API consists of the `span()` context manager for creating custom spans, and an `@adapter` decorator for making adapters. It can be used to augment the built-in instrumentation. @@ -72,7 +72,7 @@ class RetrievalSpanData(pydantic.BaseModel): query: str count: int | None = None -async with ai.experimental_telemetry.span(RetrievalSpanData(query=q)) as sp: +async with ai.telemetry.span(RetrievalSpanData(query=q)) as sp: docs = await search(q) sp.data.count = len(docs) # typed ``` @@ -109,7 +109,7 @@ one call; the response `message`, `usage`, `finish_reason`, `response_id`, and `response_model` are set at span end. `AiGenerateSpanData` carries the same request and response fields for -`ai.experimental_generate` calls. +`ai.generate` calls. Each `ai.ops` function has its own span data. All carry `model` and `provider`, as well as modality-specific fields. @@ -158,7 +158,7 @@ raised) and pushes on exit: ```python import ai -async with ai.experimental_telemetry.span("retrieval") as sp: +async with ai.telemetry.span("retrieval") as sp: sp.set_attrs(query=query) docs = await search(query) sp.set_attrs(count=len(docs)) @@ -222,7 +222,7 @@ The following utilities connect manual spans to the built-in instrumentation. ```python -sp = ai.experimental_telemetry.create_span("turn") +sp = ai.telemetry.create_span("turn") sp.set_attrs(session=session_id) sp.stamp_start() await sp.push() # visible to adapters @@ -258,15 +258,15 @@ class Sink(Protocol): `push_all(spans)` can be used to send a list of spans to the current sink. ```python -sink = ai.experimental_telemetry.DictSink() +sink = ai.telemetry.DictSink() -async with ai.experimental_telemetry.use_sink(sink): +async with ai.telemetry.use_sink(sink): ... # spans go to the sink payload = [s.model_dump(mode="json") for s in sink.finished_spans] # elsewhere, with the default sink on: -await ai.experimental_telemetry.push_all(payload) +await ai.telemetry.push_all(payload) ``` - `DictSink.spans` exposes the dict of span id to span. @@ -296,7 +296,7 @@ If an adapter raises an error, the SDK logs it and skips the adapter. Telemetry `@adapter` builds an adapter from one async generator function. ```python -@ai.experimental_telemetry.adapter +@ai.telemetry.adapter async def vendor(span): with sdk.start_span(span.name) as v: # span start while (ev := (yield)) is not None: # each event, live @@ -305,7 +305,7 @@ async def vendor(span): v.set_error(span.error.message) v.update(output=span.data.model_dump(mode="json")) -ai.experimental_telemetry.register(vendor) +ai.telemetry.register(vendor) ``` - Code before the loop runs at span start. Each span event resumes the @@ -324,7 +324,7 @@ When the adapter needs configuration or state, decorate a class instead. `__call__` must be an async generator method with the same yield loop: ```python -@ai.experimental_telemetry.adapter +@ai.telemetry.adapter class Vendor: def __init__(self, *, api_key): self._client = sdk.Client(api_key) @@ -334,7 +334,7 @@ class Vendor: while (ev := (yield)) is not None: v.log_event(ev.name) -ai.experimental_telemetry.register(Vendor(api_key="...")) +ai.telemetry.register(Vendor(api_key="...")) ``` Decorating a class mixes the adapter machinery (`AdapterMixin`) into it. @@ -365,7 +365,7 @@ Implement the protocol directly when the callback shape fits your target better ## The OpenTelemetry adapter -`ai.experimental_telemetry.otel` maps framework spans onto OpenTelemetry +`ai.telemetry.otel` maps framework spans onto OpenTelemetry spans, following the `gen_ai` semantic conventions. It requires the `otel` extra: @@ -377,7 +377,7 @@ Configure a tracer provider and exporter before registering the adapter: ```python import ai -from ai.experimental_telemetry import otel +from ai.telemetry import otel from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -385,7 +385,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor tracer_provider = TracerProvider() tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) adapter = otel.OtelAdapter(tracer_provider=tracer_provider) -ai.experimental_telemetry.register(adapter) +ai.telemetry.register(adapter) ``` `OtelAdapter` uses the global tracer provider unless you pass one. It does @@ -472,7 +472,7 @@ workflow = vercel.workflow.Workflows() @workflow.workflow -@ai.experimental_telemetry.use_time(vercel.workflow.time_ns) +@ai.telemetry.use_time(vercel.workflow.time_ns) async def run_turn(turn_input): ... ``` @@ -510,7 +510,7 @@ import ai async def handle(request): - async with ai.experimental_telemetry.span("handle_request") as sp: + async with ai.telemetry.span("handle_request") as sp: sp.set_attrs(request_id=request.id) async with agent.run(model, request.messages) as run: async for event in run: @@ -535,7 +535,7 @@ class RetrievalSpanData(pydantic.BaseModel): count: int | None = None -@ai.experimental_telemetry.adapter +@ai.telemetry.adapter async def retrieval_metrics(span): if not isinstance(span.data, RetrievalSpanData): return # skip everything else @@ -569,7 +569,7 @@ workflow = vercel.workflow.Workflows() @workflow.workflow @ai.messages.use_random(vercel.workflow.random) # span and message ids -@ai.experimental_telemetry.use_time(vercel.workflow.time_ns) # timestamps +@ai.telemetry.use_time(vercel.workflow.time_ns) # timestamps async def my_workflow(payload): ... ``` @@ -582,8 +582,8 @@ and the `use_sink` context manager. ```python @workflow.workflow async def my_workflow(payload): - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): ... # any instrumented work: agent runs, custom spans await report_spans( @@ -593,7 +593,7 @@ async def my_workflow(payload): @workflow.step async def report_spans(spans_data): - await ai.experimental_telemetry.push_all(spans_data) + await ai.telemetry.push_all(spans_data) ``` Finally, we need to be able to serialize and restore spans in order to pass them into @@ -602,26 +602,26 @@ framework reports them to the vendor. ```python # where the run begins: mint the span; nothing is reported yet -if ai.experimental_telemetry.is_enabled(): +if ai.telemetry.is_enabled(): payload["run_span"] = ( - ai.experimental_telemetry.create_span("run") + ai.telemetry.create_span("run") .stamp_start() .model_dump(mode="json") ) # in any step or process that continues the work: -run_span = ai.experimental_telemetry.Span.model_validate(payload["run_span"]) -async with ai.experimental_telemetry.use_span(run_span): +run_span = ai.telemetry.Span.model_validate(payload["run_span"]) +async with ai.telemetry.use_span(run_span): ... # spans opened here parent under run_span # where the run ends: complete the span and report it -run_span = ai.experimental_telemetry.Span.model_validate(payload["run_span"]) +run_span = ai.telemetry.Span.model_validate(payload["run_span"]) await run_span.stamp_end().push() ``` ## Module exports -`ai.experimental_telemetry` exports: +`ai.telemetry` exports: - Spans: `Span`, `SpanData`, `SpanEvent`, `SpanError`. - Span data types: `RunSpanData`, `LoopTurnSpanData`, `AiStreamSpanData`, @@ -637,4 +637,4 @@ await run_span.stamp_end().push() - Sinks and clocks: `Sink`, `use_sink`, `DictSink`, `push_all`, `use_time`, `now_ns`. -`ai.experimental_telemetry.otel` exports `OtelAdapter`. +`ai.telemetry.otel` exports `OtelAdapter`. diff --git a/examples/.test_scripts/run-examples.py b/examples/.test_scripts/run-examples.py index bfd07af6..846891dc 100755 --- a/examples/.test_scripts/run-examples.py +++ b/examples/.test_scripts/run-examples.py @@ -15,7 +15,7 @@ uv run examples/.test_scripts/run-examples.py --model MODEL # patch ai.get_model() to use the given model for every sample uv run examples/.test_scripts/run-examples.py --protocol=responses - # patch model/provider helpers, ai.stream(), and experimental_generate() + # patch model/provider helpers, ai.stream(), and generate() """ import argparse diff --git a/examples/.test_scripts/run-with-patched-model.py b/examples/.test_scripts/run-with-patched-model.py index 8f146b5e..2df81919 100644 --- a/examples/.test_scripts/run-with-patched-model.py +++ b/examples/.test_scripts/run-with-patched-model.py @@ -86,7 +86,7 @@ def main() -> None: original_get_model = _model.get_model original_stream = _api.stream - original_generate = _api.experimental_generate + original_generate = _api.generate def selected_protocol() -> ai.ProviderProtocol[Any] | None: if protocol_factory is None: @@ -205,10 +205,10 @@ def __init__( cast("Any", core).stream = patched_stream cast("Any", _api).stream = patched_stream - cast("Any", ai).experimental_generate = patched_generate - cast("Any", models).experimental_generate = patched_generate - cast("Any", core).experimental_generate = patched_generate - cast("Any", _api).experimental_generate = patched_generate + cast("Any", ai).generate = patched_generate + cast("Any", models).generate = patched_generate + cast("Any", core).generate = patched_generate + cast("Any", _api).generate = patched_generate sys.argv = [args.file] runpy.run_path(args.file, run_name="__main__") diff --git a/examples/agents/telemetry.py b/examples/agents/telemetry.py index efea10c1..02104f02 100644 --- a/examples/agents/telemetry.py +++ b/examples/agents/telemetry.py @@ -1,21 +1,21 @@ """Agent run with telemetry: console adapter + a custom span in a tool. For an out-of-process viewer, run -``python -m ai.experimental_telemetry.utils.viewer`` and see -``ai/experimental_telemetry/utils/viewer.py`` for the otel exporter +``python -m ai.telemetry.utils.viewer`` and see +``ai/telemetry/utils/viewer.py`` for the otel exporter setup. """ import asyncio import ai -from ai.experimental_telemetry.utils import console +from ai.telemetry.utils import console @ai.tool async def get_weather(city: str) -> str: """Get current weather for a city.""" - async with ai.experimental_telemetry.span("lookup") as span: + async with ai.telemetry.span("lookup") as span: span.set_attrs(city=city) await asyncio.sleep(0.1) span.set_attrs(source="cache") @@ -23,7 +23,7 @@ async def get_weather(city: str) -> str: async def main() -> None: - ai.experimental_telemetry.register(console.ConsoleAdapter()) + ai.telemetry.register(console.ConsoleAdapter()) model = ai.get_model("anthropic/claude-sonnet-4.6") my_agent = ai.Agent(tools=[get_weather]) diff --git a/examples/external/handoff.py b/examples/external/handoff.py index ce60f4be..b45e5a6a 100644 --- a/examples/external/handoff.py +++ b/examples/external/handoff.py @@ -17,7 +17,7 @@ async def main() -> None: ) ] async with ai.external.ClaudeCode(cwd=tmp) as claude: - reply = await ai.experimental_generate(claude, messages) + reply = await ai.generate(claude, messages) print( f"Claude Code: {reply.text.strip()} " f"({len(reply.builtin_tool_calls)} tool call(s))" @@ -36,7 +36,7 @@ async def main() -> None: ), ] async with ai.external.Codex(cwd=tmp) as codex: - reply = await ai.experimental_generate(codex, messages) + reply = await ai.generate(codex, messages) print("Codex:", reply.text.strip()) notes = pathlib.Path(tmp, "notes.txt").read_text() diff --git a/examples/external/resume.py b/examples/external/resume.py index ba745076..bdd5a544 100644 --- a/examples/external/resume.py +++ b/examples/external/resume.py @@ -9,7 +9,7 @@ async def main() -> None: with tempfile.TemporaryDirectory() as tmp: async with ai.external.ClaudeCode(cwd=tmp) as harness: - await ai.experimental_generate( + await ai.generate( harness, [ ai.user_message( @@ -36,7 +36,7 @@ async def main() -> None: *again.session.messages, ai.user_message(question), ] - answer = await ai.experimental_generate(again, messages) + answer = await ai.generate(again, messages) print(answer.text) diff --git a/examples/external/streaming.py b/examples/external/streaming.py index f0f2de84..3789ee15 100644 --- a/examples/external/streaming.py +++ b/examples/external/streaming.py @@ -64,7 +64,7 @@ async def main() -> None: "Which file did you write? Reply with its name only." ), ] - print((await ai.experimental_generate(harness, messages)).text) + print((await ai.generate(harness, messages)).text) if __name__ == "__main__": diff --git a/examples/models/generate.py b/examples/models/generate.py index a6de947d..dc5f6b88 100644 --- a/examples/models/generate.py +++ b/examples/models/generate.py @@ -1,6 +1,6 @@ """Non-streaming LLM call. -``ai.experimental_generate()`` is the counterpart of ``ai.stream()``: +``ai.generate()`` is the counterpart of ``ai.stream()``: same arguments, but it returns the final message instead of yielding events. """ @@ -18,7 +18,7 @@ async def main() -> None: - message = await ai.experimental_generate(model, messages) + message = await ai.generate(model, messages) print(message.text) diff --git a/skills/ai/references/custom-provider.md b/skills/ai/references/custom-provider.md index b53a33dc..53520f9f 100644 --- a/skills/ai/references/custom-provider.md +++ b/skills/ai/references/custom-provider.md @@ -78,7 +78,7 @@ Direct provider model IDs use `ai.get_model("handle:model")`; slash-separated IDs route through AI Gateway. Implement `ProviderProtocol.generate` when the upstream API has a native -buffered language-model endpoint. Otherwise, `ai.experimental_generate` falls +buffered language-model endpoint. Otherwise, `ai.generate` falls back to `stream`. Implement `generate_image`, `generate_video`, `generate_audio`, `embed`, `transcribe`, or `rerank` only for the dedicated model operations that the provider supports. diff --git a/src/ai/__init__.py b/src/ai/__init__.py index bee92c7b..61f350a2 100644 --- a/src/ai/__init__.py +++ b/src/ai/__init__.py @@ -1,11 +1,14 @@ +import warnings +from typing import TYPE_CHECKING + from . import ( durable, errors, - experimental_telemetry, external, models, ops, providers, + telemetry, testing, ui, util, @@ -98,7 +101,7 @@ TopKSamplerParams, TopPSamplerParams, Unset, - experimental_generate, + generate, get_model, probe, stream, @@ -204,6 +207,7 @@ "experimental_telemetry", "external", "file_part", + "generate", "get_hook_registry", "get_model", "get_provider", @@ -218,6 +222,7 @@ "resolve_hook", "stream", "system_message", + "telemetry", "testing", "text_part", "thinking", @@ -231,3 +236,26 @@ "util", "yield_from", ] + +if TYPE_CHECKING: + experimental_generate = generate + from . import ( + experimental_telemetry as experimental_telemetry, # noqa: TC004 + ) +else: + + def __getattr__(name: str) -> object: + if name == "experimental_generate": + warnings.warn( + f"{__name__}.experimental_generate is deprecated; " + "use ai.generate", + DeprecationWarning, + stacklevel=2, + ) + return generate + if name == "experimental_telemetry": + # Avoid adding an importlib frame to the shim's warning stack. + return __import__( + f"{__name__}.experimental_telemetry", fromlist=("__name__",) + ) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ai/agents/agent.py b/src/ai/agents/agent.py index 23952495..6fc9169c 100644 --- a/src/ai/agents/agent.py +++ b/src/ai/agents/agent.py @@ -38,8 +38,7 @@ # Use the typing_extensions backport so this works on 3.12 too. from typing_extensions import TypeVar -from .. import experimental_telemetry as telemetry -from .. import models, type_utils, types, util +from .. import models, telemetry, type_utils, types, util from ..types import builders from ..types import events as events_ from ..types.messages import MessageBundle diff --git a/src/ai/agents/hooks.py b/src/ai/agents/hooks.py index 0cc22a3b..73d9e27e 100644 --- a/src/ai/agents/hooks.py +++ b/src/ai/agents/hooks.py @@ -27,8 +27,7 @@ import pydantic -from .. import experimental_telemetry as telemetry -from .. import types +from .. import telemetry, types from ..types import messages as messages_ from . import runtime as runtime_ diff --git a/src/ai/experimental_telemetry/__init__.py b/src/ai/experimental_telemetry/__init__.py index a6313427..7c4d3ca8 100644 --- a/src/ai/experimental_telemetry/__init__.py +++ b/src/ai/experimental_telemetry/__init__.py @@ -1,111 +1,28 @@ -"""Telemetry: spans, sinks, adapters, and the ambient current span. - -Experimental: not part of the stable API, may change or be removed. - -See :mod:`.span` for the full story. -""" +"""Deprecated import path for :mod:`ai.telemetry`.""" +import importlib +import sys import warnings -from typing import TYPE_CHECKING, Any - -from .span import ( - FIRST_TOKEN, - HOOK_CANCELLED, - HOOK_DEFERRED, - HOOK_RESOLVED, - RESPONSE_COMPLETE, - AdapterCallable, - AdapterMixin, - AdapterProtocol, - AiGenerateSpanData, - AiStreamSpanData, - CustomSpanData, - DecideSpanData, - DictSink, - EmbedSpanData, - GenerateAudioSpanData, - GenerateImageSpanData, - GenerateVideoSpanData, - HookSpanData, - LoopTurnSpanData, - RerankSpanData, - RunSpanData, - Sink, - Span, - SpanData, - SpanError, - SpanEvent, - ToolExecutionSpanData, - TranscribeSpanData, - adapter, - create_span, - current_span, - is_enabled, - now_ns, - push_all, - register, - span, - unregister, - use_sink, - use_span, - use_time, -) +from pathlib import Path +from typing import TYPE_CHECKING -__all__ = [ - "FIRST_TOKEN", - "HOOK_CANCELLED", - "HOOK_DEFERRED", - "HOOK_RESOLVED", - "RESPONSE_COMPLETE", - "AdapterCallable", - "AdapterMixin", - "AdapterProtocol", - "AiGenerateSpanData", - "AiStreamSpanData", - "CustomSpanData", - "DecideSpanData", - "DictSink", - "EmbedSpanData", - "GenerateAudioSpanData", - "GenerateImageSpanData", - "GenerateVideoSpanData", - "HookSpanData", - "LoopTurnSpanData", - "RerankSpanData", - "RunSpanData", - "Sink", - "Span", - "SpanData", - "SpanError", - "SpanEvent", - "ToolExecutionSpanData", - "TranscribeSpanData", - "adapter", - "create_span", - "current_span", - "is_enabled", - "now_ns", - "push_all", - "register", - "span", - "unregister", - "use_sink", - "use_span", - "use_time", -] +from ..telemetry import * # noqa: F403 +from ..telemetry import __all__ as __all__ if TYPE_CHECKING: - EvaluateSpanData = DecideSpanData + from ..telemetry import EvaluateSpanData as EvaluateSpanData else: + from ..telemetry import __getattr__ as __getattr__ - def __getattr__(name: str) -> Any: - # Deprecated names, kept as aliases of their replacements. - if name == "EvaluateSpanData": - warnings.warn( - "ai.experimental_telemetry.EvaluateSpanData is deprecated, " - "use ai.experimental_telemetry.DecideSpanData", - DeprecationWarning, - stacklevel=2, - ) - return DecideSpanData - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + +warnings.warn( + "ai.experimental_telemetry is deprecated; use ai.telemetry", + DeprecationWarning, + stacklevel=2, + skip_file_prefixes=(str(Path(__file__).parent.parent) + "/",), +) + +# Loading the legacy submodule later would overwrite the span() re-export. +sys.modules[f"{__name__}.span"] = importlib.import_module( + "..telemetry.span", __name__ +) diff --git a/src/ai/experimental_telemetry/otel.py b/src/ai/experimental_telemetry/otel.py index 68a0551a..3cf9a224 100644 --- a/src/ai/experimental_telemetry/otel.py +++ b/src/ai/experimental_telemetry/otel.py @@ -1,665 +1,10 @@ -"""OpenTelemetry adapter. +"""Deprecated import path for :mod:`ai.telemetry.otel`.""" -Experimental: not part of the stable API, may change or be removed. - -:: - - import ai - from ai.experimental_telemetry import otel - - ai.experimental_telemetry.register(otel.OtelAdapter()) - -The adapter uses the global tracer provider by default. To set up a -tracer provider and OTLP export, see the OpenTelemetry docs: -https://opentelemetry.io/docs/languages/python/exporters/#otlp - -Follows the ``gen_ai`` semantic conventions. - -There's a certain amount of jank in this adapter, because we want it to work in -the durable execution case, which conflicts with the OTel spec. - -Messages are emitted in the semconv shape for maximum downstream compatibility. -Capturing message content is opt-in. -""" - -from __future__ import annotations - -import contextlib -import contextvars -import hashlib -import json -import logging -import os -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable - -from .. import errors -from .. import experimental_telemetry as telemetry - -try: - import opentelemetry.context - import opentelemetry.sdk.trace.id_generator - import opentelemetry.trace -except ModuleNotFoundError as exc: # pragma: no cover - raise errors.InstallationError( - "could not import `opentelemetry`, which is required for the otel " - 'telemetry adapter, you can install it with `pip install "ai[otel]"` ' - 'or `uv add "ai[otel]"`' - ) from exc +import sys +from importlib import import_module +from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import AsyncGenerator - - from ..types import messages as messages_ - from ..types import usage as usage_ - -logger = logging.getLogger(__name__) - - -# standard flag for capturing message content -CAPTURE_CONTENT_ENV = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" - - -def _should_capture_content() -> bool: - value = os.environ.get(CAPTURE_CONTENT_ENV, "") - return value.strip().lower() in ("true", "span_only", "span_and_event") - - -def _semconv_name(sp: telemetry.Span) -> str: - match sp.data: - case ( - (telemetry.AiStreamSpanData() | telemetry.AiGenerateSpanData()) as d - ): - return f"chat {d.model}" - case telemetry.DecideSpanData() as d: - return f"decide {d.model}" - case telemetry.EmbedSpanData() as d: - return f"embeddings {d.model}" - case ( - ( - telemetry.GenerateAudioSpanData() - | telemetry.GenerateImageSpanData() - | telemetry.GenerateVideoSpanData() - ) as d - ): - return f"generate_content {d.model}" - case telemetry.RerankSpanData() as d: - return f"rerank {d.model}" - case telemetry.TranscribeSpanData() as d: - return f"transcribe {d.model}" - case telemetry.ToolExecutionSpanData() as d: - return f"execute_tool {d.tool_name}" - case telemetry.RunSpanData() as d: - return f"invoke_agent {d.agent}" - case _: - return sp.name - - -def _semconv_kind(sp: telemetry.Span) -> opentelemetry.trace.SpanKind: - # Inference spans SHOULD be CLIENT per semconv; agent and tool - # spans run in-process and stay INTERNAL. - match sp.data: - case ( - telemetry.AiStreamSpanData() - | telemetry.AiGenerateSpanData() - | telemetry.DecideSpanData() - | telemetry.EmbedSpanData() - | telemetry.GenerateAudioSpanData() - | telemetry.GenerateImageSpanData() - | telemetry.GenerateVideoSpanData() - | telemetry.RerankSpanData() - | telemetry.TranscribeSpanData() - ): - return opentelemetry.trace.SpanKind.CLIENT - case _: - return opentelemetry.trace.SpanKind.INTERNAL - - -def _maybe_json_loads(raw: str) -> Any: - try: - return json.loads(raw) - except ValueError: - return raw - - -def _semconv_part(part: dict[str, Any]) -> dict[str, Any]: - """Convert part to gen_ai semconv shape. - - https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/non-normative/models.py - """ - match part["kind"]: - case "text": - return {"type": "text", "content": part["text"]} - case "reasoning": - return {"type": "reasoning", "content": part["text"]} - case "tool_call": - return { - "type": "tool_call", - "id": part["tool_call_id"], - "name": part["tool_name"], - "arguments": _maybe_json_loads(part["tool_args"]), - } - case "tool_result": - return { - "type": "tool_call_response", - "id": part["tool_call_id"], - "response": part["result"], - } - case "builtin_tool_call": - return { - "type": "server_tool_call", - "id": part["tool_call_id"], - "name": part["tool_name"], - "server_tool_call": { - "type": part["tool_name"], - "arguments": _maybe_json_loads(part["tool_args"]), - }, - } - case "builtin_tool_return": - return { - "type": "server_tool_call_response", - "id": part["tool_call_id"], - "server_tool_call_response": { - "type": part["tool_name"], - "response": part["result"], - }, - } - case "file": - media_type = part["media_type"] - modality = media_type.split("/")[0] - if modality not in ("image", "video", "audio"): - modality = "document" - # data is a str after the JSON dump: a URL or base-64 - # content. - if part["data"].startswith(("http://", "https://")): - return { - "type": "uri", - "modality": modality, - "mime_type": media_type, - "uri": part["data"], - } - return { - "type": "blob", - "modality": modality, - "mime_type": media_type, - "content": part["data"], - } - case kind: - return {"type": kind} - - -def _semconv_parts( - message: messages_.Message, -) -> tuple[str, list[dict[str, Any]]]: - dumped = message.model_dump(mode="json", fallback=str) - return dumped["role"], [_semconv_part(p) for p in dumped["parts"]] - - -def _semconv_content_attributes( - messages: list[messages_.Message], - output: messages_.Message | None, - *, - error: bool, - finish_reason: str | None = None, -) -> dict[str, str]: - """Extract and convert message content attributes.""" - system_parts: list[dict[str, Any]] = [] - inputs: list[dict[str, Any]] = [] - for message in messages: - role, parts = _semconv_parts(message) - if role == "system": - # these live in gen_ai.system_instructions - system_parts += parts - else: - inputs.append({"role": role, "parts": parts}) - attrs = {"gen_ai.input.messages": json.dumps(inputs)} - if system_parts: - attrs["gen_ai.system_instructions"] = json.dumps(system_parts) - if output is not None: - role, parts = _semconv_parts(output) - if finish_reason is not None: - finish = finish_reason - elif error: - finish = "error" - elif any(p["type"] == "tool_call" for p in parts): - finish = "tool_call" - else: - finish = "stop" - attrs["gen_ai.output.messages"] = json.dumps( - [{"role": role, "parts": parts, "finish_reason": finish}] - ) - return attrs - - -def _hack_get_field(obj: Any, name: str) -> Any: - # this is caused by the params import hack in span.py - if isinstance(obj, dict): - return obj.get(name) - return getattr(obj, name, None) - - -_SEMCONV_SAMPLER_FIELDS = ( - "temperature", - "top_k", - "top_p", - "frequency_penalty", - "presence_penalty", - "seed", -) - - -def _semconv_request_attributes(params: Any) -> dict[str, Any]: - """Extract and convert request attributes from inference params.""" - attrs: dict[str, Any] = {} - sampling = _hack_get_field(params, "sampling") - if isinstance(sampling, dict): - for sampler in sampling.values(): - for name in _SEMCONV_SAMPLER_FIELDS: - value = _hack_get_field(sampler, name) - if isinstance(value, int | float) and not isinstance( - value, bool - ): - attrs[f"gen_ai.request.{name}"] = value - max_tokens = _hack_get_field( - _hack_get_field(params, "output"), "max_tokens" - ) - if isinstance(max_tokens, int): - attrs["gen_ai.request.max_tokens"] = max_tokens - effort = _hack_get_field(_hack_get_field(params, "reasoning"), "effort") - if isinstance(effort, str): - attrs["gen_ai.request.reasoning.level"] = effort - return attrs - - -def _semconv_usage_attributes(usage: usage_.Usage) -> dict[str, Any]: - attrs: dict[str, Any] = { - "gen_ai.usage.input_tokens": usage.input_tokens, - "gen_ai.usage.output_tokens": usage.output_tokens, - } - if usage.reasoning_tokens is not None: - attrs["gen_ai.usage.reasoning.output_tokens"] = usage.reasoning_tokens - if usage.cache_read_tokens is not None: - attrs["gen_ai.usage.cache_read.input_tokens"] = usage.cache_read_tokens - if usage.cache_write_tokens is not None: - attrs["gen_ai.usage.cache_creation.input_tokens"] = ( - usage.cache_write_tokens - ) - return attrs - - -def _time_to_first_chunk(sp: telemetry.Span) -> float | None: - if sp.started_at is None: - return None - for event in sp.events: - if event.name == telemetry.FIRST_TOKEN: - return (event.time_ns - sp.started_at) / 1e9 - return None - - -def _semconv_tool_definitions(names: list[str]) -> str: - return json.dumps([{"type": "function", "name": n} for n in names]) - - -def _attributes(sp: telemetry.Span, *, capture_content: bool) -> dict[str, Any]: - # process trace_attrs - attrs: dict[str, Any] = { - # otel only allows scalar attribute values or lists of them - key: value - if isinstance(value, str | bool | int | float) - else repr(value) - for key, value in sp.trace_attrs.items() - } - if sp.replay: - attrs["ai.replay"] = True - match sp.data: - case ( - (telemetry.AiStreamSpanData() | telemetry.AiGenerateSpanData()) as d - ): - attrs["gen_ai.operation.name"] = "chat" - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - streamed = isinstance(d, telemetry.AiStreamSpanData) - attrs["gen_ai.request.stream"] = streamed - if d.output_type is not None: - # Structured output was requested; the semconv value is - # the output kind, not the Python type name. - attrs["gen_ai.output.type"] = "json" - attrs |= _semconv_request_attributes(d.params) - if streamed: - ttfc = _time_to_first_chunk(sp) - if ttfc is not None: - attrs["gen_ai.response.time_to_first_chunk"] = ttfc - if d.finish_reason is not None: - attrs["gen_ai.response.finish_reasons"] = [d.finish_reason] - if d.response_id is not None: - attrs["gen_ai.response.id"] = d.response_id - if d.response_model is not None: - attrs["gen_ai.response.model"] = d.response_model - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - if capture_content: - if d.tool_names: - attrs["gen_ai.tool.definitions"] = ( - _semconv_tool_definitions(d.tool_names) - ) - attrs |= _semconv_content_attributes( - d.messages, - d.message, - error=sp.error is not None, - finish_reason=d.finish_reason, - ) - case telemetry.DecideSpanData() as d: - attrs["gen_ai.operation.name"] = "decide" - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - case telemetry.EmbedSpanData() as d: - attrs["gen_ai.operation.name"] = "embeddings" - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - if d.dimensions is not None: - attrs["gen_ai.embeddings.dimension.count"] = d.dimensions - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - case ( - ( - telemetry.GenerateAudioSpanData() - | telemetry.GenerateImageSpanData() - | telemetry.GenerateVideoSpanData() - ) as d - ): - attrs["gen_ai.operation.name"] = "generate_content" - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - case telemetry.RerankSpanData() as d: - attrs["gen_ai.operation.name"] = "rerank" - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - case telemetry.TranscribeSpanData() as d: - attrs["gen_ai.operation.name"] = "transcribe" - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - case telemetry.ToolExecutionSpanData() as d: - attrs["gen_ai.operation.name"] = "execute_tool" - attrs["gen_ai.tool.name"] = d.tool_name - attrs["gen_ai.tool.type"] = "function" - attrs["gen_ai.tool.call.id"] = d.tool_call_id - if d.tool_description is not None: - attrs["gen_ai.tool.description"] = d.tool_description - if d.is_error: - # The exception type is not captured when the error is - # only a model-facing result (see NEW_DATA_CAPTURE.md); - # ``_OTHER`` is semconv's fallback class. Overwritten - # with the real type when the span carries an error. - attrs["error.type"] = "_OTHER" - if capture_content: - if d.args is not None: - attrs["gen_ai.tool.call.arguments"] = json.dumps( - d.args, default=str - ) - if d.result is not None: - attrs["gen_ai.tool.call.result"] = json.dumps( - d.result, default=str - ) - case telemetry.RunSpanData() as d: - attrs["gen_ai.operation.name"] = "invoke_agent" - attrs["gen_ai.agent.name"] = d.agent - if d.provider is not None: - attrs["gen_ai.provider.name"] = d.provider - attrs["gen_ai.request.model"] = d.model - if d.output_type is not None: - # Structured output was requested; the semconv value is - # the output kind, not the Python type name. - attrs["gen_ai.output.type"] = "json" - attrs |= _semconv_request_attributes(d.params) - if d.usage is not None: - attrs |= _semconv_usage_attributes(d.usage) - if capture_content: - if d.tool_names: - attrs["gen_ai.tool.definitions"] = ( - _semconv_tool_definitions(d.tool_names) - ) - attrs |= _semconv_content_attributes( - d.messages, d.final_message, error=sp.error is not None - ) - case telemetry.HookSpanData() as d: - attrs["ai.hook.label"] = d.label - attrs["ai.hook.type"] = d.hook_type - attrs["ai.hook.status"] = d.status - case telemetry.CustomSpanData() as d: - for key, value in d.attrs.items(): - # otel only allows scalar attribute values or lists of them - attrs[key] = ( - value - if isinstance(value, str | bool | int | float) - else repr(value) - ) - return attrs - - -def _derive_trace_id(id_: str) -> int: - """Derive a stable otel trace id (128 bits) from one of ours.""" - return int.from_bytes(hashlib.sha256(id_.encode()).digest()[:16]) - - -def _derive_span_id(id_: str) -> int: - """Derive a stable otel span id (64 bits) from one of ours.""" - return int.from_bytes(hashlib.sha256(id_.encode()).digest()[:8]) - - -# otel spec requires ids to be always minted by the otel sdk. we can't have -# that because of the durable execution case, where we can't always emit the -# span right where we created it. that's why we're using a smuggling hack to -# smuggle our ids in via a contextvar. fable is telling me that's a classic -# trick. -_smuggled_ids: contextvars.ContextVar[tuple[int, int] | None] = ( - contextvars.ContextVar("otel_smuggled_ids", default=None) -) - - -class _SmugglingIdGenerator( - opentelemetry.sdk.trace.id_generator.RandomIdGenerator -): - """Fake id generator for smuggling ids via a contextvar.""" - - def __init__( - self, inner: opentelemetry.sdk.trace.id_generator.IdGenerator - ) -> None: - self._inner = inner # stock id generator - - def generate_trace_id(self) -> int: - smuggled = _smuggled_ids.get() - return ( - smuggled[0] - if smuggled is not None - else self._inner.generate_trace_id() - ) - - def generate_span_id(self) -> int: - smuggled = _smuggled_ids.get() - return ( - smuggled[1] - if smuggled is not None - else self._inner.generate_span_id() - ) - - -@runtime_checkable -class _Flushable(Protocol): - """Duck type whether tracer provider can flush.""" - - def force_flush(self) -> bool: ... - def shutdown(self) -> None: ... - - -@telemetry.adapter -class OtelAdapter: - """Maps framework spans onto otel spans.""" - - def __init__( - self, - *, - tracer_provider: opentelemetry.trace.TracerProvider | None = None, - capture_content: bool | None = None, - ) -> None: - provider = tracer_provider or opentelemetry.trace.get_tracer_provider() - self._provider = provider - self._live: dict[str, opentelemetry.trace.Span] = {} - - self._is_capturing_content = ( - _should_capture_content() - if capture_content is None - else capture_content - ) - - self._is_smuggling_ids = False - # printed a warning once when failing to smuggle ids - self._warning_issued = False - - # swap TracerProvider's id generator with a smuggling one - if isinstance(provider, opentelemetry.sdk.trace.TracerProvider): - provider.id_generator = _SmugglingIdGenerator(provider.id_generator) - self._is_smuggling_ids = True - - self._tracer = opentelemetry.trace.get_tracer( - "ai", tracer_provider=provider - ) - - def span_name(self, span: telemetry.Span, /) -> str: - """Return the exported otel span name. Override to customize.""" - return _semconv_name(span) - - def span_attrs(self, span: telemetry.Span, /) -> dict[str, Any]: - """Return the attributes set at span end. Override to enrich. - - :: - - class MyAdapter(otel.OtelAdapter): - def span_attrs(self, span): - return super().span_attrs(span) | {"k": "v"} - """ - return _attributes(span, capture_content=self._is_capturing_content) - - def flush(self) -> None: - """Flush the provider's exporters.""" - # otel *api protocol* doesn't assume the tracer provider would have - # a buffer, so it doesn't declare a flush method. - # otel *sdk's* tracer provider DOES have a buffer, and therefore needs - # to flush. users may also pass their own tracer providers modelled - # after the stock one. - if isinstance(self._provider, _Flushable): - self._provider.force_flush() - - def shutdown(self) -> None: - """Flush and stop the provider; spans pushed after this are lost.""" - if isinstance(self._provider, _Flushable): - self._provider.shutdown() - - def _parent_context( - self, span: telemetry.Span - ) -> opentelemetry.context.Context | None: - # sometimes we need to set a parent span that isn't currently live. - if span.parent_id is None or span.parent_id in self._live: - # here the parent span is actually live, proceed as normal - return None - - # parent is indeed not live - # opentelemetry.io/docs/languages/python/cookbook/ - # 1. wrap the ids in a ``SpanContext`` - # 2. wrap that in a ``NonRecordingSpan`` - # 3. set it in a *fresh* ``Context()`` - - if not self._is_smuggling_ids and not self._warning_issued: - # trying to restore a parent when not smuggling ids - # will result in a broken tree - self._warning_issued = True - logger.warning( - "otel adapter: continuing a trace from another process " - "needs an SDK tracer provider to control span ids; ids " - "will not line up across processes" - ) - return opentelemetry.trace.set_span_in_context( - opentelemetry.trace.NonRecordingSpan( - opentelemetry.trace.SpanContext( - trace_id=_derive_trace_id(span.trace_id), - span_id=_derive_span_id(span.parent_id), - is_remote=True, # marks the parent as living elsewhere - trace_flags=opentelemetry.trace.TraceFlags( - opentelemetry.trace.TraceFlags.SAMPLED - ), - ) - ), - opentelemetry.context.Context(), - ) - - async def __call__( - self, span: telemetry.Span, / - ) -> AsyncGenerator[None, Any]: - parent_context = self._parent_context(span) - - # convert framework's span ids into otel's and prepare for smuggling - smuggled_token = _smuggled_ids.set( - (_derive_trace_id(span.trace_id), _derive_span_id(span.id)) - ) - try: - otel_span = self._tracer.start_span( - self.span_name(span), - context=parent_context, - kind=_semconv_kind(span), - start_time=span.started_at, - ) - finally: - _smuggled_ids.reset(smuggled_token) - - self._live[span.id] = otel_span + from ..telemetry.otel import * # noqa: F403 - try: - # set live span as otel's current, so raw spans user opens in their - # code can nest under it. - with ( - opentelemetry.trace.use_span( - otel_span, - end_on_exit=False, # we end it ourselves below - # errors come it via span.error, not via raising, because - # we need them to serialize - record_exception=False, - set_status_on_exception=False, - ) - if span.set_as_current - else contextlib.nullcontext() - ): - # span end resumes with None - while (ev := (yield)) is not None: - otel_span.add_event( - ev.name, - { - k: v # squash everything into scalars - if isinstance(v, str | bool | int | float) - else repr(v) - for k, v in ev.attrs.items() - }, - timestamp=ev.time_ns, - ) - finally: - self._live.pop(span.id, None) - for key, value in self.span_attrs(span).items(): - otel_span.set_attribute(key, value) - if span.error is not None: - otel_span.set_attribute("error.type", span.error.type) - otel_span.set_status( - opentelemetry.trace.StatusCode.ERROR, - f"{span.error.type}: {span.error.message}", - ) - otel_span.end(end_time=span.ended_at) +sys.modules[__name__] = import_module("ai.telemetry.otel") diff --git a/src/ai/experimental_telemetry/span.py b/src/ai/experimental_telemetry/span.py index 4e913980..dfd96ad3 100644 --- a/src/ai/experimental_telemetry/span.py +++ b/src/ai/experimental_telemetry/span.py @@ -1,1216 +1,10 @@ -"""Telemetry API: spans, sinks, and adapters. +"""Deprecated import path for :mod:`ai.telemetry.span`.""" -Experimental: not part of the stable API, may change or be removed. - -:class:`Span` is a record of work. It carries ids, timestamps, parent id, and -work-specific data:: - - async with ai.experimental_telemetry.span("retrieval") as sp: - sp.set_attrs(query=q) - docs = await search(q) - sp.set_attrs(count=len(docs)) - -Nesting is automatic: the current span is tracked using a context var. - -In cases where the high-level API doesn't work (e.g. durable execution), -selectively using low-level API may be a better fit. - -1. make a span using `create_span` -2. set `.started_at`, `ended_at`, and necessary data fields -3. push updates to the sink one or more times - -:meth:`Span.push` delivers a snapshot of the span to the current *sink*. -The default sink is the adapter registry, meaning on push all registered -adapters will get the updated version of the span:: - - sp = create_span("turn") # identity only, reports nothing - sp.set_attrs(session=sid) - sp.stamp_start() # writes started_at - await sp.push() # visible to sinks/adapters - ... # possibly elsewhere, later: - await sp.stamp_end().push() # complete - -Because ``Span`` is a pydantic model it crosses process boundaries like -any other data (``model_dump`` / ``model_validate``); you can restore it -and set it as current using :func:`use_span` or pass to a child span explicity. - -An adapter processes spans and decides what to do with them:: - - class MyAdapter: - async def on_span_start(self, span): ... - async def on_span_end(self, span): ... - async def on_span_event(self, span, event): ... - - ai.experimental_telemetry.register(MyAdapter()) - -Adapters dispatch on the type of ``span.data``. An adapter that crashes -is logged and skipped, it never kills the run. - -:func:`adapter` builds an adapter from an async generator function or -class, using the pytest fixture-style trick. - -A :class:`Sink` is what sits *before* the adapters and routes span snapshots. -It exists, again, because of the durable execution case: the user can control -whether spans in the current context get sent to telemetry adapters, collected -into a list to be pushed later, or discarded. - -:func:`use_sink` sets the current sink. - -If telemetry is off, i.e. no adapters registered and no sink used, the module -doesn't use random or read the clock. - -A :class:`SpanEvent` is a named, timestamped milestone inside a span's -lifetime (``first_token``, ``hook_resolved``, ...): append it to -``span.events`` and push. - -Span timestamps come from the ambient clock (:func:`now_ns`); -:func:`use_time` overrides it per-context, e.g. with a deterministic -clock inside a durable workflow. -""" - -from __future__ import annotations - -import contextlib -import contextvars -import inspect -import logging -import time -from typing import ( - TYPE_CHECKING, - Annotated, - Any, - ClassVar, - Generic, - Literal, - Protocol, - Self, - cast, - overload, - runtime_checkable, -) - -import pydantic - -# ``typing.TypeVar`` lacks the ``default=`` kwarg on Python <3.13. -# Use the typing_extensions backport so this works on 3.12 too. -from typing_extensions import TypeVar - -from ..types import messages as messages_ -from ..types import usage as usage_ +import sys +from importlib import import_module +from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Iterable, - Mapping, - ) - - from ..models.core import params as params_ - - # 1. RunSpanData and friends are pydantic models. RunSpanData.params is - # typed as InferenceRequestParams. - # 2. InferenceRequestParams.sampling (and other fields) have - # a ModelProviderDefault annotation. - # 3. ModelProviderDefault is a plain setinel class that causes pydantic to - # freak out when building a schema. - - # also, importing without a guard would cause a circular import - # in models/core/api.py - _InferenceParams = params_.InferenceRequestParams -else: - _InferenceParams = Any - -logger = logging.getLogger(__name__) - - -# this api is for plugging a deterministic clock in, so that the sdk -# could be used inside a deterministic workflow body - -_span_clock: contextvars.ContextVar[Callable[[], int] | None] = ( - contextvars.ContextVar("span_clock", default=None) -) - - -def now_ns() -> int: - """Nanoseconds since the epoch, from the ambient span clock. - - The wall clock by default; whatever :func:`use_time` installed - otherwise. - """ - clock = _span_clock.get() - return clock() if clock is not None else time.time_ns() - - -@contextlib.asynccontextmanager -async def use_time(now_ns: Callable[[], int]) -> AsyncIterator[None]: - """Read span timestamps from ``now_ns`` within this context. - - Framework's observability creates timestamps. This API can be - used to plug an approved clock function in durable execution - settings:: - - async with ai.experimental_telemetry.use_time(workflow.time_ns): - ... # spans opened here read time from workflow.time_ns - - This can also be used as a decorator on async functions:: - - @ai.experimental_telemetry.use_time(clock.time_ns) - async def run(...): - ... - """ - token = _span_clock.set(now_ns) - try: - yield - finally: - _span_clock.reset(token) - - -# span data types -# the data type tells you what kind of span it is. - - -class SpanData(Protocol): - """Anything with a ``kind`` is span data. - - Implement it with a pydantic model to make your own typed spans. - - class RetrievalSpanData(pydantic.BaseModel): - kind: Literal["retrieval"] = "retrieval" - query: str - count: int | None = None - - data = RetrievalSpanData(query=q) - async with ai.experimental_telemetry.span(data) as sp: - docs = await search(q) - sp.data.count = len(docs) # typed - - ``kind`` is what travels with the serialized data. Restore with - ``Span[RetrievalSpanData].model_validate(...)`` to get it typed, otherwise - you'll get a plain dict. - """ - - @property - def kind(self) -> str: ... - - -class RunSpanData(pydantic.BaseModel): - """One ``Agent.run``: the whole loop. - - ``blocked``/``final_message``/``usage`` are set at span end: - ``blocked`` is True when the run ended suspended on an unresolved - hook (see ``AgentStream.blocked``), ``final_message`` is the last - assistant message produced, if any, ``usage`` is the sum of usage - across the messages the run produced. - """ - - kind: Literal["run"] = "run" - agent: str - model: str - messages: list[messages_.Message] - provider: str | None = None - tool_names: list[str] | None = None - output_type: str | None = None - params: _InferenceParams | None = None - blocked: bool = False - final_message: messages_.Message | None = None - usage: usage_.Usage | None = None - - -class LoopTurnSpanData(pydantic.BaseModel): - """One turn of the default agent loop. - - Carries no fields — it exists so adapters can group a turn's model - and tool spans. Turn order is given by ``started_at``/``parent_id``. - """ - - kind: Literal["loop_turn"] = "loop_turn" - - -class AiStreamSpanData(pydantic.BaseModel): - """One streaming LLM call.""" - - kind: Literal["ai_stream"] = "ai_stream" - model: str - messages: list[messages_.Message] - params: _InferenceParams | None = None - provider: str | None = None - tool_names: list[str] | None = None - output_type: str | None = None - message: messages_.Message | None = None - usage: usage_.Usage | None = None - finish_reason: str | None = None - response_id: str | None = None - response_model: str | None = None - - -class AiGenerateSpanData(pydantic.BaseModel): - """One buffered (non-streaming) LLM call. - - ``message``/``usage`` and the response identity fields are set at - span end. - """ - - kind: Literal["ai_generate"] = "ai_generate" - model: str - messages: list[messages_.Message] - params: _InferenceParams | None = None - provider: str | None = None - tool_names: list[str] | None = None - output_type: str | None = None - message: messages_.Message | None = None - usage: usage_.Usage | None = None - finish_reason: str | None = None - response_id: str | None = None - response_model: str | None = None - - -class DecideSpanData(pydantic.BaseModel): - """One typed decision operation.""" - - kind: Literal["decide"] = "decide" - model: str - provider: str | None = None - question_count: int - usage: usage_.Usage | None = None - answer_count: int | None = None - warnings: list[dict[str, Any]] | None = None - - -class EmbedSpanData(pydantic.BaseModel): - """One embedding operation.""" - - kind: Literal["embed"] = "embed" - model: str - provider: str | None = None - input_count: int - usage: usage_.Usage | None = None - output_count: int | None = None - dimensions: int | None = None - warnings: list[dict[str, Any]] | None = None - - -class GenerateAudioSpanData(pydantic.BaseModel): - """One audio generation operation.""" - - kind: Literal["generate_audio"] = "generate_audio" - model: str - provider: str | None = None - usage: usage_.Usage | None = None - output_count: int | None = None - warnings: list[dict[str, Any]] | None = None - - -class GenerateImageSpanData(pydantic.BaseModel): - """One image generation operation.""" - - kind: Literal["generate_image"] = "generate_image" - model: str - provider: str | None = None - usage: usage_.Usage | None = None - output_count: int | None = None - warnings: list[dict[str, Any]] | None = None - - -class GenerateVideoSpanData(pydantic.BaseModel): - """One video generation operation.""" - - kind: Literal["generate_video"] = "generate_video" - model: str - provider: str | None = None - usage: usage_.Usage | None = None - output_count: int | None = None - warnings: list[dict[str, Any]] | None = None - - -class RerankSpanData(pydantic.BaseModel): - """One reranking operation.""" - - kind: Literal["rerank"] = "rerank" - model: str - provider: str | None = None - input_count: int - top_n: int | None = None - usage: usage_.Usage | None = None - output_count: int | None = None - warnings: list[dict[str, Any]] | None = None - - -class TranscribeSpanData(pydantic.BaseModel): - """One transcription operation.""" - - kind: Literal["transcribe"] = "transcribe" - model: str - provider: str | None = None - usage: usage_.Usage | None = None - duration_seconds: float | None = None - warnings: list[dict[str, Any]] | None = None - - -class ToolExecutionSpanData(pydantic.BaseModel): - """One tool execution, from dispatch to result. - - ``model_input`` is the value the LLM sees on its next turn, set - only when it differs from ``result`` (aggregator-backed tools). - ``tool_description`` is the tool's model-facing description, - stamped at dispatch. - """ - - kind: Literal["tool_execution"] = "tool_execution" - tool_name: str - tool_call_id: str - tool_description: str | None = None - args: dict[str, Any] | None = None - result: Any = None - model_input: Any = None - is_error: bool = False - - -class HookSpanData(pydantic.BaseModel): - """One hook suspension, from deferred until resolved or cancelled. - - ``tool_call_id`` links the hook to the tool call it suspends, if - any (e.g. approval gating). ``resolution`` carries the resolution - data when the hook resolves. - """ - - kind: Literal["hook"] = "hook" - label: str - hook_type: str - metadata: dict[str, Any] - tool_call_id: str | None = None - status: Literal["pending", "resolved", "cancelled"] = "pending" - resolution: dict[str, Any] | None = None - - -class CustomSpanData(pydantic.BaseModel): - """A user span made with ``span("name")``. - - Attributes are set with :meth:`Span.set_attrs`. - """ - - kind: Literal["custom"] = "custom" - attrs: dict[str, Any] - - -# names for span events shared between producers, -# adapters, and tests -FIRST_TOKEN = "first_token" -RESPONSE_COMPLETE = "response_complete" -HOOK_DEFERRED = "hook_deferred" -HOOK_RESOLVED = "hook_resolved" -HOOK_CANCELLED = "hook_cancelled" - - -class SpanEvent(pydantic.BaseModel): - """A named, timestamped milestone inside a span's lifetime. - - Lives in ``span.events``; the next push delivers it. - :meth:`Span.add_event` is the shorthand for the common case:: - - sp.add_event(FIRST_TOKEN) - await sp.push() - - Ordering: events on one span are delivered in list order; there is - no ordering guarantee across spans. - """ - - name: str - time_ns: int - attrs: dict[str, Any] - - -class SpanError(pydantic.BaseModel): - """A serializable record of the failure that ended a span. - - Spans cross process boundaries, so the error they carry is plain - data, not a live exception. - """ - - type: str - message: str - - @classmethod - def from_exception(cls, exc: BaseException) -> SpanError: - return cls(type=type(exc).__name__, message=str(exc)) - - -_FrameworkData = Annotated[ - RunSpanData - | LoopTurnSpanData - | AiStreamSpanData - | AiGenerateSpanData - | DecideSpanData - | EmbedSpanData - | GenerateAudioSpanData - | GenerateImageSpanData - | GenerateVideoSpanData - | RerankSpanData - | TranscribeSpanData - | ToolExecutionSpanData - | HookSpanData - | CustomSpanData, - pydantic.Field(discriminator="kind"), -] - -# adapters only read the data, they don't swap it out, so it's safe to make it -# covariant. -if TYPE_CHECKING: - DataT_co = TypeVar( - "DataT_co", bound=SpanData, default=SpanData, covariant=True - ) -else: - # try to deserialize to _FrameworkData and fallback to Any - DataT_co = TypeVar("DataT_co", default=_FrameworkData | Any) -# covariant type can't be used for a function's input parameter, and we need -# to type span() -DataT = TypeVar("DataT", bound=SpanData) - - -class Span(pydantic.BaseModel, Generic[DataT_co]): - """A serializable record of a unit of work. - - Generic in its data type: ``span(RetrievalSpanData(...))`` gives a - ``Span[RetrievalSpanData]``, so late assignments to ``sp.data`` - fields are type checked. A bare ``Span`` is ``Span[SpanData]``. - - Lifecycle is encoded in timestamp fields: - - - if ``started_at=None``, this span hasn't started, - - if ``ended_at`` is set, the span is complete. - - Mutate fields, then :meth:`push` to report the new state. Nothing is - reported except by pushing. - - A span round-trips through ``model_dump``/``model_validate`` like - any pydantic model. Use ``Span[MyData].model_validate(...)`` to restore - custom data types. - - ``replay=True`` marks work that is being replayed (resume, - serverless re-entry) rather than performed live. - - A span created while telemetry was off (see :func:`is_enabled`) has an - empty ``id`` and is a noop: :meth:`push` delivers nothing. - - ``set_as_current=False`` marks a span that does not set itself - as a current. This is used by ai.stream's span that stays open for - the duration of a loop turn because of the context manager api. - Adapters must not make those spans "current" either to avoid - nesting discrepancies. - - ``trace_attrs`` is user data that propagates down from parent to child - on creation. - - ``schema_version`` tracks the shape of spans and their data types. - """ - - name: str - data: DataT_co - id: str - trace_id: str - parent_id: str | None = None - started_at: int | None = None # ns since the epoch, see ``now_ns`` - ended_at: int | None = None - error: SpanError | None = None - replay: bool = False - set_as_current: bool = True - events: list[SpanEvent] = pydantic.Field(default_factory=list) - trace_attrs: dict[str, Any] = pydantic.Field(default_factory=dict) - - schema_version: ClassVar[int] = 5 - - def set_attrs( - self, attrs: Mapping[str, Any] | None = None, /, **kwargs: Any - ) -> None: - """Attach attributes to a span created with ``span("name")``. - - Attribute names that aren't valid Python keywords (viewers use - dotted names like ``"output.value"``) go in the positional - mapping; it merges with the keyword arguments:: - - sp.set_attrs({"output.value": title}, model="haiku") - """ - if not isinstance(self.data, CustomSpanData): - raise TypeError( - "set_attrs() only works on user spans; framework " - "spans carry typed data, assign its fields directly" - ) - self.data.attrs.update({**(attrs or {}), **kwargs}) - - def add_event( - self, - name: str, - attrs: Mapping[str, Any] | None = None, - /, - **kwargs: Any, - ) -> SpanEvent: - """Append a named event stamped with the ambient clock. - - Appends only; the event is delivered by the next :meth:`push`, or when - exiting the ``span()`` context manager:: - - sp.add_event(FIRST_TOKEN) - await sp.push() - - Returns the appended event. - """ - event = SpanEvent( - name=name, - # a noop span (telemetry off at creation) reads no clock - time_ns=now_ns() if self.id else 0, - attrs={**(attrs or {}), **kwargs}, - ) - self.events.append(event) - return event - - def stamp_start(self) -> Self: - """Set ``started_at`` from the ambient clock; return the span. - - Only writes the field; nothing is reported until :meth:`push`. - Returns the span so minting reads as one expression:: - - payload = create_span("turn").stamp_start().model_dump() - """ - self.started_at = now_ns() - return self - - def stamp_end( - self, *, error: SpanError | BaseException | None = None - ) -> Self: - """Set ``ended_at`` from the ambient clock; return the span. - - Also records ``error`` when one is given by converting an exception - to a serializable :class:`SpanError`. ``error=None`` - keeps any error already on the span. Only writes fields; - push to report:: - - await sp.stamp_end(error=exc).push() - """ - self.ended_at = now_ns() - if isinstance(error, BaseException): - self.error = SpanError.from_exception(error) - elif error is not None: - self.error = error - return self - - async def push(self) -> None: - """Deliver a frozen copy of this span to the current sink. - - The default sink drives the registered adapters (see - :func:`register`); :func:`use_sink` reroutes pushes within a - context. Each push snapshots the whole span, so keep mutating - and push again as the work progresses. The last push with - ``ended_at`` set is the complete record. - - Telemetry never kills the run: a sink (or adapter) failure is - logged and swallowed. - """ - if not self.id: - return # noop: telemetry was off at creation - sink = _current_sink.get() or _registry_sink - try: - await sink.on_push(self.model_copy(deep=True)) - except Exception: - logger.exception("telemetry sink %r raised in on_push", sink) - - -# utilities for managing the current span - -_current: contextvars.ContextVar[Span | None] = contextvars.ContextVar( - "current_span", default=None -) - - -def current_span() -> Span | None: - """Return the current span, or ``None`` when no span is open.""" - return _current.get() - - -@contextlib.asynccontextmanager -async def use_span(span_: Span | None) -> AsyncIterator[None]: - """Make ``span_`` the current span within this context. - - Unlike :func:`span` this is pure context plumbing with no timestamps - and no pushes. Use it to continue a trace around existing work, e.g. - parenting under a span restored from another process:: - - turn_span = Span.model_validate(payload["turn_span"]) - async with ai.experimental_telemetry.use_span(turn_span): - ... # spans opened here parent under turn_span - - ``None`` is a no-op. - """ - if span_ is None: - yield - return - token = _current.set(span_) - try: - yield - finally: - _current.reset(token) - - -# sinks are used to control where spans go within the current context. -# that could be the adapter registry (so they immediately get sent to the -# observability backend; or it could be a DictSink that allows us to defer -# sending them until later. - - -class Sink(Protocol): - """Anything that accepts pushed span snapshots.""" - - async def on_push(self, span: Span, /) -> None: ... - - -_current_sink: contextvars.ContextVar[Sink | None] = contextvars.ContextVar( - "current_sink", default=None -) - - -@contextlib.asynccontextmanager -async def use_sink(sink: Sink | None) -> AsyncIterator[None]: - """Route span pushes to ``sink`` within this context. - - The default (outside any ``use_sink``) is the adapter registry. - Inside a durable workflow body where side effects are not allowed, you - can route to a :class:`DictSink` instead and re-push the collected spans - from a step / activity. - - ``None`` is a no-op. - """ - if sink is None: - yield - return - token = _current_sink.set(sink) - try: - yield - finally: - _current_sink.reset(token) - - -class DictSink: - """A sink that keeps the latest snapshot of every span pushed to it. - - ``spans`` maps span id to the most recent snapshot, in first-push - order. Scoop the complete ones out as data (:attr:`finished_spans`) and - re-push them where the real sink is available:: - - sink = DictSink() - async with use_sink(sink): - ... # replayed / suspendable code - payload = [s.model_dump(mode="json") for s in sink.finished_spans] - - # elsewhere (a workflow step, another process): - await push_all(payload) - """ - - def __init__(self) -> None: - self.spans: dict[str, Span] = {} - - async def on_push(self, span: Span, /) -> None: - self.spans[span.id] = span - - @property - def finished_spans(self) -> list[Span]: - """The collected spans that have ended, in first-push order.""" - return [s for s in self.spans.values() if s.ended_at is not None] - - -async def push_all(spans: Iterable[Span | Mapping[str, Any]]) -> None: - """Push each span, in order; dumped spans are validated first. - - This is a utility that can be used with :class:`DictSink` to push - a bunch of collected spans all at once:: - - await push_all(payload) - """ - for item in spans: - span = item if isinstance(item, Span) else Span.model_validate(item) - await span.push() - - -class _RegistrySink: - """The default sink: drives registered adapters. - - Translates a stream of span snapshots into - ``on_span_start`` / ``on_span_event`` / ``on_span_end`` callbacks. - - Rules: - - - a snapshot with ``started_at=None`` reports nothing (the span - hasn't started); - - the first snapshot of an id fires ``on_span_start``, then - ``on_span_event`` per event, then ``on_span_end`` if the snapshot - is already complete; so a span that lived elsewhere and arrives - finished is delivered whole; - - later snapshots fire ``on_span_event`` for events not seen - before, and ``on_span_end`` once ``ended_at`` appears; - - after the end the id is forgotten: pushing a completed span again - re-delivers it in full. - """ - - def __init__(self) -> None: - self._views: dict[str, Span] = {} - - async def on_push(self, span: Span, /) -> None: - if span.started_at is None: - return - view = self._views.get(span.id) - if view is None: - view = span - self._views[view.id] = view - await _dispatch(lambda a: a.on_span_start(view)) - fresh = list(view.events) - else: - seen = len(view.events) - # we need to preserve the view's identity - # but update all of the fields - view.__dict__.update(span.__dict__) - fresh = view.events[seen:] - for event in fresh: - # the default binds `event` now; a plain lambda would trip B023 - def call( - a: AdapterProtocol, ev: SpanEvent = event - ) -> Awaitable[None]: - return a.on_span_event(view, ev) - - await _dispatch(call) - if view.ended_at is not None: - del self._views[view.id] - await _dispatch(lambda a: a.on_span_end(view)) - - -_registry_sink = _RegistrySink() - - -# adapter registry utilities - - -@runtime_checkable -class AdapterProtocol(Protocol): - """The async callbacks implemented by a telemetry adapter.""" - - async def on_span_start(self, span: Span, /) -> None: ... - - async def on_span_event(self, span: Span, event: SpanEvent, /) -> None: ... - - async def on_span_end(self, span: Span, /) -> None: ... - - -class AdapterCallable(Protocol): - """The ``__call__`` shape the :func:`adapter` decorator accepts. - - Static-typing companion to :class:`AdapterProtocol`: checkers don't - see the callbacks :func:`adapter` mixes into a decorated class, so - :func:`register` accepts this shape as well. - """ - - def __call__(self, span: Span, /) -> AsyncGenerator[None, Any] | None: ... - - -_adapters: list[AdapterProtocol] = [] - - -def register(adapter: AdapterProtocol | AdapterCallable) -> None: - """Add an adapter. Multiple adapters coexist independently.""" - if not isinstance(adapter, AdapterProtocol): - raise TypeError( - f"{adapter!r} does not implement the adapter callbacks; " - "an adapter class must be decorated with @adapter" - ) - _adapters.append(adapter) - - -def unregister(adapter: AdapterProtocol | AdapterCallable) -> None: - """Remove a previously registered adapter.""" - assert isinstance(adapter, AdapterProtocol) # register() enforced it - _adapters.remove(adapter) - - -def is_enabled() -> bool: - """Whether anything is listening for spans. - - True when a sink is routed with :func:`use_sink` or at least one - adapter is registered. - """ - return _current_sink.get() is not None or bool(_adapters) - - -async def _dispatch( - call: Callable[[AdapterProtocol], Awaitable[None]], -) -> None: - for adapter in list(_adapters): - try: - await call(adapter) - except Exception: - logger.exception("telemetry adapter %r raised", adapter) - - -# span building utilities - - -@overload -def create_span( - name_or_data: str, - /, - *, - parent: Span | None = None, - replay: bool = False, - set_as_current: bool = True, -) -> Span[CustomSpanData]: ... - - -@overload -def create_span( - name_or_data: DataT, - /, - *, - parent: Span | None = None, - replay: bool = False, - set_as_current: bool = True, -) -> Span[DataT]: ... - - -def create_span( - name_or_data: str | SpanData, - /, - *, - parent: Span | None = None, - replay: bool = False, - set_as_current: bool = True, -) -> Span[Any]: - """Create a span: identity only, nothing is reported. - - Mints the span id, and takes trace id, parentage, and a copy of - ``trace_attrs`` from ``parent`` (default: the current span; a fresh - trace when there is none). ``parent`` may be a span restored from - JSON. - - The span has no timestamps yet: :meth:`Span.stamp_start` and - :meth:`Span.push` when the work begins, or hand the whole - lifecycle to the :func:`span` context manager. - - While telemetry is off (see :func:`is_enabled`) this returns a noop - span instead. - """ - if isinstance(name_or_data, str): - name = name_or_data - data: SpanData = CustomSpanData(attrs={}) - else: - name = name_or_data.kind - data = name_or_data - if not is_enabled(): - return Span( - name=name, - data=data, - id="", - trace_id="", - parent_id=None, - replay=replay, - set_as_current=False, - ) - if parent is None: - parent = _current.get() - if parent is None: - trace_id, parent_id = messages_.generate_id("trace"), None - trace_attrs: dict[str, Any] = {} - else: - trace_id, parent_id = parent.trace_id, parent.id - trace_attrs = dict(parent.trace_attrs) - return Span( - name=name, - data=data, - id=messages_.generate_id("span"), - trace_id=trace_id, - parent_id=parent_id, - replay=replay, - set_as_current=set_as_current, - trace_attrs=trace_attrs, - ) - - -@overload -def span( - name_or_data: str, - /, - *, - parent: Span | None = None, - replay: bool = False, - set_as_current: bool = True, -) -> contextlib.AbstractAsyncContextManager[Span[CustomSpanData]]: ... - - -@overload -def span( - name_or_data: DataT, - /, - *, - parent: Span | None = None, - replay: bool = False, - set_as_current: bool = True, -) -> contextlib.AbstractAsyncContextManager[Span[DataT]]: ... - - -def span( - name_or_data: str | SpanData, - /, - *, - parent: Span | None = None, - replay: bool = False, - set_as_current: bool = True, -) -> contextlib.AbstractAsyncContextManager[Span[Any]]: - """Open a span; it sets itself as current inside the block. - - Sugar over the data api: creates the span, stamps ``started_at`` - and pushes on enter; stamps ``ended_at`` (and ``error``, if the - block raised) and pushes on exit. - - Pass a name for a user span (set attributes on it with - :meth:`Span.set_attrs`), or a :class:`SpanData` instance - for a typed one. Exceptions are recorded on the span and - re-raised. - - ``parent`` overrides the ambient parent for this span: a live - :class:`Span`, or one restored from another process to continue its - trace here. The default parents under the current span. - - ``set_as_current=False`` keeps the span from becoming current: - work done while it is open parents to *its* parent instead. Used by - ai.stream because of the context manager api. - """ - # type checkers can't apply asynccontextmanager - # to an overloaded function directly. - return _span_impl( - name_or_data, - parent=parent, - replay=replay, - set_as_current=set_as_current, - ) - - -@contextlib.asynccontextmanager -async def _span_impl( - name_or_data: str | SpanData, - /, - *, - parent: Span | None, - replay: bool, - set_as_current: bool, -) -> AsyncIterator[Span[Any]]: - sp: Span[Any] = create_span( - name_or_data, - parent=parent, - replay=replay, - set_as_current=set_as_current, - ) - if not sp.id: - # noop: no timestamps, no pushes, never current - yield sp - return - sp.stamp_start() - await sp.push() - token = _current.set(sp) if set_as_current else None - try: - yield sp - except BaseException as exc: - # GeneratorExit is how a consumer closes a stream early. - if not isinstance(exc, GeneratorExit) and sp.error is None: - sp.error = SpanError.from_exception(exc) - raise - finally: - # a span that set itself as current must be closed while it is - # still current, in the task that opened it. - misordered = False - if token is not None: - if _current.get() is not sp: - misordered = True - else: - try: - _current.reset(token) - except ValueError: - # token from a different task's context. - misordered = True - sp.ended_at = now_ns() - await sp.push() - if misordered: - raise RuntimeError( - f"span {sp.name!r} closed out of order: it is not the " - "current span at close, or was closed in a different " - "task than it was opened in. A span that sets itself " - "as current must be closed innermost-first in the " - "opening task; open overlapping spans with " - "set_as_current=False." - ) - - -class AdapterMixin(AdapterProtocol): - """The driver :func:`adapter` mixes into a decorated class. - - Implements the adapter callbacks. Don't subclass it directly. Put - :func:`adapter` over a class with ``__call__`` async generator method. - """ - - # Name-mangled so the driver's bookkeeping can never collide with - # an attribute of the class it is mixed into; created lazily so - # that class keeps its ``__init__`` entirely to itself. - __live: dict[str, AsyncGenerator[None, Any]] | None = None - - # Positional-only params on all overridable methods, so overrides - # are free to pick their own names. - def __call__(self, span: Span, /) -> AsyncGenerator[None, Any] | None: - """Return the generator frame for one span, or ``None``. - - The decorated class's ``__call__`` overrides this. - """ - return None - - async def on_span_start(self, span: Span, /) -> None: - gen = self(span) - if gen is None: - return - try: - await anext(gen) - except StopAsyncIteration: - return # returned before yielding: opted out of this span - if self.__live is None: - self.__live = {} - self.__live[span.id] = gen - - async def on_span_event(self, span: Span, event: SpanEvent, /) -> None: - live = self.__live - gen = live.get(span.id) if live is not None else None - if live is None or gen is None: - return # opted out, span already ended, or start raised - try: - await gen.asend(event) - except StopAsyncIteration: - # Finished mid-span: opted out of the rest of this span, - # including its end. - del live[span.id] - except BaseException: - # The generator frame is dead, don't resume it again at - # span end. The error itself is logged by _dispatch. - del live[span.id] - raise - - async def on_span_end(self, span: Span, /) -> None: - gen = self.__live.pop(span.id, None) if self.__live else None - if gen is None: # opted out, or start raised (and was logged) - return - try: - await anext(gen) - except StopAsyncIteration: - return - await gen.aclose() - raise RuntimeError("adapter generator yielded again after span end") - - -class _AdapterFn(AdapterMixin): - """Adapter built by :func:`adapter` from a function.""" - - def __init__(self, fn: Callable[[Span], AsyncGenerator[None, Any]]) -> None: - self._fn = fn - - def __repr__(self) -> str: - return f"adapter({getattr(self._fn, '__qualname__', self._fn)!r})" - - def __call__(self, span: Span, /) -> AsyncGenerator[None, Any]: - return self._fn(span) - - -_AdapterCallableT = TypeVar("_AdapterCallableT", bound=AdapterCallable) - - -@overload -def adapter(target: type[_AdapterCallableT]) -> type[_AdapterCallableT]: ... - - -@overload -def adapter( - target: Callable[[Span], AsyncGenerator[None, Any]], -) -> AdapterMixin: ... - - -def adapter( - target: type[_AdapterCallableT] - | Callable[[Span], AsyncGenerator[None, Any]], -) -> type[_AdapterCallableT] | AdapterMixin: - """Build an adapter from an async generator function or class. - - Decorating a free-standing async generator function returns a - ready-to-register adapter. - - Decorating a class whose ``__call__`` is an - async generator method returns that class with the adapter - machinery (:class:`AdapterMixin`) mixed in:: - - @adapter - class Vendor: - def __init__(self, *, api_key): - self._client = sdk.Client(api_key) - - async def __call__(self, span): - ... # same yield loop as the function form - - ai.experimental_telemetry.register(Vendor(api_key="...")) - - How to use to bridge a vendor SDKs: - write the vendor's ``with``/``async with`` around one yield loop. - Code before the loop runs at span start; each span event resumes the - yield with the :class:`SpanEvent`, live; span end resumes it with - ``None``, and the code after the loop runs with ``span.data`` - fully populated and ``ended_at`` set:: - - @adapter - async def vendor(span): - with sdk.start_span(span.name) as v: - while (ev := (yield)) is not None: # each event, live - v.log_event(ev.name, timestamp=ev.time_ns) - v.update(output=span.data) # span end - - ai.experimental_telemetry.register(vendor) + from ..telemetry.span import * # noqa: F403 - - A failed span ends the loop normally, like any other; read - ``span.error`` (a serializable :class:`SpanError`, never a live - exception — the span may have failed in another process) after - the loop to report it. - - A span that lived elsewhere and arrives complete is replayed to - the generator as start, events, end, back to back — write the - bridge against the span's fields and it handles both live and - after-the-fact delivery. - - Returning before the first ``yield`` skips that span — cheap - filtering by span type. Returning mid-span, from inside the - loop, opts out of the rest of that span, including its end. - - Don't make a bare ``yield`` that ignores the sent value, because - it runs the code after it on the first event, not at span - end. Always loop until the yield returns ``None``. - - Like any adapter, a generator that raises is logged and skipped; - it never kills the run. - """ - if inspect.isclass(target): - if issubclass(target, AdapterMixin): - raise TypeError(f"{target.__name__} is already an adapter") - call = inspect.getattr_static(target, "__call__", None) - if not inspect.isasyncgenfunction(call): - raise TypeError( - "@adapter on a class requires __call__ to be an async " - "generator method (`async def` containing a `yield` loop)" - ) - mixed = type( - target.__name__, - (target, AdapterMixin), - { - "__module__": target.__module__, - "__qualname__": target.__qualname__, - "__doc__": target.__doc__, - }, - ) - return cast("type[_AdapterCallableT]", mixed) - if not inspect.isasyncgenfunction(target): - raise TypeError( - "@adapter requires an async generator function " - "(`async def` containing a `yield` loop)" - ) - return _AdapterFn(target) +sys.modules[__name__] = import_module("ai.telemetry.span") diff --git a/src/ai/experimental_telemetry/utils/__init__.py b/src/ai/experimental_telemetry/utils/__init__.py index 6330bd76..347a9ae0 100644 --- a/src/ai/experimental_telemetry/utils/__init__.py +++ b/src/ai/experimental_telemetry/utils/__init__.py @@ -1,8 +1 @@ -"""Debugging helpers: the console adapter and the terminal trace viewer. - -Experimental: not part of the stable API, may change or be removed. - -Import the submodules directly:: - - from ai.experimental_telemetry.utils import console -""" +"""Deprecated import path for :mod:`ai.telemetry.utils`.""" diff --git a/src/ai/experimental_telemetry/utils/console.py b/src/ai/experimental_telemetry/utils/console.py index 82d2c6bf..6a17aca5 100644 --- a/src/ai/experimental_telemetry/utils/console.py +++ b/src/ai/experimental_telemetry/utils/console.py @@ -1,155 +1,10 @@ -"""Console adapter: prints spans to a terminal as they happen. - -Experimental: not part of the stable API, may change or be removed. - -Register it like any adapter:: - - from ai.experimental_telemetry.utils import console - ai.experimental_telemetry.register(console.ConsoleAdapter()) - -Prints one ``▸`` line when a span starts (so long runs are visible -live) and, when a trace's root span ends, the whole tree with -durations. -""" - -from __future__ import annotations +"""Deprecated import path for :mod:`ai.telemetry.utils.console`.""" import sys +from importlib import import_module from typing import TYPE_CHECKING -from ... import experimental_telemetry as telemetry - if TYPE_CHECKING: - from typing import TextIO - - -def _short(text: str, limit: int = 60) -> str: - return text if len(text) <= limit else text[: limit - 1] + "…" - - -def _label(sp: telemetry.Span) -> str: - match sp.data: - case telemetry.AiStreamSpanData() as d: - tokens = ( - f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" - if d.usage is not None - else "" - ) - return f"chat {d.model}{tokens}" - case telemetry.AiGenerateSpanData() as d: - tokens = ( - f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" - if d.usage is not None - else "" - ) - return f"generate {d.model}{tokens}" - case telemetry.DecideSpanData() as d: - return f"decide {d.model}" - case telemetry.EmbedSpanData() as d: - tokens = ( - f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" - if d.usage is not None - else "" - ) - return f"embed {d.model}{tokens}" - case telemetry.GenerateAudioSpanData() as d: - return f"generate_audio {d.model}" - case telemetry.GenerateImageSpanData() as d: - return f"generate_image {d.model}" - case telemetry.GenerateVideoSpanData() as d: - return f"generate_video {d.model}" - case telemetry.RerankSpanData() as d: - return f"rerank {d.model}" - case telemetry.TranscribeSpanData() as d: - return f"transcribe {d.model}" - case telemetry.ToolExecutionSpanData() as d: - args = ", ".join(f"{k}={v!r}" for k, v in (d.args or {}).items()) - return f"tool {d.tool_name}({_short(args)})" - case telemetry.HookSpanData() as d: - return f"hook {d.label} {d.hook_type} [{d.status}]" - case telemetry.RunSpanData() as d: - return f"run {d.agent} ({d.model})" - case telemetry.CustomSpanData() as d: - attrs = ", ".join(f"{k}={v!r}" for k, v in d.attrs.items()) - return sp.name + (f" ({_short(attrs)})" if attrs else "") - case _: - return sp.name - - -def _line(sp: telemetry.Span) -> str: - started = sp.started_at or 0 - end = sp.ended_at or started - # A span's lifetime can extend past the response (tool dispatch - # while the stream is open); when the milestone is there, report - # the true response latency instead. - for ev in sp.events: - if ev.name == telemetry.RESPONSE_COMPLETE: - end = ev.time_ns - break - duration = (end - started) / 1e9 - replay = "↻ " if sp.replay else "" - error = ( - f" ✗ {sp.error.type}: {_short(sp.error.message)}" - if sp.error is not None - else "" - ) - return f"{replay}{_label(sp)} {duration:.2f}s{error}" - - -class ConsoleAdapter: - """Print spans to ``out`` (default: stdout).""" - - def __init__(self, *, out: TextIO | None = None) -> None: - self._out = out if out is not None else sys.stdout - self._depth: dict[str, int] = {} - self._ended: dict[str, list[telemetry.Span]] = {} - - async def on_span_start(self, span: telemetry.Span) -> None: - depth = 0 - if span.parent_id is not None: - depth = self._depth.get(span.parent_id, -1) + 1 - self._depth[span.id] = depth - replay = "↻ " if span.replay else "" - self._out.write(f"▸ {' ' * depth}{replay}{_label(span)}\n") - - async def on_span_event( - self, span: telemetry.Span, event: telemetry.SpanEvent - ) -> None: - depth = self._depth.get(span.id, 0) + 1 - offset_ms = (event.time_ns - (span.started_at or 0)) / 1e6 - attrs = ", ".join(f"{k}={v!r}" for k, v in event.attrs.items()) - suffix = f" ({_short(attrs)})" if attrs else "" - self._out.write( - f"· {' ' * depth}{event.name} +{offset_ms:.0f}ms{suffix}\n" - ) - - async def on_span_end(self, span: telemetry.Span) -> None: - self._ended.setdefault(span.trace_id, []).append(span) - if span.parent_id is not None: - return - - # Root ended: print the tree and forget the trace. - spans = self._ended.pop(span.trace_id) - for s in spans: - self._depth.pop(s.id, None) - children: dict[str | None, list[telemetry.Span]] = {} - for s in spans: - children.setdefault(s.parent_id, []).append(s) - - lines = [f"trace {span.trace_id}"] - - def render(s: telemetry.Span, prefix: str, kid_prefix: str) -> None: - lines.append(prefix + _line(s)) - kids = sorted( - children.get(s.id, []), key=lambda c: c.started_at or 0 - ) - for i, kid in enumerate(kids): - last = i == len(kids) - 1 - render( - kid, - kid_prefix + ("└─ " if last else "├─ "), - kid_prefix + (" " if last else "│ "), - ) + from ...telemetry.utils.console import * # noqa: F403 - render(span, "", "") - self._out.write("\n".join(lines) + "\n") +sys.modules[__name__] = import_module("ai.telemetry.utils.console") diff --git a/src/ai/experimental_telemetry/utils/viewer.py b/src/ai/experimental_telemetry/utils/viewer.py index d1702fe0..c6f1e41d 100644 --- a/src/ai/experimental_telemetry/utils/viewer.py +++ b/src/ai/experimental_telemetry/utils/viewer.py @@ -1,133 +1,15 @@ -"""Terminal trace viewer: an OTLP/HTTP server that prints span trees. +"""Deprecated import path for :mod:`ai.telemetry.utils.viewer`.""" -Experimental: not part of the stable API, may change or be removed. - -Run:: - - python -m ai.experimental_telemetry.utils.viewer [--port 4318] - -and point any OTLP/HTTP exporter at ``http://127.0.0.1:4318/v1/traces`` -(the OTLP/HTTP default); see -https://opentelemetry.io/docs/languages/python/exporters/#otlp for the -exporter setup. Then register the adapter:: - - import ai - from ai.experimental_telemetry import otel - - ai.experimental_telemetry.register(otel.OtelAdapter()) - -Spans are buffered per trace, and the tree is printed when the trace's -root span arrives (roots end — and therefore export — last). -""" - -from __future__ import annotations - -import argparse import sys -from http import server -from typing import Any - -from ... import errors - -try: - from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 -except ModuleNotFoundError as exc: # pragma: no cover - raise errors.InstallationError( - "could not import `opentelemetry.proto`, which the telemetry viewer " - "needs to decode OTLP, you can install it with " - "`pip install opentelemetry-exporter-otlp-proto-http`" - ) from exc - -_traces: dict[bytes, list[Any]] = {} - - -def _value(any_value: Any) -> Any: - kind = any_value.WhichOneof("value") - return getattr(any_value, kind) if kind else None - - -def _line(span: Any) -> str: - attrs = {kv.key: _value(kv.value) for kv in span.attributes} - duration = (span.end_time_unix_nano - span.start_time_unix_nano) / 1e9 - tokens = "" - if "gen_ai.usage.input_tokens" in attrs: - tokens = ( - f" in:{attrs['gen_ai.usage.input_tokens']}" - f" out:{attrs.get('gen_ai.usage.output_tokens', '?')} tok" - ) - replay = "↻ " if attrs.get("ai.replay") else "" - error = " ✗" if span.status.code == 2 else "" # STATUS_CODE_ERROR - return f"{replay}{span.name}{tokens} {duration:.2f}s{error}" - - -def _print_trace(spans: list[Any]) -> None: - children: dict[bytes, list[Any]] = {} - roots = [] - for span in spans: - if span.parent_span_id: - children.setdefault(span.parent_span_id, []).append(span) - else: - roots.append(span) - - lines = [f"trace {spans[0].trace_id.hex()}"] - - def render(span: Any, prefix: str, kid_prefix: str) -> None: - lines.append(prefix + _line(span)) - kids = sorted( - children.get(span.span_id, []), - key=lambda s: s.start_time_unix_nano, - ) - for i, kid in enumerate(kids): - last = i == len(kids) - 1 - render( - kid, - kid_prefix + ("└─ " if last else "├─ "), - kid_prefix + (" " if last else "│ "), - ) - - for root in roots: - render(root, "", "") - sys.stdout.write("\n".join(lines) + "\n") - sys.stdout.flush() - - -class _Handler(server.BaseHTTPRequestHandler): - def do_POST(self) -> None: # noqa: N802 — stdlib API name - length = int(self.headers.get("content-length") or 0) - request = trace_service_pb2.ExportTraceServiceRequest() - request.ParseFromString(self.rfile.read(length)) - for resource_spans in request.resource_spans: - for scope_spans in resource_spans.scope_spans: - for span in scope_spans.spans: - _traces.setdefault(span.trace_id, []).append(span) - if not span.parent_span_id: - _print_trace(_traces.pop(span.trace_id)) - payload = trace_service_pb2.ExportTraceServiceResponse() - body = payload.SerializeToString() - self.send_response(200) - self.send_header("content-type", "application/x-protobuf") - self.send_header("content-length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: Any) -> None: - """Silence per-request logging.""" - +from importlib import import_module +from typing import TYPE_CHECKING -def main() -> None: - parser = argparse.ArgumentParser( - description="Print OTLP traces to the terminal as trees." - ) - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=4318) - args = parser.parse_args() - httpd = server.HTTPServer((args.host, args.port), _Handler) - sys.stdout.write(f"listening on http://{args.host}:{args.port}/v1/traces\n") - try: - httpd.serve_forever() - except KeyboardInterrupt: - pass +from ...telemetry.utils.viewer import main +if TYPE_CHECKING: + from ...telemetry.utils.viewer import * # noqa: F403 if __name__ == "__main__": main() +else: + sys.modules[__name__] = import_module("ai.telemetry.utils.viewer") diff --git a/src/ai/external/_harness.py b/src/ai/external/_harness.py index d89a61aa..743e6016 100644 --- a/src/ai/external/_harness.py +++ b/src/ai/external/_harness.py @@ -148,7 +148,7 @@ async def generate( output_type: type[pydantic.BaseModel] | None = None, params: params_.InferenceRequestParams | None = None, ) -> messages_.Message: - """Not implemented: `ai.experimental_generate` drains `stream()`.""" + """Not implemented: `ai.generate` drains `stream()`.""" raise NotImplementedError diff --git a/src/ai/external/_session.py b/src/ai/external/_session.py index 5277aeb1..2ad269b0 100644 --- a/src/ai/external/_session.py +++ b/src/ai/external/_session.py @@ -19,7 +19,7 @@ import pydantic -from .. import experimental_telemetry as telemetry +from .. import telemetry from ..types import events as events_ from ..types import messages as messages_ from ..types import tools as tools_ diff --git a/src/ai/models/__init__.py b/src/ai/models/__init__.py index eb9f80ac..ca5a6336 100644 --- a/src/ai/models/__init__.py +++ b/src/ai/models/__init__.py @@ -32,10 +32,13 @@ ids = await ai.get_provider("openai").list_models() """ +import warnings +from typing import TYPE_CHECKING + from ..providers.base import Provider, ProviderProtocol from .core.api import ( Stream, - experimental_generate, + generate, probe, stream, ) @@ -111,7 +114,23 @@ "TopPSamplerParams", "Unset", "experimental_generate", + "generate", "get_model", "probe", "stream", ] + +if TYPE_CHECKING: + experimental_generate = generate +else: + + def __getattr__(name: str) -> object: + if name == "experimental_generate": + warnings.warn( + f"{__name__}.experimental_generate is deprecated; " + "use ai.generate", + DeprecationWarning, + stacklevel=2, + ) + return generate + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ai/models/core/__init__.py b/src/ai/models/core/__init__.py index 41c7596e..579c144e 100644 --- a/src/ai/models/core/__init__.py +++ b/src/ai/models/core/__init__.py @@ -1,10 +1,13 @@ """Core types for models.""" +import warnings +from typing import TYPE_CHECKING + from ...providers.base import Provider from . import helpers from .api import ( Stream, - experimental_generate, + generate, probe, stream, ) @@ -77,8 +80,24 @@ "TopPSamplerParams", "Unset", "experimental_generate", + "generate", "get_model", "helpers", "probe", "stream", ] + +if TYPE_CHECKING: + experimental_generate = generate +else: + + def __getattr__(name: str) -> object: + if name == "experimental_generate": + warnings.warn( + f"{__name__}.experimental_generate is deprecated; " + "use ai.generate", + DeprecationWarning, + stacklevel=2, + ) + return generate + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ai/models/core/api.py b/src/ai/models/core/api.py index 117fb65d..957b647e 100644 --- a/src/ai/models/core/api.py +++ b/src/ai/models/core/api.py @@ -2,6 +2,7 @@ import contextlib import dataclasses +import warnings from contextlib import AbstractAsyncContextManager from typing import ( TYPE_CHECKING, @@ -18,10 +19,9 @@ # ``typing.TypeVar`` lacks the ``default=`` kwarg on Python <3.13. # Use the typing_extensions backport so this works on 3.12 too. -from typing_extensions import TypeVar +from typing_extensions import TypeVar, deprecated -from ... import errors, types -from ... import experimental_telemetry as telemetry +from ... import errors, telemetry, types if TYPE_CHECKING: from collections.abc import AsyncGenerator, AsyncIterator, Sequence @@ -240,14 +240,11 @@ async def __anext__(self: Self) -> types.events.Event: return event @property - def experimental_span( + def telemetry_span( self, ) -> telemetry.Span[telemetry.AiStreamSpanData] | None: """The telemetry span bracketing this stream. - Experimental: not part of the stable API, may change or be - removed. - Set when the stream came from :func:`stream` (live and replay both); ``None`` for directly constructed streams. Lets consumers of the live event stream correlate what they see @@ -255,6 +252,16 @@ def experimental_span( """ return self._span + @property + @deprecated( + "Stream.experimental_span is deprecated; use Stream.telemetry_span" + ) + def experimental_span( + self, + ) -> telemetry.Span[telemetry.AiStreamSpanData] | None: + """Deprecated alias for :attr:`telemetry_span`.""" + return self.telemetry_span + @property def message(self) -> types.messages.Message: return self._hydrator.message @@ -464,14 +471,14 @@ async def _stream( @overload -async def experimental_generate( +async def generate( *, context: StreamContext, output_type: type[pydantic.BaseModel] | None = None, params: params_.InferenceRequestParams | None = None, ) -> types.messages.Message: ... @overload -async def experimental_generate( +async def generate( model: model_.ModelLike, messages: list[types.messages.Message], *, @@ -479,7 +486,7 @@ async def experimental_generate( output_type: type[pydantic.BaseModel] | None = None, params: params_.InferenceRequestParams | None = None, ) -> types.messages.Message: ... -async def experimental_generate( +async def generate( model: model_.ModelLike | None = None, messages: list[types.messages.Message] | None = None, *, @@ -491,19 +498,17 @@ async def experimental_generate( """Generate an LLM response and return the complete message. :: - message = await ai.experimental_generate(model, messages) - message = await ai.experimental_generate(context=context) + message = await ai.generate(model, messages) + message = await ai.generate(context=context) When the provider implements a native non-streaming ``generate()``, it is used; otherwise the response is streamed internally, drained, and the resulting message returned. - - Experimental: not part of the stable API, may change or be removed. """ if context is not None: if model is not None or messages is not None or tools is not None: raise TypeError( - "experimental_generate() takes either model/messages/tools " + "generate() takes either model/messages/tools " "or context=, not both" ) model = context.model @@ -515,8 +520,7 @@ async def experimental_generate( params = context.params elif model is None or messages is None: raise TypeError( - "experimental_generate() requires either model and messages " - "or context=" + "generate() requires either model and messages " "or context=" ) request = _StreamRequest( @@ -566,3 +570,19 @@ async def experimental_generate( async def probe(model: model_.Model) -> None: """Raise unless the model's provider is reachable and the model exists.""" await model.provider.probe(model) + + +if TYPE_CHECKING: + experimental_generate = generate +else: + + def __getattr__(name: str) -> object: + if name == "experimental_generate": + warnings.warn( + f"{__name__}.experimental_generate is deprecated; " + "use ai.generate", + DeprecationWarning, + stacklevel=2, + ) + return generate + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ai/models/core/model.py b/src/ai/models/core/model.py index 89d1ae7d..40babfd6 100644 --- a/src/ai/models/core/model.py +++ b/src/ai/models/core/model.py @@ -64,7 +64,7 @@ async def generate( """Answer without streaming. Optional: raise `NotImplementedError`, and - `ai.experimental_generate` drains `stream` instead. + `ai.generate` drains `stream` instead. """ ... diff --git a/src/ai/ops/audio.py b/src/ai/ops/audio.py index a5b7caef..4d6f72ee 100644 --- a/src/ai/ops/audio.py +++ b/src/ai/ops/audio.py @@ -15,7 +15,7 @@ import dataclasses from typing import TYPE_CHECKING, Any -from .. import experimental_telemetry as telemetry +from .. import telemetry if TYPE_CHECKING: from collections.abc import Mapping diff --git a/src/ai/ops/embeddings.py b/src/ai/ops/embeddings.py index 463895a4..e6ab79f7 100644 --- a/src/ai/ops/embeddings.py +++ b/src/ai/ops/embeddings.py @@ -15,7 +15,7 @@ import dataclasses from typing import TYPE_CHECKING, Any -from .. import experimental_telemetry as telemetry +from .. import telemetry if TYPE_CHECKING: from collections.abc import Mapping diff --git a/src/ai/ops/experimental/decisions.py b/src/ai/ops/experimental/decisions.py index cacb073c..e0d9578c 100644 --- a/src/ai/ops/experimental/decisions.py +++ b/src/ai/ops/experimental/decisions.py @@ -37,7 +37,7 @@ class Answers(pydantic.BaseModel): import pydantic -from ... import experimental_telemetry as telemetry +from ... import telemetry from ...types import messages from .. import items diff --git a/src/ai/ops/images.py b/src/ai/ops/images.py index 715b1b46..b24f8f70 100644 --- a/src/ai/ops/images.py +++ b/src/ai/ops/images.py @@ -15,7 +15,7 @@ import dataclasses from typing import TYPE_CHECKING, Any -from .. import experimental_telemetry as telemetry +from .. import telemetry if TYPE_CHECKING: from collections.abc import Mapping, Sequence diff --git a/src/ai/ops/reranking.py b/src/ai/ops/reranking.py index 42164fa9..f847ef1d 100644 --- a/src/ai/ops/reranking.py +++ b/src/ai/ops/reranking.py @@ -18,7 +18,7 @@ import pydantic -from .. import experimental_telemetry as telemetry +from .. import telemetry from . import items if TYPE_CHECKING: diff --git a/src/ai/ops/transcriptions.py b/src/ai/ops/transcriptions.py index 91013202..666a5e2a 100644 --- a/src/ai/ops/transcriptions.py +++ b/src/ai/ops/transcriptions.py @@ -18,7 +18,7 @@ import pydantic -from .. import experimental_telemetry as telemetry +from .. import telemetry if TYPE_CHECKING: from collections.abc import Mapping diff --git a/src/ai/ops/videos.py b/src/ai/ops/videos.py index 865d626c..90868466 100644 --- a/src/ai/ops/videos.py +++ b/src/ai/ops/videos.py @@ -15,7 +15,7 @@ import dataclasses from typing import TYPE_CHECKING, Any, Literal -from .. import experimental_telemetry as telemetry +from .. import telemetry if TYPE_CHECKING: from collections.abc import Mapping, Sequence diff --git a/src/ai/providers/base.py b/src/ai/providers/base.py index b3ed61e8..efe146ba 100644 --- a/src/ai/providers/base.py +++ b/src/ai/providers/base.py @@ -172,7 +172,7 @@ async def generate( """Generate a non-streaming language-model response using *client*. Optional: protocols that don't implement it fall back to the - streaming path in :func:`ai.experimental_generate`. + streaming path in :func:`ai.generate`. """ raise NotImplementedError( f"protocol {type(self).__name__!r} does not support generate()" @@ -516,7 +516,7 @@ async def generate( """Generate a non-streaming response from this provider. Optional: providers that don't implement it fall back to the - streaming path in :func:`ai.experimental_generate`. + streaming path in :func:`ai.generate`. """ selected_protocol = model.protocol or self.protocol return await selected_protocol.generate( diff --git a/src/ai/telemetry/__init__.py b/src/ai/telemetry/__init__.py new file mode 100644 index 00000000..d9fa7a6a --- /dev/null +++ b/src/ai/telemetry/__init__.py @@ -0,0 +1,109 @@ +"""Telemetry: spans, sinks, adapters, and the ambient current span. + +See :mod:`.span` for the full story. +""" + +import warnings +from typing import TYPE_CHECKING, Any + +from .span import ( + FIRST_TOKEN, + HOOK_CANCELLED, + HOOK_DEFERRED, + HOOK_RESOLVED, + RESPONSE_COMPLETE, + AdapterCallable, + AdapterMixin, + AdapterProtocol, + AiGenerateSpanData, + AiStreamSpanData, + CustomSpanData, + DecideSpanData, + DictSink, + EmbedSpanData, + GenerateAudioSpanData, + GenerateImageSpanData, + GenerateVideoSpanData, + HookSpanData, + LoopTurnSpanData, + RerankSpanData, + RunSpanData, + Sink, + Span, + SpanData, + SpanError, + SpanEvent, + ToolExecutionSpanData, + TranscribeSpanData, + adapter, + create_span, + current_span, + is_enabled, + now_ns, + push_all, + register, + span, + unregister, + use_sink, + use_span, + use_time, +) + +__all__ = [ + "FIRST_TOKEN", + "HOOK_CANCELLED", + "HOOK_DEFERRED", + "HOOK_RESOLVED", + "RESPONSE_COMPLETE", + "AdapterCallable", + "AdapterMixin", + "AdapterProtocol", + "AiGenerateSpanData", + "AiStreamSpanData", + "CustomSpanData", + "DecideSpanData", + "DictSink", + "EmbedSpanData", + "GenerateAudioSpanData", + "GenerateImageSpanData", + "GenerateVideoSpanData", + "HookSpanData", + "LoopTurnSpanData", + "RerankSpanData", + "RunSpanData", + "Sink", + "Span", + "SpanData", + "SpanError", + "SpanEvent", + "ToolExecutionSpanData", + "TranscribeSpanData", + "adapter", + "create_span", + "current_span", + "is_enabled", + "now_ns", + "push_all", + "register", + "span", + "unregister", + "use_sink", + "use_span", + "use_time", +] + +if TYPE_CHECKING: + EvaluateSpanData = DecideSpanData +else: + + def __getattr__(name: str) -> Any: + # Deprecated names, kept as aliases of their replacements. + if name == "EvaluateSpanData": + warnings.warn( + "ai.telemetry.EvaluateSpanData is deprecated, " + "use ai.telemetry.DecideSpanData", + DeprecationWarning, + stacklevel=2, + ) + return DecideSpanData + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ai/telemetry/otel.py b/src/ai/telemetry/otel.py new file mode 100644 index 00000000..1a76905d --- /dev/null +++ b/src/ai/telemetry/otel.py @@ -0,0 +1,662 @@ +"""OpenTelemetry adapter. + +:: + + import ai + from ai.telemetry import otel + + ai.telemetry.register(otel.OtelAdapter()) + +The adapter uses the global tracer provider by default. To set up a +tracer provider and OTLP export, see the OpenTelemetry docs: +https://opentelemetry.io/docs/languages/python/exporters/#otlp + +Follows the ``gen_ai`` semantic conventions. + +There's a certain amount of jank in this adapter, because we want it to work in +the durable execution case, which conflicts with the OTel spec. + +Messages are emitted in the semconv shape for maximum downstream compatibility. +Capturing message content is opt-in. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import hashlib +import json +import logging +import os +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +from .. import errors, telemetry + +try: + import opentelemetry.context + import opentelemetry.sdk.trace.id_generator + import opentelemetry.trace +except ModuleNotFoundError as exc: # pragma: no cover + raise errors.InstallationError( + "could not import `opentelemetry`, which is required for the otel " + 'telemetry adapter, you can install it with `pip install "ai[otel]"` ' + 'or `uv add "ai[otel]"`' + ) from exc + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from ..types import messages as messages_ + from ..types import usage as usage_ + +logger = logging.getLogger(__name__) + + +# standard flag for capturing message content +CAPTURE_CONTENT_ENV = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" + + +def _should_capture_content() -> bool: + value = os.environ.get(CAPTURE_CONTENT_ENV, "") + return value.strip().lower() in ("true", "span_only", "span_and_event") + + +def _semconv_name(sp: telemetry.Span) -> str: + match sp.data: + case ( + (telemetry.AiStreamSpanData() | telemetry.AiGenerateSpanData()) as d + ): + return f"chat {d.model}" + case telemetry.DecideSpanData() as d: + return f"decide {d.model}" + case telemetry.EmbedSpanData() as d: + return f"embeddings {d.model}" + case ( + ( + telemetry.GenerateAudioSpanData() + | telemetry.GenerateImageSpanData() + | telemetry.GenerateVideoSpanData() + ) as d + ): + return f"generate_content {d.model}" + case telemetry.RerankSpanData() as d: + return f"rerank {d.model}" + case telemetry.TranscribeSpanData() as d: + return f"transcribe {d.model}" + case telemetry.ToolExecutionSpanData() as d: + return f"execute_tool {d.tool_name}" + case telemetry.RunSpanData() as d: + return f"invoke_agent {d.agent}" + case _: + return sp.name + + +def _semconv_kind(sp: telemetry.Span) -> opentelemetry.trace.SpanKind: + # Inference spans SHOULD be CLIENT per semconv; agent and tool + # spans run in-process and stay INTERNAL. + match sp.data: + case ( + telemetry.AiStreamSpanData() + | telemetry.AiGenerateSpanData() + | telemetry.DecideSpanData() + | telemetry.EmbedSpanData() + | telemetry.GenerateAudioSpanData() + | telemetry.GenerateImageSpanData() + | telemetry.GenerateVideoSpanData() + | telemetry.RerankSpanData() + | telemetry.TranscribeSpanData() + ): + return opentelemetry.trace.SpanKind.CLIENT + case _: + return opentelemetry.trace.SpanKind.INTERNAL + + +def _maybe_json_loads(raw: str) -> Any: + try: + return json.loads(raw) + except ValueError: + return raw + + +def _semconv_part(part: dict[str, Any]) -> dict[str, Any]: + """Convert part to gen_ai semconv shape. + + https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/non-normative/models.py + """ + match part["kind"]: + case "text": + return {"type": "text", "content": part["text"]} + case "reasoning": + return {"type": "reasoning", "content": part["text"]} + case "tool_call": + return { + "type": "tool_call", + "id": part["tool_call_id"], + "name": part["tool_name"], + "arguments": _maybe_json_loads(part["tool_args"]), + } + case "tool_result": + return { + "type": "tool_call_response", + "id": part["tool_call_id"], + "response": part["result"], + } + case "builtin_tool_call": + return { + "type": "server_tool_call", + "id": part["tool_call_id"], + "name": part["tool_name"], + "server_tool_call": { + "type": part["tool_name"], + "arguments": _maybe_json_loads(part["tool_args"]), + }, + } + case "builtin_tool_return": + return { + "type": "server_tool_call_response", + "id": part["tool_call_id"], + "server_tool_call_response": { + "type": part["tool_name"], + "response": part["result"], + }, + } + case "file": + media_type = part["media_type"] + modality = media_type.split("/")[0] + if modality not in ("image", "video", "audio"): + modality = "document" + # data is a str after the JSON dump: a URL or base-64 + # content. + if part["data"].startswith(("http://", "https://")): + return { + "type": "uri", + "modality": modality, + "mime_type": media_type, + "uri": part["data"], + } + return { + "type": "blob", + "modality": modality, + "mime_type": media_type, + "content": part["data"], + } + case kind: + return {"type": kind} + + +def _semconv_parts( + message: messages_.Message, +) -> tuple[str, list[dict[str, Any]]]: + dumped = message.model_dump(mode="json", fallback=str) + return dumped["role"], [_semconv_part(p) for p in dumped["parts"]] + + +def _semconv_content_attributes( + messages: list[messages_.Message], + output: messages_.Message | None, + *, + error: bool, + finish_reason: str | None = None, +) -> dict[str, str]: + """Extract and convert message content attributes.""" + system_parts: list[dict[str, Any]] = [] + inputs: list[dict[str, Any]] = [] + for message in messages: + role, parts = _semconv_parts(message) + if role == "system": + # these live in gen_ai.system_instructions + system_parts += parts + else: + inputs.append({"role": role, "parts": parts}) + attrs = {"gen_ai.input.messages": json.dumps(inputs)} + if system_parts: + attrs["gen_ai.system_instructions"] = json.dumps(system_parts) + if output is not None: + role, parts = _semconv_parts(output) + if finish_reason is not None: + finish = finish_reason + elif error: + finish = "error" + elif any(p["type"] == "tool_call" for p in parts): + finish = "tool_call" + else: + finish = "stop" + attrs["gen_ai.output.messages"] = json.dumps( + [{"role": role, "parts": parts, "finish_reason": finish}] + ) + return attrs + + +def _hack_get_field(obj: Any, name: str) -> Any: + # this is caused by the params import hack in span.py + if isinstance(obj, dict): + return obj.get(name) + return getattr(obj, name, None) + + +_SEMCONV_SAMPLER_FIELDS = ( + "temperature", + "top_k", + "top_p", + "frequency_penalty", + "presence_penalty", + "seed", +) + + +def _semconv_request_attributes(params: Any) -> dict[str, Any]: + """Extract and convert request attributes from inference params.""" + attrs: dict[str, Any] = {} + sampling = _hack_get_field(params, "sampling") + if isinstance(sampling, dict): + for sampler in sampling.values(): + for name in _SEMCONV_SAMPLER_FIELDS: + value = _hack_get_field(sampler, name) + if isinstance(value, int | float) and not isinstance( + value, bool + ): + attrs[f"gen_ai.request.{name}"] = value + max_tokens = _hack_get_field( + _hack_get_field(params, "output"), "max_tokens" + ) + if isinstance(max_tokens, int): + attrs["gen_ai.request.max_tokens"] = max_tokens + effort = _hack_get_field(_hack_get_field(params, "reasoning"), "effort") + if isinstance(effort, str): + attrs["gen_ai.request.reasoning.level"] = effort + return attrs + + +def _semconv_usage_attributes(usage: usage_.Usage) -> dict[str, Any]: + attrs: dict[str, Any] = { + "gen_ai.usage.input_tokens": usage.input_tokens, + "gen_ai.usage.output_tokens": usage.output_tokens, + } + if usage.reasoning_tokens is not None: + attrs["gen_ai.usage.reasoning.output_tokens"] = usage.reasoning_tokens + if usage.cache_read_tokens is not None: + attrs["gen_ai.usage.cache_read.input_tokens"] = usage.cache_read_tokens + if usage.cache_write_tokens is not None: + attrs["gen_ai.usage.cache_creation.input_tokens"] = ( + usage.cache_write_tokens + ) + return attrs + + +def _time_to_first_chunk(sp: telemetry.Span) -> float | None: + if sp.started_at is None: + return None + for event in sp.events: + if event.name == telemetry.FIRST_TOKEN: + return (event.time_ns - sp.started_at) / 1e9 + return None + + +def _semconv_tool_definitions(names: list[str]) -> str: + return json.dumps([{"type": "function", "name": n} for n in names]) + + +def _attributes(sp: telemetry.Span, *, capture_content: bool) -> dict[str, Any]: + # process trace_attrs + attrs: dict[str, Any] = { + # otel only allows scalar attribute values or lists of them + key: value + if isinstance(value, str | bool | int | float) + else repr(value) + for key, value in sp.trace_attrs.items() + } + if sp.replay: + attrs["ai.replay"] = True + match sp.data: + case ( + (telemetry.AiStreamSpanData() | telemetry.AiGenerateSpanData()) as d + ): + attrs["gen_ai.operation.name"] = "chat" + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + streamed = isinstance(d, telemetry.AiStreamSpanData) + attrs["gen_ai.request.stream"] = streamed + if d.output_type is not None: + # Structured output was requested; the semconv value is + # the output kind, not the Python type name. + attrs["gen_ai.output.type"] = "json" + attrs |= _semconv_request_attributes(d.params) + if streamed: + ttfc = _time_to_first_chunk(sp) + if ttfc is not None: + attrs["gen_ai.response.time_to_first_chunk"] = ttfc + if d.finish_reason is not None: + attrs["gen_ai.response.finish_reasons"] = [d.finish_reason] + if d.response_id is not None: + attrs["gen_ai.response.id"] = d.response_id + if d.response_model is not None: + attrs["gen_ai.response.model"] = d.response_model + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + if capture_content: + if d.tool_names: + attrs["gen_ai.tool.definitions"] = ( + _semconv_tool_definitions(d.tool_names) + ) + attrs |= _semconv_content_attributes( + d.messages, + d.message, + error=sp.error is not None, + finish_reason=d.finish_reason, + ) + case telemetry.DecideSpanData() as d: + attrs["gen_ai.operation.name"] = "decide" + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + case telemetry.EmbedSpanData() as d: + attrs["gen_ai.operation.name"] = "embeddings" + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + if d.dimensions is not None: + attrs["gen_ai.embeddings.dimension.count"] = d.dimensions + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + case ( + ( + telemetry.GenerateAudioSpanData() + | telemetry.GenerateImageSpanData() + | telemetry.GenerateVideoSpanData() + ) as d + ): + attrs["gen_ai.operation.name"] = "generate_content" + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + case telemetry.RerankSpanData() as d: + attrs["gen_ai.operation.name"] = "rerank" + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + case telemetry.TranscribeSpanData() as d: + attrs["gen_ai.operation.name"] = "transcribe" + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + case telemetry.ToolExecutionSpanData() as d: + attrs["gen_ai.operation.name"] = "execute_tool" + attrs["gen_ai.tool.name"] = d.tool_name + attrs["gen_ai.tool.type"] = "function" + attrs["gen_ai.tool.call.id"] = d.tool_call_id + if d.tool_description is not None: + attrs["gen_ai.tool.description"] = d.tool_description + if d.is_error: + # The exception type is not captured when the error is + # only a model-facing result (see NEW_DATA_CAPTURE.md); + # ``_OTHER`` is semconv's fallback class. Overwritten + # with the real type when the span carries an error. + attrs["error.type"] = "_OTHER" + if capture_content: + if d.args is not None: + attrs["gen_ai.tool.call.arguments"] = json.dumps( + d.args, default=str + ) + if d.result is not None: + attrs["gen_ai.tool.call.result"] = json.dumps( + d.result, default=str + ) + case telemetry.RunSpanData() as d: + attrs["gen_ai.operation.name"] = "invoke_agent" + attrs["gen_ai.agent.name"] = d.agent + if d.provider is not None: + attrs["gen_ai.provider.name"] = d.provider + attrs["gen_ai.request.model"] = d.model + if d.output_type is not None: + # Structured output was requested; the semconv value is + # the output kind, not the Python type name. + attrs["gen_ai.output.type"] = "json" + attrs |= _semconv_request_attributes(d.params) + if d.usage is not None: + attrs |= _semconv_usage_attributes(d.usage) + if capture_content: + if d.tool_names: + attrs["gen_ai.tool.definitions"] = ( + _semconv_tool_definitions(d.tool_names) + ) + attrs |= _semconv_content_attributes( + d.messages, d.final_message, error=sp.error is not None + ) + case telemetry.HookSpanData() as d: + attrs["ai.hook.label"] = d.label + attrs["ai.hook.type"] = d.hook_type + attrs["ai.hook.status"] = d.status + case telemetry.CustomSpanData() as d: + for key, value in d.attrs.items(): + # otel only allows scalar attribute values or lists of them + attrs[key] = ( + value + if isinstance(value, str | bool | int | float) + else repr(value) + ) + return attrs + + +def _derive_trace_id(id_: str) -> int: + """Derive a stable otel trace id (128 bits) from one of ours.""" + return int.from_bytes(hashlib.sha256(id_.encode()).digest()[:16]) + + +def _derive_span_id(id_: str) -> int: + """Derive a stable otel span id (64 bits) from one of ours.""" + return int.from_bytes(hashlib.sha256(id_.encode()).digest()[:8]) + + +# otel spec requires ids to be always minted by the otel sdk. we can't have +# that because of the durable execution case, where we can't always emit the +# span right where we created it. that's why we're using a smuggling hack to +# smuggle our ids in via a contextvar. fable is telling me that's a classic +# trick. +_smuggled_ids: contextvars.ContextVar[tuple[int, int] | None] = ( + contextvars.ContextVar("otel_smuggled_ids", default=None) +) + + +class _SmugglingIdGenerator( + opentelemetry.sdk.trace.id_generator.RandomIdGenerator +): + """Fake id generator for smuggling ids via a contextvar.""" + + def __init__( + self, inner: opentelemetry.sdk.trace.id_generator.IdGenerator + ) -> None: + self._inner = inner # stock id generator + + def generate_trace_id(self) -> int: + smuggled = _smuggled_ids.get() + return ( + smuggled[0] + if smuggled is not None + else self._inner.generate_trace_id() + ) + + def generate_span_id(self) -> int: + smuggled = _smuggled_ids.get() + return ( + smuggled[1] + if smuggled is not None + else self._inner.generate_span_id() + ) + + +@runtime_checkable +class _Flushable(Protocol): + """Duck type whether tracer provider can flush.""" + + def force_flush(self) -> bool: ... + def shutdown(self) -> None: ... + + +@telemetry.adapter +class OtelAdapter: + """Maps framework spans onto otel spans.""" + + def __init__( + self, + *, + tracer_provider: opentelemetry.trace.TracerProvider | None = None, + capture_content: bool | None = None, + ) -> None: + provider = tracer_provider or opentelemetry.trace.get_tracer_provider() + self._provider = provider + self._live: dict[str, opentelemetry.trace.Span] = {} + + self._is_capturing_content = ( + _should_capture_content() + if capture_content is None + else capture_content + ) + + self._is_smuggling_ids = False + # printed a warning once when failing to smuggle ids + self._warning_issued = False + + # swap TracerProvider's id generator with a smuggling one + if isinstance(provider, opentelemetry.sdk.trace.TracerProvider): + provider.id_generator = _SmugglingIdGenerator(provider.id_generator) + self._is_smuggling_ids = True + + self._tracer = opentelemetry.trace.get_tracer( + "ai", tracer_provider=provider + ) + + def span_name(self, span: telemetry.Span, /) -> str: + """Return the exported otel span name. Override to customize.""" + return _semconv_name(span) + + def span_attrs(self, span: telemetry.Span, /) -> dict[str, Any]: + """Return the attributes set at span end. Override to enrich. + + :: + + class MyAdapter(otel.OtelAdapter): + def span_attrs(self, span): + return super().span_attrs(span) | {"k": "v"} + """ + return _attributes(span, capture_content=self._is_capturing_content) + + def flush(self) -> None: + """Flush the provider's exporters.""" + # otel *api protocol* doesn't assume the tracer provider would have + # a buffer, so it doesn't declare a flush method. + # otel *sdk's* tracer provider DOES have a buffer, and therefore needs + # to flush. users may also pass their own tracer providers modelled + # after the stock one. + if isinstance(self._provider, _Flushable): + self._provider.force_flush() + + def shutdown(self) -> None: + """Flush and stop the provider; spans pushed after this are lost.""" + if isinstance(self._provider, _Flushable): + self._provider.shutdown() + + def _parent_context( + self, span: telemetry.Span + ) -> opentelemetry.context.Context | None: + # sometimes we need to set a parent span that isn't currently live. + if span.parent_id is None or span.parent_id in self._live: + # here the parent span is actually live, proceed as normal + return None + + # parent is indeed not live + # opentelemetry.io/docs/languages/python/cookbook/ + # 1. wrap the ids in a ``SpanContext`` + # 2. wrap that in a ``NonRecordingSpan`` + # 3. set it in a *fresh* ``Context()`` + + if not self._is_smuggling_ids and not self._warning_issued: + # trying to restore a parent when not smuggling ids + # will result in a broken tree + self._warning_issued = True + logger.warning( + "otel adapter: continuing a trace from another process " + "needs an SDK tracer provider to control span ids; ids " + "will not line up across processes" + ) + return opentelemetry.trace.set_span_in_context( + opentelemetry.trace.NonRecordingSpan( + opentelemetry.trace.SpanContext( + trace_id=_derive_trace_id(span.trace_id), + span_id=_derive_span_id(span.parent_id), + is_remote=True, # marks the parent as living elsewhere + trace_flags=opentelemetry.trace.TraceFlags( + opentelemetry.trace.TraceFlags.SAMPLED + ), + ) + ), + opentelemetry.context.Context(), + ) + + async def __call__( + self, span: telemetry.Span, / + ) -> AsyncGenerator[None, Any]: + parent_context = self._parent_context(span) + + # convert framework's span ids into otel's and prepare for smuggling + smuggled_token = _smuggled_ids.set( + (_derive_trace_id(span.trace_id), _derive_span_id(span.id)) + ) + try: + otel_span = self._tracer.start_span( + self.span_name(span), + context=parent_context, + kind=_semconv_kind(span), + start_time=span.started_at, + ) + finally: + _smuggled_ids.reset(smuggled_token) + + self._live[span.id] = otel_span + + try: + # set live span as otel's current, so raw spans user opens in their + # code can nest under it. + with ( + opentelemetry.trace.use_span( + otel_span, + end_on_exit=False, # we end it ourselves below + # errors come it via span.error, not via raising, because + # we need them to serialize + record_exception=False, + set_status_on_exception=False, + ) + if span.set_as_current + else contextlib.nullcontext() + ): + # span end resumes with None + while (ev := (yield)) is not None: + otel_span.add_event( + ev.name, + { + k: v # squash everything into scalars + if isinstance(v, str | bool | int | float) + else repr(v) + for k, v in ev.attrs.items() + }, + timestamp=ev.time_ns, + ) + finally: + self._live.pop(span.id, None) + for key, value in self.span_attrs(span).items(): + otel_span.set_attribute(key, value) + if span.error is not None: + otel_span.set_attribute("error.type", span.error.type) + otel_span.set_status( + opentelemetry.trace.StatusCode.ERROR, + f"{span.error.type}: {span.error.message}", + ) + otel_span.end(end_time=span.ended_at) diff --git a/src/ai/telemetry/span.py b/src/ai/telemetry/span.py new file mode 100644 index 00000000..6b90b1b7 --- /dev/null +++ b/src/ai/telemetry/span.py @@ -0,0 +1,1214 @@ +"""Telemetry API: spans, sinks, and adapters. + +:class:`Span` is a record of work. It carries ids, timestamps, parent id, and +work-specific data:: + + async with ai.telemetry.span("retrieval") as sp: + sp.set_attrs(query=q) + docs = await search(q) + sp.set_attrs(count=len(docs)) + +Nesting is automatic: the current span is tracked using a context var. + +In cases where the high-level API doesn't work (e.g. durable execution), +selectively using low-level API may be a better fit. + +1. make a span using `create_span` +2. set `.started_at`, `ended_at`, and necessary data fields +3. push updates to the sink one or more times + +:meth:`Span.push` delivers a snapshot of the span to the current *sink*. +The default sink is the adapter registry, meaning on push all registered +adapters will get the updated version of the span:: + + sp = create_span("turn") # identity only, reports nothing + sp.set_attrs(session=sid) + sp.stamp_start() # writes started_at + await sp.push() # visible to sinks/adapters + ... # possibly elsewhere, later: + await sp.stamp_end().push() # complete + +Because ``Span`` is a pydantic model it crosses process boundaries like +any other data (``model_dump`` / ``model_validate``); you can restore it +and set it as current using :func:`use_span` or pass to a child span explicity. + +An adapter processes spans and decides what to do with them:: + + class MyAdapter: + async def on_span_start(self, span): ... + async def on_span_end(self, span): ... + async def on_span_event(self, span, event): ... + + ai.telemetry.register(MyAdapter()) + +Adapters dispatch on the type of ``span.data``. An adapter that crashes +is logged and skipped, it never kills the run. + +:func:`adapter` builds an adapter from an async generator function or +class, using the pytest fixture-style trick. + +A :class:`Sink` is what sits *before* the adapters and routes span snapshots. +It exists, again, because of the durable execution case: the user can control +whether spans in the current context get sent to telemetry adapters, collected +into a list to be pushed later, or discarded. + +:func:`use_sink` sets the current sink. + +If telemetry is off, i.e. no adapters registered and no sink used, the module +doesn't use random or read the clock. + +A :class:`SpanEvent` is a named, timestamped milestone inside a span's +lifetime (``first_token``, ``hook_resolved``, ...): append it to +``span.events`` and push. + +Span timestamps come from the ambient clock (:func:`now_ns`); +:func:`use_time` overrides it per-context, e.g. with a deterministic +clock inside a durable workflow. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import inspect +import logging +import time +from typing import ( + TYPE_CHECKING, + Annotated, + Any, + ClassVar, + Generic, + Literal, + Protocol, + Self, + cast, + overload, + runtime_checkable, +) + +import pydantic + +# ``typing.TypeVar`` lacks the ``default=`` kwarg on Python <3.13. +# Use the typing_extensions backport so this works on 3.12 too. +from typing_extensions import TypeVar + +from ..types import messages as messages_ +from ..types import usage as usage_ + +if TYPE_CHECKING: + from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Iterable, + Mapping, + ) + + from ..models.core import params as params_ + + # 1. RunSpanData and friends are pydantic models. RunSpanData.params is + # typed as InferenceRequestParams. + # 2. InferenceRequestParams.sampling (and other fields) have + # a ModelProviderDefault annotation. + # 3. ModelProviderDefault is a plain setinel class that causes pydantic to + # freak out when building a schema. + + # also, importing without a guard would cause a circular import + # in models/core/api.py + _InferenceParams = params_.InferenceRequestParams +else: + _InferenceParams = Any + +logger = logging.getLogger(__name__) + + +# this api is for plugging a deterministic clock in, so that the sdk +# could be used inside a deterministic workflow body + +_span_clock: contextvars.ContextVar[Callable[[], int] | None] = ( + contextvars.ContextVar("span_clock", default=None) +) + + +def now_ns() -> int: + """Nanoseconds since the epoch, from the ambient span clock. + + The wall clock by default; whatever :func:`use_time` installed + otherwise. + """ + clock = _span_clock.get() + return clock() if clock is not None else time.time_ns() + + +@contextlib.asynccontextmanager +async def use_time(now_ns: Callable[[], int]) -> AsyncIterator[None]: + """Read span timestamps from ``now_ns`` within this context. + + Framework's observability creates timestamps. This API can be + used to plug an approved clock function in durable execution + settings:: + + async with ai.telemetry.use_time(workflow.time_ns): + ... # spans opened here read time from workflow.time_ns + + This can also be used as a decorator on async functions:: + + @ai.telemetry.use_time(clock.time_ns) + async def run(...): + ... + """ + token = _span_clock.set(now_ns) + try: + yield + finally: + _span_clock.reset(token) + + +# span data types +# the data type tells you what kind of span it is. + + +class SpanData(Protocol): + """Anything with a ``kind`` is span data. + + Implement it with a pydantic model to make your own typed spans. + + class RetrievalSpanData(pydantic.BaseModel): + kind: Literal["retrieval"] = "retrieval" + query: str + count: int | None = None + + data = RetrievalSpanData(query=q) + async with ai.telemetry.span(data) as sp: + docs = await search(q) + sp.data.count = len(docs) # typed + + ``kind`` is what travels with the serialized data. Restore with + ``Span[RetrievalSpanData].model_validate(...)`` to get it typed, otherwise + you'll get a plain dict. + """ + + @property + def kind(self) -> str: ... + + +class RunSpanData(pydantic.BaseModel): + """One ``Agent.run``: the whole loop. + + ``blocked``/``final_message``/``usage`` are set at span end: + ``blocked`` is True when the run ended suspended on an unresolved + hook (see ``AgentStream.blocked``), ``final_message`` is the last + assistant message produced, if any, ``usage`` is the sum of usage + across the messages the run produced. + """ + + kind: Literal["run"] = "run" + agent: str + model: str + messages: list[messages_.Message] + provider: str | None = None + tool_names: list[str] | None = None + output_type: str | None = None + params: _InferenceParams | None = None + blocked: bool = False + final_message: messages_.Message | None = None + usage: usage_.Usage | None = None + + +class LoopTurnSpanData(pydantic.BaseModel): + """One turn of the default agent loop. + + Carries no fields — it exists so adapters can group a turn's model + and tool spans. Turn order is given by ``started_at``/``parent_id``. + """ + + kind: Literal["loop_turn"] = "loop_turn" + + +class AiStreamSpanData(pydantic.BaseModel): + """One streaming LLM call.""" + + kind: Literal["ai_stream"] = "ai_stream" + model: str + messages: list[messages_.Message] + params: _InferenceParams | None = None + provider: str | None = None + tool_names: list[str] | None = None + output_type: str | None = None + message: messages_.Message | None = None + usage: usage_.Usage | None = None + finish_reason: str | None = None + response_id: str | None = None + response_model: str | None = None + + +class AiGenerateSpanData(pydantic.BaseModel): + """One buffered (non-streaming) LLM call. + + ``message``/``usage`` and the response identity fields are set at + span end. + """ + + kind: Literal["ai_generate"] = "ai_generate" + model: str + messages: list[messages_.Message] + params: _InferenceParams | None = None + provider: str | None = None + tool_names: list[str] | None = None + output_type: str | None = None + message: messages_.Message | None = None + usage: usage_.Usage | None = None + finish_reason: str | None = None + response_id: str | None = None + response_model: str | None = None + + +class DecideSpanData(pydantic.BaseModel): + """One typed decision operation.""" + + kind: Literal["decide"] = "decide" + model: str + provider: str | None = None + question_count: int + usage: usage_.Usage | None = None + answer_count: int | None = None + warnings: list[dict[str, Any]] | None = None + + +class EmbedSpanData(pydantic.BaseModel): + """One embedding operation.""" + + kind: Literal["embed"] = "embed" + model: str + provider: str | None = None + input_count: int + usage: usage_.Usage | None = None + output_count: int | None = None + dimensions: int | None = None + warnings: list[dict[str, Any]] | None = None + + +class GenerateAudioSpanData(pydantic.BaseModel): + """One audio generation operation.""" + + kind: Literal["generate_audio"] = "generate_audio" + model: str + provider: str | None = None + usage: usage_.Usage | None = None + output_count: int | None = None + warnings: list[dict[str, Any]] | None = None + + +class GenerateImageSpanData(pydantic.BaseModel): + """One image generation operation.""" + + kind: Literal["generate_image"] = "generate_image" + model: str + provider: str | None = None + usage: usage_.Usage | None = None + output_count: int | None = None + warnings: list[dict[str, Any]] | None = None + + +class GenerateVideoSpanData(pydantic.BaseModel): + """One video generation operation.""" + + kind: Literal["generate_video"] = "generate_video" + model: str + provider: str | None = None + usage: usage_.Usage | None = None + output_count: int | None = None + warnings: list[dict[str, Any]] | None = None + + +class RerankSpanData(pydantic.BaseModel): + """One reranking operation.""" + + kind: Literal["rerank"] = "rerank" + model: str + provider: str | None = None + input_count: int + top_n: int | None = None + usage: usage_.Usage | None = None + output_count: int | None = None + warnings: list[dict[str, Any]] | None = None + + +class TranscribeSpanData(pydantic.BaseModel): + """One transcription operation.""" + + kind: Literal["transcribe"] = "transcribe" + model: str + provider: str | None = None + usage: usage_.Usage | None = None + duration_seconds: float | None = None + warnings: list[dict[str, Any]] | None = None + + +class ToolExecutionSpanData(pydantic.BaseModel): + """One tool execution, from dispatch to result. + + ``model_input`` is the value the LLM sees on its next turn, set + only when it differs from ``result`` (aggregator-backed tools). + ``tool_description`` is the tool's model-facing description, + stamped at dispatch. + """ + + kind: Literal["tool_execution"] = "tool_execution" + tool_name: str + tool_call_id: str + tool_description: str | None = None + args: dict[str, Any] | None = None + result: Any = None + model_input: Any = None + is_error: bool = False + + +class HookSpanData(pydantic.BaseModel): + """One hook suspension, from deferred until resolved or cancelled. + + ``tool_call_id`` links the hook to the tool call it suspends, if + any (e.g. approval gating). ``resolution`` carries the resolution + data when the hook resolves. + """ + + kind: Literal["hook"] = "hook" + label: str + hook_type: str + metadata: dict[str, Any] + tool_call_id: str | None = None + status: Literal["pending", "resolved", "cancelled"] = "pending" + resolution: dict[str, Any] | None = None + + +class CustomSpanData(pydantic.BaseModel): + """A user span made with ``span("name")``. + + Attributes are set with :meth:`Span.set_attrs`. + """ + + kind: Literal["custom"] = "custom" + attrs: dict[str, Any] + + +# names for span events shared between producers, +# adapters, and tests +FIRST_TOKEN = "first_token" +RESPONSE_COMPLETE = "response_complete" +HOOK_DEFERRED = "hook_deferred" +HOOK_RESOLVED = "hook_resolved" +HOOK_CANCELLED = "hook_cancelled" + + +class SpanEvent(pydantic.BaseModel): + """A named, timestamped milestone inside a span's lifetime. + + Lives in ``span.events``; the next push delivers it. + :meth:`Span.add_event` is the shorthand for the common case:: + + sp.add_event(FIRST_TOKEN) + await sp.push() + + Ordering: events on one span are delivered in list order; there is + no ordering guarantee across spans. + """ + + name: str + time_ns: int + attrs: dict[str, Any] + + +class SpanError(pydantic.BaseModel): + """A serializable record of the failure that ended a span. + + Spans cross process boundaries, so the error they carry is plain + data, not a live exception. + """ + + type: str + message: str + + @classmethod + def from_exception(cls, exc: BaseException) -> SpanError: + return cls(type=type(exc).__name__, message=str(exc)) + + +_FrameworkData = Annotated[ + RunSpanData + | LoopTurnSpanData + | AiStreamSpanData + | AiGenerateSpanData + | DecideSpanData + | EmbedSpanData + | GenerateAudioSpanData + | GenerateImageSpanData + | GenerateVideoSpanData + | RerankSpanData + | TranscribeSpanData + | ToolExecutionSpanData + | HookSpanData + | CustomSpanData, + pydantic.Field(discriminator="kind"), +] + +# adapters only read the data, they don't swap it out, so it's safe to make it +# covariant. +if TYPE_CHECKING: + DataT_co = TypeVar( + "DataT_co", bound=SpanData, default=SpanData, covariant=True + ) +else: + # try to deserialize to _FrameworkData and fallback to Any + DataT_co = TypeVar("DataT_co", default=_FrameworkData | Any) +# covariant type can't be used for a function's input parameter, and we need +# to type span() +DataT = TypeVar("DataT", bound=SpanData) + + +class Span(pydantic.BaseModel, Generic[DataT_co]): + """A serializable record of a unit of work. + + Generic in its data type: ``span(RetrievalSpanData(...))`` gives a + ``Span[RetrievalSpanData]``, so late assignments to ``sp.data`` + fields are type checked. A bare ``Span`` is ``Span[SpanData]``. + + Lifecycle is encoded in timestamp fields: + + - if ``started_at=None``, this span hasn't started, + - if ``ended_at`` is set, the span is complete. + + Mutate fields, then :meth:`push` to report the new state. Nothing is + reported except by pushing. + + A span round-trips through ``model_dump``/``model_validate`` like + any pydantic model. Use ``Span[MyData].model_validate(...)`` to restore + custom data types. + + ``replay=True`` marks work that is being replayed (resume, + serverless re-entry) rather than performed live. + + A span created while telemetry was off (see :func:`is_enabled`) has an + empty ``id`` and is a noop: :meth:`push` delivers nothing. + + ``set_as_current=False`` marks a span that does not set itself + as a current. This is used by ai.stream's span that stays open for + the duration of a loop turn because of the context manager api. + Adapters must not make those spans "current" either to avoid + nesting discrepancies. + + ``trace_attrs`` is user data that propagates down from parent to child + on creation. + + ``schema_version`` tracks the shape of spans and their data types. + """ + + name: str + data: DataT_co + id: str + trace_id: str + parent_id: str | None = None + started_at: int | None = None # ns since the epoch, see ``now_ns`` + ended_at: int | None = None + error: SpanError | None = None + replay: bool = False + set_as_current: bool = True + events: list[SpanEvent] = pydantic.Field(default_factory=list) + trace_attrs: dict[str, Any] = pydantic.Field(default_factory=dict) + + schema_version: ClassVar[int] = 5 + + def set_attrs( + self, attrs: Mapping[str, Any] | None = None, /, **kwargs: Any + ) -> None: + """Attach attributes to a span created with ``span("name")``. + + Attribute names that aren't valid Python keywords (viewers use + dotted names like ``"output.value"``) go in the positional + mapping; it merges with the keyword arguments:: + + sp.set_attrs({"output.value": title}, model="haiku") + """ + if not isinstance(self.data, CustomSpanData): + raise TypeError( + "set_attrs() only works on user spans; framework " + "spans carry typed data, assign its fields directly" + ) + self.data.attrs.update({**(attrs or {}), **kwargs}) + + def add_event( + self, + name: str, + attrs: Mapping[str, Any] | None = None, + /, + **kwargs: Any, + ) -> SpanEvent: + """Append a named event stamped with the ambient clock. + + Appends only; the event is delivered by the next :meth:`push`, or when + exiting the ``span()`` context manager:: + + sp.add_event(FIRST_TOKEN) + await sp.push() + + Returns the appended event. + """ + event = SpanEvent( + name=name, + # a noop span (telemetry off at creation) reads no clock + time_ns=now_ns() if self.id else 0, + attrs={**(attrs or {}), **kwargs}, + ) + self.events.append(event) + return event + + def stamp_start(self) -> Self: + """Set ``started_at`` from the ambient clock; return the span. + + Only writes the field; nothing is reported until :meth:`push`. + Returns the span so minting reads as one expression:: + + payload = create_span("turn").stamp_start().model_dump() + """ + self.started_at = now_ns() + return self + + def stamp_end( + self, *, error: SpanError | BaseException | None = None + ) -> Self: + """Set ``ended_at`` from the ambient clock; return the span. + + Also records ``error`` when one is given by converting an exception + to a serializable :class:`SpanError`. ``error=None`` + keeps any error already on the span. Only writes fields; + push to report:: + + await sp.stamp_end(error=exc).push() + """ + self.ended_at = now_ns() + if isinstance(error, BaseException): + self.error = SpanError.from_exception(error) + elif error is not None: + self.error = error + return self + + async def push(self) -> None: + """Deliver a frozen copy of this span to the current sink. + + The default sink drives the registered adapters (see + :func:`register`); :func:`use_sink` reroutes pushes within a + context. Each push snapshots the whole span, so keep mutating + and push again as the work progresses. The last push with + ``ended_at`` set is the complete record. + + Telemetry never kills the run: a sink (or adapter) failure is + logged and swallowed. + """ + if not self.id: + return # noop: telemetry was off at creation + sink = _current_sink.get() or _registry_sink + try: + await sink.on_push(self.model_copy(deep=True)) + except Exception: + logger.exception("telemetry sink %r raised in on_push", sink) + + +# utilities for managing the current span + +_current: contextvars.ContextVar[Span | None] = contextvars.ContextVar( + "current_span", default=None +) + + +def current_span() -> Span | None: + """Return the current span, or ``None`` when no span is open.""" + return _current.get() + + +@contextlib.asynccontextmanager +async def use_span(span_: Span | None) -> AsyncIterator[None]: + """Make ``span_`` the current span within this context. + + Unlike :func:`span` this is pure context plumbing with no timestamps + and no pushes. Use it to continue a trace around existing work, e.g. + parenting under a span restored from another process:: + + turn_span = Span.model_validate(payload["turn_span"]) + async with ai.telemetry.use_span(turn_span): + ... # spans opened here parent under turn_span + + ``None`` is a no-op. + """ + if span_ is None: + yield + return + token = _current.set(span_) + try: + yield + finally: + _current.reset(token) + + +# sinks are used to control where spans go within the current context. +# that could be the adapter registry (so they immediately get sent to the +# observability backend; or it could be a DictSink that allows us to defer +# sending them until later. + + +class Sink(Protocol): + """Anything that accepts pushed span snapshots.""" + + async def on_push(self, span: Span, /) -> None: ... + + +_current_sink: contextvars.ContextVar[Sink | None] = contextvars.ContextVar( + "current_sink", default=None +) + + +@contextlib.asynccontextmanager +async def use_sink(sink: Sink | None) -> AsyncIterator[None]: + """Route span pushes to ``sink`` within this context. + + The default (outside any ``use_sink``) is the adapter registry. + Inside a durable workflow body where side effects are not allowed, you + can route to a :class:`DictSink` instead and re-push the collected spans + from a step / activity. + + ``None`` is a no-op. + """ + if sink is None: + yield + return + token = _current_sink.set(sink) + try: + yield + finally: + _current_sink.reset(token) + + +class DictSink: + """A sink that keeps the latest snapshot of every span pushed to it. + + ``spans`` maps span id to the most recent snapshot, in first-push + order. Scoop the complete ones out as data (:attr:`finished_spans`) and + re-push them where the real sink is available:: + + sink = DictSink() + async with use_sink(sink): + ... # replayed / suspendable code + payload = [s.model_dump(mode="json") for s in sink.finished_spans] + + # elsewhere (a workflow step, another process): + await push_all(payload) + """ + + def __init__(self) -> None: + self.spans: dict[str, Span] = {} + + async def on_push(self, span: Span, /) -> None: + self.spans[span.id] = span + + @property + def finished_spans(self) -> list[Span]: + """The collected spans that have ended, in first-push order.""" + return [s for s in self.spans.values() if s.ended_at is not None] + + +async def push_all(spans: Iterable[Span | Mapping[str, Any]]) -> None: + """Push each span, in order; dumped spans are validated first. + + This is a utility that can be used with :class:`DictSink` to push + a bunch of collected spans all at once:: + + await push_all(payload) + """ + for item in spans: + span = item if isinstance(item, Span) else Span.model_validate(item) + await span.push() + + +class _RegistrySink: + """The default sink: drives registered adapters. + + Translates a stream of span snapshots into + ``on_span_start`` / ``on_span_event`` / ``on_span_end`` callbacks. + + Rules: + + - a snapshot with ``started_at=None`` reports nothing (the span + hasn't started); + - the first snapshot of an id fires ``on_span_start``, then + ``on_span_event`` per event, then ``on_span_end`` if the snapshot + is already complete; so a span that lived elsewhere and arrives + finished is delivered whole; + - later snapshots fire ``on_span_event`` for events not seen + before, and ``on_span_end`` once ``ended_at`` appears; + - after the end the id is forgotten: pushing a completed span again + re-delivers it in full. + """ + + def __init__(self) -> None: + self._views: dict[str, Span] = {} + + async def on_push(self, span: Span, /) -> None: + if span.started_at is None: + return + view = self._views.get(span.id) + if view is None: + view = span + self._views[view.id] = view + await _dispatch(lambda a: a.on_span_start(view)) + fresh = list(view.events) + else: + seen = len(view.events) + # we need to preserve the view's identity + # but update all of the fields + view.__dict__.update(span.__dict__) + fresh = view.events[seen:] + for event in fresh: + # the default binds `event` now; a plain lambda would trip B023 + def call( + a: AdapterProtocol, ev: SpanEvent = event + ) -> Awaitable[None]: + return a.on_span_event(view, ev) + + await _dispatch(call) + if view.ended_at is not None: + del self._views[view.id] + await _dispatch(lambda a: a.on_span_end(view)) + + +_registry_sink = _RegistrySink() + + +# adapter registry utilities + + +@runtime_checkable +class AdapterProtocol(Protocol): + """The async callbacks implemented by a telemetry adapter.""" + + async def on_span_start(self, span: Span, /) -> None: ... + + async def on_span_event(self, span: Span, event: SpanEvent, /) -> None: ... + + async def on_span_end(self, span: Span, /) -> None: ... + + +class AdapterCallable(Protocol): + """The ``__call__`` shape the :func:`adapter` decorator accepts. + + Static-typing companion to :class:`AdapterProtocol`: checkers don't + see the callbacks :func:`adapter` mixes into a decorated class, so + :func:`register` accepts this shape as well. + """ + + def __call__(self, span: Span, /) -> AsyncGenerator[None, Any] | None: ... + + +_adapters: list[AdapterProtocol] = [] + + +def register(adapter: AdapterProtocol | AdapterCallable) -> None: + """Add an adapter. Multiple adapters coexist independently.""" + if not isinstance(adapter, AdapterProtocol): + raise TypeError( + f"{adapter!r} does not implement the adapter callbacks; " + "an adapter class must be decorated with @adapter" + ) + _adapters.append(adapter) + + +def unregister(adapter: AdapterProtocol | AdapterCallable) -> None: + """Remove a previously registered adapter.""" + assert isinstance(adapter, AdapterProtocol) # register() enforced it + _adapters.remove(adapter) + + +def is_enabled() -> bool: + """Whether anything is listening for spans. + + True when a sink is routed with :func:`use_sink` or at least one + adapter is registered. + """ + return _current_sink.get() is not None or bool(_adapters) + + +async def _dispatch( + call: Callable[[AdapterProtocol], Awaitable[None]], +) -> None: + for adapter in list(_adapters): + try: + await call(adapter) + except Exception: + logger.exception("telemetry adapter %r raised", adapter) + + +# span building utilities + + +@overload +def create_span( + name_or_data: str, + /, + *, + parent: Span | None = None, + replay: bool = False, + set_as_current: bool = True, +) -> Span[CustomSpanData]: ... + + +@overload +def create_span( + name_or_data: DataT, + /, + *, + parent: Span | None = None, + replay: bool = False, + set_as_current: bool = True, +) -> Span[DataT]: ... + + +def create_span( + name_or_data: str | SpanData, + /, + *, + parent: Span | None = None, + replay: bool = False, + set_as_current: bool = True, +) -> Span[Any]: + """Create a span: identity only, nothing is reported. + + Mints the span id, and takes trace id, parentage, and a copy of + ``trace_attrs`` from ``parent`` (default: the current span; a fresh + trace when there is none). ``parent`` may be a span restored from + JSON. + + The span has no timestamps yet: :meth:`Span.stamp_start` and + :meth:`Span.push` when the work begins, or hand the whole + lifecycle to the :func:`span` context manager. + + While telemetry is off (see :func:`is_enabled`) this returns a noop + span instead. + """ + if isinstance(name_or_data, str): + name = name_or_data + data: SpanData = CustomSpanData(attrs={}) + else: + name = name_or_data.kind + data = name_or_data + if not is_enabled(): + return Span( + name=name, + data=data, + id="", + trace_id="", + parent_id=None, + replay=replay, + set_as_current=False, + ) + if parent is None: + parent = _current.get() + if parent is None: + trace_id, parent_id = messages_.generate_id("trace"), None + trace_attrs: dict[str, Any] = {} + else: + trace_id, parent_id = parent.trace_id, parent.id + trace_attrs = dict(parent.trace_attrs) + return Span( + name=name, + data=data, + id=messages_.generate_id("span"), + trace_id=trace_id, + parent_id=parent_id, + replay=replay, + set_as_current=set_as_current, + trace_attrs=trace_attrs, + ) + + +@overload +def span( + name_or_data: str, + /, + *, + parent: Span | None = None, + replay: bool = False, + set_as_current: bool = True, +) -> contextlib.AbstractAsyncContextManager[Span[CustomSpanData]]: ... + + +@overload +def span( + name_or_data: DataT, + /, + *, + parent: Span | None = None, + replay: bool = False, + set_as_current: bool = True, +) -> contextlib.AbstractAsyncContextManager[Span[DataT]]: ... + + +def span( + name_or_data: str | SpanData, + /, + *, + parent: Span | None = None, + replay: bool = False, + set_as_current: bool = True, +) -> contextlib.AbstractAsyncContextManager[Span[Any]]: + """Open a span; it sets itself as current inside the block. + + Sugar over the data api: creates the span, stamps ``started_at`` + and pushes on enter; stamps ``ended_at`` (and ``error``, if the + block raised) and pushes on exit. + + Pass a name for a user span (set attributes on it with + :meth:`Span.set_attrs`), or a :class:`SpanData` instance + for a typed one. Exceptions are recorded on the span and + re-raised. + + ``parent`` overrides the ambient parent for this span: a live + :class:`Span`, or one restored from another process to continue its + trace here. The default parents under the current span. + + ``set_as_current=False`` keeps the span from becoming current: + work done while it is open parents to *its* parent instead. Used by + ai.stream because of the context manager api. + """ + # type checkers can't apply asynccontextmanager + # to an overloaded function directly. + return _span_impl( + name_or_data, + parent=parent, + replay=replay, + set_as_current=set_as_current, + ) + + +@contextlib.asynccontextmanager +async def _span_impl( + name_or_data: str | SpanData, + /, + *, + parent: Span | None, + replay: bool, + set_as_current: bool, +) -> AsyncIterator[Span[Any]]: + sp: Span[Any] = create_span( + name_or_data, + parent=parent, + replay=replay, + set_as_current=set_as_current, + ) + if not sp.id: + # noop: no timestamps, no pushes, never current + yield sp + return + sp.stamp_start() + await sp.push() + token = _current.set(sp) if set_as_current else None + try: + yield sp + except BaseException as exc: + # GeneratorExit is how a consumer closes a stream early. + if not isinstance(exc, GeneratorExit) and sp.error is None: + sp.error = SpanError.from_exception(exc) + raise + finally: + # a span that set itself as current must be closed while it is + # still current, in the task that opened it. + misordered = False + if token is not None: + if _current.get() is not sp: + misordered = True + else: + try: + _current.reset(token) + except ValueError: + # token from a different task's context. + misordered = True + sp.ended_at = now_ns() + await sp.push() + if misordered: + raise RuntimeError( + f"span {sp.name!r} closed out of order: it is not the " + "current span at close, or was closed in a different " + "task than it was opened in. A span that sets itself " + "as current must be closed innermost-first in the " + "opening task; open overlapping spans with " + "set_as_current=False." + ) + + +class AdapterMixin(AdapterProtocol): + """The driver :func:`adapter` mixes into a decorated class. + + Implements the adapter callbacks. Don't subclass it directly. Put + :func:`adapter` over a class with ``__call__`` async generator method. + """ + + # Name-mangled so the driver's bookkeeping can never collide with + # an attribute of the class it is mixed into; created lazily so + # that class keeps its ``__init__`` entirely to itself. + __live: dict[str, AsyncGenerator[None, Any]] | None = None + + # Positional-only params on all overridable methods, so overrides + # are free to pick their own names. + def __call__(self, span: Span, /) -> AsyncGenerator[None, Any] | None: + """Return the generator frame for one span, or ``None``. + + The decorated class's ``__call__`` overrides this. + """ + return None + + async def on_span_start(self, span: Span, /) -> None: + gen = self(span) + if gen is None: + return + try: + await anext(gen) + except StopAsyncIteration: + return # returned before yielding: opted out of this span + if self.__live is None: + self.__live = {} + self.__live[span.id] = gen + + async def on_span_event(self, span: Span, event: SpanEvent, /) -> None: + live = self.__live + gen = live.get(span.id) if live is not None else None + if live is None or gen is None: + return # opted out, span already ended, or start raised + try: + await gen.asend(event) + except StopAsyncIteration: + # Finished mid-span: opted out of the rest of this span, + # including its end. + del live[span.id] + except BaseException: + # The generator frame is dead, don't resume it again at + # span end. The error itself is logged by _dispatch. + del live[span.id] + raise + + async def on_span_end(self, span: Span, /) -> None: + gen = self.__live.pop(span.id, None) if self.__live else None + if gen is None: # opted out, or start raised (and was logged) + return + try: + await anext(gen) + except StopAsyncIteration: + return + await gen.aclose() + raise RuntimeError("adapter generator yielded again after span end") + + +class _AdapterFn(AdapterMixin): + """Adapter built by :func:`adapter` from a function.""" + + def __init__(self, fn: Callable[[Span], AsyncGenerator[None, Any]]) -> None: + self._fn = fn + + def __repr__(self) -> str: + return f"adapter({getattr(self._fn, '__qualname__', self._fn)!r})" + + def __call__(self, span: Span, /) -> AsyncGenerator[None, Any]: + return self._fn(span) + + +_AdapterCallableT = TypeVar("_AdapterCallableT", bound=AdapterCallable) + + +@overload +def adapter(target: type[_AdapterCallableT]) -> type[_AdapterCallableT]: ... + + +@overload +def adapter( + target: Callable[[Span], AsyncGenerator[None, Any]], +) -> AdapterMixin: ... + + +def adapter( + target: type[_AdapterCallableT] + | Callable[[Span], AsyncGenerator[None, Any]], +) -> type[_AdapterCallableT] | AdapterMixin: + """Build an adapter from an async generator function or class. + + Decorating a free-standing async generator function returns a + ready-to-register adapter. + + Decorating a class whose ``__call__`` is an + async generator method returns that class with the adapter + machinery (:class:`AdapterMixin`) mixed in:: + + @adapter + class Vendor: + def __init__(self, *, api_key): + self._client = sdk.Client(api_key) + + async def __call__(self, span): + ... # same yield loop as the function form + + ai.telemetry.register(Vendor(api_key="...")) + + How to use to bridge a vendor SDKs: + write the vendor's ``with``/``async with`` around one yield loop. + Code before the loop runs at span start; each span event resumes the + yield with the :class:`SpanEvent`, live; span end resumes it with + ``None``, and the code after the loop runs with ``span.data`` + fully populated and ``ended_at`` set:: + + @adapter + async def vendor(span): + with sdk.start_span(span.name) as v: + while (ev := (yield)) is not None: # each event, live + v.log_event(ev.name, timestamp=ev.time_ns) + v.update(output=span.data) # span end + + ai.telemetry.register(vendor) + + - A failed span ends the loop normally, like any other; read + ``span.error`` (a serializable :class:`SpanError`, never a live + exception — the span may have failed in another process) after + the loop to report it. + - A span that lived elsewhere and arrives complete is replayed to + the generator as start, events, end, back to back — write the + bridge against the span's fields and it handles both live and + after-the-fact delivery. + - Returning before the first ``yield`` skips that span — cheap + filtering by span type. Returning mid-span, from inside the + loop, opts out of the rest of that span, including its end. + - Don't make a bare ``yield`` that ignores the sent value, because + it runs the code after it on the first event, not at span + end. Always loop until the yield returns ``None``. + - Like any adapter, a generator that raises is logged and skipped; + it never kills the run. + """ + if inspect.isclass(target): + if issubclass(target, AdapterMixin): + raise TypeError(f"{target.__name__} is already an adapter") + call = inspect.getattr_static(target, "__call__", None) + if not inspect.isasyncgenfunction(call): + raise TypeError( + "@adapter on a class requires __call__ to be an async " + "generator method (`async def` containing a `yield` loop)" + ) + mixed = type( + target.__name__, + (target, AdapterMixin), + { + "__module__": target.__module__, + "__qualname__": target.__qualname__, + "__doc__": target.__doc__, + }, + ) + return cast("type[_AdapterCallableT]", mixed) + if not inspect.isasyncgenfunction(target): + raise TypeError( + "@adapter requires an async generator function " + "(`async def` containing a `yield` loop)" + ) + return _AdapterFn(target) diff --git a/src/ai/telemetry/utils/__init__.py b/src/ai/telemetry/utils/__init__.py new file mode 100644 index 00000000..9b7bf2b8 --- /dev/null +++ b/src/ai/telemetry/utils/__init__.py @@ -0,0 +1,6 @@ +"""Debugging helpers: the console adapter and the terminal trace viewer. + +Import the submodules directly:: + + from ai.telemetry.utils import console +""" diff --git a/src/ai/telemetry/utils/console.py b/src/ai/telemetry/utils/console.py new file mode 100644 index 00000000..fdd8a522 --- /dev/null +++ b/src/ai/telemetry/utils/console.py @@ -0,0 +1,153 @@ +"""Console adapter: prints spans to a terminal as they happen. + +Register it like any adapter:: + + from ai.telemetry.utils import console + ai.telemetry.register(console.ConsoleAdapter()) + +Prints one ``▸`` line when a span starts (so long runs are visible +live) and, when a trace's root span ends, the whole tree with +durations. +""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +from ... import telemetry + +if TYPE_CHECKING: + from typing import TextIO + + +def _short(text: str, limit: int = 60) -> str: + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _label(sp: telemetry.Span) -> str: + match sp.data: + case telemetry.AiStreamSpanData() as d: + tokens = ( + f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" + if d.usage is not None + else "" + ) + return f"chat {d.model}{tokens}" + case telemetry.AiGenerateSpanData() as d: + tokens = ( + f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" + if d.usage is not None + else "" + ) + return f"generate {d.model}{tokens}" + case telemetry.DecideSpanData() as d: + return f"decide {d.model}" + case telemetry.EmbedSpanData() as d: + tokens = ( + f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" + if d.usage is not None + else "" + ) + return f"embed {d.model}{tokens}" + case telemetry.GenerateAudioSpanData() as d: + return f"generate_audio {d.model}" + case telemetry.GenerateImageSpanData() as d: + return f"generate_image {d.model}" + case telemetry.GenerateVideoSpanData() as d: + return f"generate_video {d.model}" + case telemetry.RerankSpanData() as d: + return f"rerank {d.model}" + case telemetry.TranscribeSpanData() as d: + return f"transcribe {d.model}" + case telemetry.ToolExecutionSpanData() as d: + args = ", ".join(f"{k}={v!r}" for k, v in (d.args or {}).items()) + return f"tool {d.tool_name}({_short(args)})" + case telemetry.HookSpanData() as d: + return f"hook {d.label} {d.hook_type} [{d.status}]" + case telemetry.RunSpanData() as d: + return f"run {d.agent} ({d.model})" + case telemetry.CustomSpanData() as d: + attrs = ", ".join(f"{k}={v!r}" for k, v in d.attrs.items()) + return sp.name + (f" ({_short(attrs)})" if attrs else "") + case _: + return sp.name + + +def _line(sp: telemetry.Span) -> str: + started = sp.started_at or 0 + end = sp.ended_at or started + # A span's lifetime can extend past the response (tool dispatch + # while the stream is open); when the milestone is there, report + # the true response latency instead. + for ev in sp.events: + if ev.name == telemetry.RESPONSE_COMPLETE: + end = ev.time_ns + break + duration = (end - started) / 1e9 + replay = "↻ " if sp.replay else "" + error = ( + f" ✗ {sp.error.type}: {_short(sp.error.message)}" + if sp.error is not None + else "" + ) + return f"{replay}{_label(sp)} {duration:.2f}s{error}" + + +class ConsoleAdapter: + """Print spans to ``out`` (default: stdout).""" + + def __init__(self, *, out: TextIO | None = None) -> None: + self._out = out if out is not None else sys.stdout + self._depth: dict[str, int] = {} + self._ended: dict[str, list[telemetry.Span]] = {} + + async def on_span_start(self, span: telemetry.Span) -> None: + depth = 0 + if span.parent_id is not None: + depth = self._depth.get(span.parent_id, -1) + 1 + self._depth[span.id] = depth + replay = "↻ " if span.replay else "" + self._out.write(f"▸ {' ' * depth}{replay}{_label(span)}\n") + + async def on_span_event( + self, span: telemetry.Span, event: telemetry.SpanEvent + ) -> None: + depth = self._depth.get(span.id, 0) + 1 + offset_ms = (event.time_ns - (span.started_at or 0)) / 1e6 + attrs = ", ".join(f"{k}={v!r}" for k, v in event.attrs.items()) + suffix = f" ({_short(attrs)})" if attrs else "" + self._out.write( + f"· {' ' * depth}{event.name} +{offset_ms:.0f}ms{suffix}\n" + ) + + async def on_span_end(self, span: telemetry.Span) -> None: + self._ended.setdefault(span.trace_id, []).append(span) + if span.parent_id is not None: + return + + # Root ended: print the tree and forget the trace. + spans = self._ended.pop(span.trace_id) + for s in spans: + self._depth.pop(s.id, None) + children: dict[str | None, list[telemetry.Span]] = {} + for s in spans: + children.setdefault(s.parent_id, []).append(s) + + lines = [f"trace {span.trace_id}"] + + def render(s: telemetry.Span, prefix: str, kid_prefix: str) -> None: + lines.append(prefix + _line(s)) + kids = sorted( + children.get(s.id, []), key=lambda c: c.started_at or 0 + ) + for i, kid in enumerate(kids): + last = i == len(kids) - 1 + render( + kid, + kid_prefix + ("└─ " if last else "├─ "), + kid_prefix + (" " if last else "│ "), + ) + + render(span, "", "") + self._out.write("\n".join(lines) + "\n") diff --git a/src/ai/telemetry/utils/viewer.py b/src/ai/telemetry/utils/viewer.py new file mode 100644 index 00000000..474cca8b --- /dev/null +++ b/src/ai/telemetry/utils/viewer.py @@ -0,0 +1,131 @@ +"""Terminal trace viewer: an OTLP/HTTP server that prints span trees. + +Run:: + + python -m ai.telemetry.utils.viewer [--port 4318] + +and point any OTLP/HTTP exporter at ``http://127.0.0.1:4318/v1/traces`` +(the OTLP/HTTP default); see +https://opentelemetry.io/docs/languages/python/exporters/#otlp for the +exporter setup. Then register the adapter:: + + import ai + from ai.telemetry import otel + + ai.telemetry.register(otel.OtelAdapter()) + +Spans are buffered per trace, and the tree is printed when the trace's +root span arrives (roots end — and therefore export — last). +""" + +from __future__ import annotations + +import argparse +import sys +from http import server +from typing import Any + +from ... import errors + +try: + from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 +except ModuleNotFoundError as exc: # pragma: no cover + raise errors.InstallationError( + "could not import `opentelemetry.proto`, which the telemetry viewer " + "needs to decode OTLP, you can install it with " + "`pip install opentelemetry-exporter-otlp-proto-http`" + ) from exc + +_traces: dict[bytes, list[Any]] = {} + + +def _value(any_value: Any) -> Any: + kind = any_value.WhichOneof("value") + return getattr(any_value, kind) if kind else None + + +def _line(span: Any) -> str: + attrs = {kv.key: _value(kv.value) for kv in span.attributes} + duration = (span.end_time_unix_nano - span.start_time_unix_nano) / 1e9 + tokens = "" + if "gen_ai.usage.input_tokens" in attrs: + tokens = ( + f" in:{attrs['gen_ai.usage.input_tokens']}" + f" out:{attrs.get('gen_ai.usage.output_tokens', '?')} tok" + ) + replay = "↻ " if attrs.get("ai.replay") else "" + error = " ✗" if span.status.code == 2 else "" # STATUS_CODE_ERROR + return f"{replay}{span.name}{tokens} {duration:.2f}s{error}" + + +def _print_trace(spans: list[Any]) -> None: + children: dict[bytes, list[Any]] = {} + roots = [] + for span in spans: + if span.parent_span_id: + children.setdefault(span.parent_span_id, []).append(span) + else: + roots.append(span) + + lines = [f"trace {spans[0].trace_id.hex()}"] + + def render(span: Any, prefix: str, kid_prefix: str) -> None: + lines.append(prefix + _line(span)) + kids = sorted( + children.get(span.span_id, []), + key=lambda s: s.start_time_unix_nano, + ) + for i, kid in enumerate(kids): + last = i == len(kids) - 1 + render( + kid, + kid_prefix + ("└─ " if last else "├─ "), + kid_prefix + (" " if last else "│ "), + ) + + for root in roots: + render(root, "", "") + sys.stdout.write("\n".join(lines) + "\n") + sys.stdout.flush() + + +class _Handler(server.BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 — stdlib API name + length = int(self.headers.get("content-length") or 0) + request = trace_service_pb2.ExportTraceServiceRequest() + request.ParseFromString(self.rfile.read(length)) + for resource_spans in request.resource_spans: + for scope_spans in resource_spans.scope_spans: + for span in scope_spans.spans: + _traces.setdefault(span.trace_id, []).append(span) + if not span.parent_span_id: + _print_trace(_traces.pop(span.trace_id)) + payload = trace_service_pb2.ExportTraceServiceResponse() + body = payload.SerializeToString() + self.send_response(200) + self.send_header("content-type", "application/x-protobuf") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: + """Silence per-request logging.""" + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Print OTLP traces to the terminal as trees." + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=4318) + args = parser.parse_args() + httpd = server.HTTPServer((args.host, args.port), _Handler) + sys.stdout.write(f"listening on http://{args.host}:{args.port}/v1/traces\n") + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/tests/agents/test_agent.py b/tests/agents/test_agent.py index 48e3155c..5a867d21 100644 --- a/tests/agents/test_agent.py +++ b/tests/agents/test_agent.py @@ -30,8 +30,8 @@ def _by_name( recorder: Recorder, -) -> dict[str, list[ai.experimental_telemetry.Span]]: - spans: dict[str, list[ai.experimental_telemetry.Span]] = {} +) -> dict[str, list[ai.telemetry.Span]]: + spans: dict[str, list[ai.telemetry.Span]] = {} for s in recorder.ended: spans.setdefault(s.name, []).append(s) return spans @@ -77,7 +77,7 @@ async def test_agent_run_span_tree(recorder: Recorder) -> None: @ai.tool async def lookup(x: int) -> str: """Tool that opens a user span.""" - async with ai.experimental_telemetry.span("user_work") as sp: + async with ai.telemetry.span("user_work") as sp: sp.set_attrs(x=x) return "ok" @@ -110,7 +110,7 @@ async def lookup(x: int) -> str: assert {s.trace_id for s in recorder.ended} == {run.trace_id} assert not any(s.replay for s in recorder.ended) - assert isinstance(run.data, ai.experimental_telemetry.RunSpanData) + assert isinstance(run.data, ai.telemetry.RunSpanData) assert run.data.agent == "Agent" assert run.data.model == "mock-model" assert run.data.provider == "mock" @@ -120,19 +120,17 @@ async def lookup(x: int) -> str: assert run.data.final_message is not None assert run.data.final_message.text == "done" - assert isinstance( - tool_span.data, ai.experimental_telemetry.ToolExecutionSpanData - ) + assert isinstance(tool_span.data, ai.telemetry.ToolExecutionSpanData) assert tool_span.data.tool_name == "lookup" assert tool_span.data.tool_description == "Tool that opens a user span." assert tool_span.data.args == {"x": 1} assert tool_span.data.result == "ok" assert not tool_span.data.is_error - assert isinstance(calls[0].data, ai.experimental_telemetry.AiStreamSpanData) + assert isinstance(calls[0].data, ai.telemetry.AiStreamSpanData) assert calls[0].data.provider == "mock" assert calls[0].data.tool_names == ["lookup"] - assert isinstance(calls[1].data, ai.experimental_telemetry.AiStreamSpanData) + assert isinstance(calls[1].data, ai.telemetry.AiStreamSpanData) assert calls[1].data.message is not None assert calls[1].data.message.text == "done" @@ -172,7 +170,7 @@ async def lookup(x: int) -> str: pass (run,) = _by_name(recorder)["run"] - assert isinstance(run.data, ai.experimental_telemetry.RunSpanData) + assert isinstance(run.data, ai.telemetry.RunSpanData) assert run.data.usage is not None assert run.data.usage.input_tokens == 30 assert run.data.usage.output_tokens == 5 @@ -258,7 +256,7 @@ async def test_early_break_closes_span_tree(recorder: Recorder) -> None: async for _ in stream: break - assert ai.experimental_telemetry.current_span() is None + assert ai.telemetry.current_span() is None assert {s.id for s in recorder.ended} == {s.id for s in recorder.started} names = [s.name for s in recorder.ended] assert names[-1] == "run" @@ -274,7 +272,7 @@ async def test_consumer_error_closes_span_tree(recorder: Recorder) -> None: async for _ in stream: raise ValueError("stop") - assert ai.experimental_telemetry.current_span() is None + assert ai.telemetry.current_span() is None assert {s.id for s in recorder.ended} == {s.id for s in recorder.started} assert [s.name for s in recorder.ended][-1] == "run" @@ -292,9 +290,7 @@ async def boom() -> str: pass (tool_span,) = _by_name(recorder)["tool_execution"] - assert isinstance( - tool_span.data, ai.experimental_telemetry.ToolExecutionSpanData - ) + assert isinstance(tool_span.data, ai.telemetry.ToolExecutionSpanData) # The framework converts the exception into an error result (the # run keeps going), but the failure is threaded onto the span as # serializable data. diff --git a/tests/agents/test_hooks.py b/tests/agents/test_hooks.py index ff096f65..caca060f 100644 --- a/tests/agents/test_hooks.py +++ b/tests/agents/test_hooks.py @@ -373,7 +373,7 @@ async def loop( await ai.hook(self.label, payload=Confirmation) -def _hook_spans(recorder: Recorder) -> list[ai.experimental_telemetry.Span]: +def _hook_spans(recorder: Recorder) -> list[ai.telemetry.Span]: return [s for s in recorder.ended if s.name == "hook"] @@ -389,7 +389,7 @@ async def test_live_hook_span(recorder: Recorder) -> None: ai.resolve_hook(my_agent.label, {"approved": True}) (hook_span,) = _hook_spans(recorder) - assert isinstance(hook_span.data, ai.experimental_telemetry.HookSpanData) + assert isinstance(hook_span.data, ai.telemetry.HookSpanData) assert hook_span.data.status == "resolved" assert hook_span.data.resolution == {"approved": True} assert hook_span.data.tool_call_id is None @@ -397,8 +397,8 @@ async def test_live_hook_span(recorder: Recorder) -> None: # The suspension's timeline: deferred once resolvers can see it, # resolved when the external input arrived. deferred, resolved = hook_span.events - assert deferred.name == ai.experimental_telemetry.HOOK_DEFERRED - assert resolved.name == ai.experimental_telemetry.HOOK_RESOLVED + assert deferred.name == ai.telemetry.HOOK_DEFERRED + assert resolved.name == ai.telemetry.HOOK_RESOLVED assert deferred.time_ns <= resolved.time_ns @@ -428,12 +428,12 @@ async def loop( await ai.cancel_hook(my_agent.label, reason="denied") (hook_span,) = _hook_spans(recorder) - assert isinstance(hook_span.data, ai.experimental_telemetry.HookSpanData) + assert isinstance(hook_span.data, ai.telemetry.HookSpanData) assert hook_span.data.status == "cancelled" assert hook_span.data.resolution is None deferred, cancelled = hook_span.events - assert deferred.name == ai.experimental_telemetry.HOOK_DEFERRED - assert cancelled.name == ai.experimental_telemetry.HOOK_CANCELLED + assert deferred.name == ai.telemetry.HOOK_DEFERRED + assert cancelled.name == ai.telemetry.HOOK_CANCELLED assert cancelled.attrs == {"reason": "denied"} @@ -449,7 +449,7 @@ async def test_pre_registered_hook_is_replay_span(recorder: Recorder) -> None: (hook_span,) = _hook_spans(recorder) assert hook_span.replay - assert isinstance(hook_span.data, ai.experimental_telemetry.HookSpanData) + assert isinstance(hook_span.data, ai.telemetry.HookSpanData) assert hook_span.data.status == "resolved" # Pre-registration validated against the payload type, so the # recorded resolution is the full model dump, defaults included. diff --git a/tests/agents/test_run_blocked.py b/tests/agents/test_run_blocked.py index 07fd6d86..1033ffc3 100644 --- a/tests/agents/test_run_blocked.py +++ b/tests/agents/test_run_blocked.py @@ -110,10 +110,10 @@ def index(pred: Any) -> int: # The block/unblock cycle lands on the spans: the run resolved, so # it ends unblocked; the hook records its tool call and resolution. (run_span,) = [s for s in recorder.ended if s.name == "run"] - assert isinstance(run_span.data, ai.experimental_telemetry.RunSpanData) + assert isinstance(run_span.data, ai.telemetry.RunSpanData) assert not run_span.data.blocked (hook_span,) = [s for s in recorder.ended if s.name == "hook"] - assert isinstance(hook_span.data, ai.experimental_telemetry.HookSpanData) + assert isinstance(hook_span.data, ai.telemetry.HookSpanData) assert hook_span.data.tool_call_id == "tc-1" assert hook_span.data.resolution == {"granted": True, "reason": "ok"} @@ -236,7 +236,7 @@ async def test_abort_leaves_blocked(recorder: Recorder) -> None: # The run span ends still blocked, mirroring ``stream.blocked``. (run_span,) = [s for s in recorder.ended if s.name == "run"] - assert isinstance(run_span.data, ai.experimental_telemetry.RunSpanData) + assert isinstance(run_span.data, ai.telemetry.RunSpanData) assert run_span.data.blocked diff --git a/tests/conftest.py b/tests/conftest.py index 53d2f497..941abd4b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,7 +89,7 @@ async def generate( ) if self._generate_impl is None: # Mirrors the base class: providers without a native - # generate() make ai.experimental_generate() fall back to streaming. + # generate() make ai.generate() fall back to streaming. raise NotImplementedError( "MockProvider: no generate implementation configured" ) @@ -279,29 +279,29 @@ class Recorder: """Telemetry adapter that records every span start/end it sees.""" def __init__(self) -> None: - self.started: list[ai.experimental_telemetry.Span] = [] - self.ended: list[ai.experimental_telemetry.Span] = [] + self.started: list[ai.telemetry.Span] = [] + self.ended: list[ai.telemetry.Span] = [] - async def on_span_start(self, span: ai.experimental_telemetry.Span) -> None: + async def on_span_start(self, span: ai.telemetry.Span) -> None: self.started.append(span) async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: pass - async def on_span_end(self, span: ai.experimental_telemetry.Span) -> None: + async def on_span_end(self, span: ai.telemetry.Span) -> None: self.ended.append(span) @pytest.fixture def recorder() -> Iterator[Recorder]: r = Recorder() - ai.experimental_telemetry.register(r) + ai.telemetry.register(r) yield r - ai.experimental_telemetry.unregister(r) + ai.telemetry.unregister(r) BLOCKING_TASKS: dict[ diff --git a/tests/external/test_harness.py b/tests/external/test_harness.py index 3dd4f20c..d1f14818 100644 --- a/tests/external/test_harness.py +++ b/tests/external/test_harness.py @@ -929,7 +929,7 @@ async def test_stream_reply_is_in_history_while_telemetry_is_pending() -> None: release_telemetry = asyncio.Event() class BlockingSink: - async def on_push(self, span: ai.experimental_telemetry.Span) -> None: + async def on_push(self, span: ai.telemetry.Span) -> None: if ( isinstance(span.data, external.ExternalHarnessMessageSpanData) and span.ended_at is not None @@ -946,7 +946,7 @@ async def reply() -> m.Message: pass return stream.message - async with ai.experimental_telemetry.use_sink(BlockingSink()), harness: + async with ai.telemetry.use_sink(BlockingSink()), harness: task = asyncio.create_task(reply()) try: async with asyncio.timeout(5): @@ -972,8 +972,8 @@ def reply(msg: m.Message) -> list[Any]: _wire.TurnEnd(), ] - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): harness = fakes.fake(reply) async with harness: async with ai.stream(harness, [ai.user_message("go")]) as s: @@ -993,9 +993,7 @@ def reply(msg: m.Message) -> list[Any]: ) assert isinstance(session.data, external.ExternalHarnessSessionSpanData) assert isinstance(message.data, external.ExternalHarnessMessageSpanData) - assert isinstance( - tool.data, ai.experimental_telemetry.ToolExecutionSpanData - ) + assert isinstance(tool.data, ai.telemetry.ToolExecutionSpanData) assert session.data.session_id == "s1" assert message.parent_id == session.id assert message.data.finish_reason == "stop" diff --git a/tests/external/test_live.py b/tests/external/test_live.py index 68f3b03d..f09b48bb 100644 --- a/tests/external/test_live.py +++ b/tests/external/test_live.py @@ -70,7 +70,7 @@ async def test_a_turn_with_a_cli_tool_and_a_resume( # The session id is all that a later process needs, with the settings. again = make(resume=harness.session.id) async with asyncio.timeout(240), again: - reply = await ai.experimental_generate( + reply = await ai.generate( again, [ai.user_message("Which word did you read? One word.")] ) assert "HERON" in reply.text.upper() @@ -115,7 +115,7 @@ async def test_cancel_interrupts_and_the_session_continues( break # stop reading: the turn stops cut = harness.session.messages[-1].provider_metadata or {} assert cut.get("external") == {"stop_reason": "interrupted"} - reply = await ai.experimental_generate( + reply = await ai.generate( harness, [*history, ai.user_message("Reply with the word OK only.")] ) assert "OK" in reply.text.upper() @@ -170,6 +170,6 @@ async def test_a_new_session_uses_the_given_past( ), ] async with asyncio.timeout(240), harness: - reply = await ai.experimental_generate(harness, history) + reply = await ai.generate(harness, history) assert "GENTIAN-31" in reply.text.upper() assert harness.session.id is not None diff --git a/tests/models/core/test_api.py b/tests/models/core/test_api.py index 67925265..aaba24f1 100644 --- a/tests/models/core/test_api.py +++ b/tests/models/core/test_api.py @@ -350,32 +350,42 @@ async def _generate( provider._generate_impl = _generate - result = await models.experimental_generate(model, [ai.user_message("Hi")]) + result = await models.generate(model, [ai.user_message("Hi")]) assert called assert result is sentinel (span,) = [s for s in recorder.ended if s.name == "ai_generate"] - assert isinstance(span.data, ai.experimental_telemetry.AiGenerateSpanData) + assert isinstance(span.data, ai.telemetry.AiGenerateSpanData) assert span.data.provider == "mock" # Adapters see pushed snapshots, so compare by value, not identity. assert span.data.message == sentinel assert span.data.usage == sentinel.usage +@pytest.mark.parametrize("module", [ai, models, models.core, models.core.api]) +async def test_experimental_generate_alias(module: Any) -> None: + mock_llm([[text_msg("legacy reply")]]) + + with pytest.warns(DeprecationWarning, match="use ai.generate"): + generate = module.experimental_generate + + assert generate is ai.generate + result = await generate(MOCK_MODEL, [ai.user_message("Hi")]) + assert result.text == "legacy reply" + + async def test_generate_falls_back_to_stream(recorder: Recorder) -> None: - # MOCK_PROVIDER has no native generate(): experimental_generate drains the + # MOCK_PROVIDER has no native generate(): generate drains the # stream and returns the aggregated message. mock_llm([[text_msg("streamed reply")]]) - result = await models.experimental_generate( - MOCK_MODEL, [ai.user_message("Hi")] - ) + result = await models.generate(MOCK_MODEL, [ai.user_message("Hi")]) assert result.text == "streamed reply" (span,) = [s for s in recorder.ended if s.name == "ai_generate"] - assert isinstance(span.data, ai.experimental_telemetry.AiGenerateSpanData) + assert isinstance(span.data, ai.telemetry.AiGenerateSpanData) assert span.data.message is not None assert span.data.message.text == "streamed reply" @@ -396,7 +406,7 @@ async def _dying_stream( model = models.Model(id="dying-model", provider=provider) with pytest.raises(ai.errors.ProviderIncompleteResponseError): - await models.experimental_generate(model, [ai.user_message("Hi")]) + await models.generate(model, [ai.user_message("Hi")]) async def test_generate_uses_model_protocol() -> None: @@ -424,7 +434,7 @@ async def generate( _ = client, model, messages, tools, output_type, params, provider return sentinel - result = await models.experimental_generate( + result = await models.generate( MOCK_MODEL.with_protocol(OverrideProtocol()), [ai.user_message("Hi")], ) @@ -641,20 +651,24 @@ async def test_stream_span_milestones(recorder: Recorder) -> None: mock_llm([[text_msg("Hello world")]]) async with models.stream(MOCK_MODEL, [ai.user_message("Hi")]) as stream: - assert stream.experimental_span is not None - assert stream.experimental_span.events == [] + assert stream.telemetry_span is not None + assert stream.telemetry_span.events == [] + with pytest.warns( + DeprecationWarning, match="use Stream.telemetry_span" + ): + assert stream.experimental_span is stream.telemetry_span async for _ in stream: pass (call,) = [s for s in recorder.ended if s.name == "ai_stream"] - assert stream.experimental_span is not None - assert stream.experimental_span.id == call.id - assert isinstance(call.data, ai.experimental_telemetry.AiStreamSpanData) + assert stream.telemetry_span is not None + assert stream.telemetry_span.id == call.id + assert isinstance(call.data, ai.telemetry.AiStreamSpanData) assert call.data.provider == "mock" assert call.data.tool_names is None # no tools were sent assert [e.name for e in call.events] == [ - ai.experimental_telemetry.FIRST_TOKEN, - ai.experimental_telemetry.RESPONSE_COMPLETE, + ai.telemetry.FIRST_TOKEN, + ai.telemetry.RESPONSE_COMPLETE, ] first, complete = call.events # The milestone says what kind of output arrived first. @@ -685,9 +699,9 @@ async def _no_starts_stream( async for _ in stream: pass - assert stream.experimental_span is not None - first = stream.experimental_span.events[0] - assert first.name == ai.experimental_telemetry.FIRST_TOKEN + assert stream.telemetry_span is not None + first = stream.telemetry_span.events[0] + assert first.name == ai.telemetry.FIRST_TOKEN assert first.attrs == {"event_type": "TextDelta"} @@ -711,8 +725,8 @@ async def test_replay_stream_has_no_milestones(recorder: Recorder) -> None: assert events, "replay should still emit ToolEnd events" (call,) = [s for s in recorder.ended if s.name == "ai_stream"] - assert stream.experimental_span is not None - assert stream.experimental_span.id == call.id + assert stream.telemetry_span is not None + assert stream.telemetry_span.id == call.id assert call.events == [] @@ -726,9 +740,7 @@ async def test_early_close_no_response_complete(recorder: Recorder) -> None: break (call,) = [s for s in recorder.ended if s.name == "ai_stream"] - assert [e.name for e in call.events] == [ - ai.experimental_telemetry.FIRST_TOKEN - ] + assert [e.name for e in call.events] == [ai.telemetry.FIRST_TOKEN] async def test_stream_span_captures_response_identity( @@ -761,7 +773,7 @@ class Out(pydantic.BaseModel): pass (call,) = [s for s in recorder.ended if s.name == "ai_stream"] - assert isinstance(call.data, ai.experimental_telemetry.AiStreamSpanData) + assert isinstance(call.data, ai.telemetry.AiStreamSpanData) assert call.data.output_type == "Out" assert call.data.finish_reason == "length" assert call.data.response_id == "resp-1" @@ -772,7 +784,7 @@ async def test_directly_constructed_stream_has_no_span() -> None: async with ai.Stream.replay_message(text_msg("hi")) as stream: async for _ in stream: pass - assert stream.experimental_span is None + assert stream.telemetry_span is None async def test_replayed_turn_gets_replay_span(recorder: Recorder) -> None: @@ -785,6 +797,6 @@ async def test_replayed_turn_gets_replay_span(recorder: Recorder) -> None: (call,) = [s for s in recorder.ended if s.name == "ai_stream"] assert call.replay - assert isinstance(call.data, ai.experimental_telemetry.AiStreamSpanData) + assert isinstance(call.data, ai.telemetry.AiStreamSpanData) assert call.data.message is not None assert call.data.message.text == "prior turn" diff --git a/tests/models/core/test_model.py b/tests/models/core/test_model.py index 0027dc76..8b7b3d2e 100644 --- a/tests/models/core/test_model.py +++ b/tests/models/core/test_model.py @@ -68,7 +68,7 @@ async def test_stream_takes_a_model_like() -> None: async def test_generate_takes_a_model_like() -> None: - message = await ai.experimental_generate(Echo(), [ai.user_message("hi")]) + message = await ai.generate(Echo(), [ai.user_message("hi")]) assert message.text == "hi" diff --git a/tests/ops/experimental/test_decisions.py b/tests/ops/experimental/test_decisions.py index b56ad2b8..d1b1580e 100644 --- a/tests/ops/experimental/test_decisions.py +++ b/tests/ops/experimental/test_decisions.py @@ -170,7 +170,7 @@ async def test_decide_dispatch_and_span(recorder: conftest.Recorder) -> None: assert result.provider_metadata == {"test": {"request_id": "req_1"}} (span,) = recorder.ended - assert isinstance(span.data, ai.experimental_telemetry.DecideSpanData) + assert isinstance(span.data, ai.telemetry.DecideSpanData) assert span.data.question_count == 3 assert span.data.answer_count == 3 assert span.data.usage == ai.types.usage.Usage(input_tokens=12) diff --git a/tests/ops/test_audio.py b/tests/ops/test_audio.py index 81199279..b43e653e 100644 --- a/tests/ops/test_audio.py +++ b/tests/ops/test_audio.py @@ -49,7 +49,7 @@ async def test_generate_audio_span(recorder: conftest.Recorder) -> None: (span,) = recorder.ended data = span.data - assert isinstance(data, ai.experimental_telemetry.GenerateAudioSpanData) + assert isinstance(data, ai.telemetry.GenerateAudioSpanData) assert data.model == "mock-audio-model" assert data.provider == "mock-audio" assert data.output_count == 1 diff --git a/tests/ops/test_embeddings.py b/tests/ops/test_embeddings.py index 55aa781b..29db71c3 100644 --- a/tests/ops/test_embeddings.py +++ b/tests/ops/test_embeddings.py @@ -51,7 +51,7 @@ async def test_embed_span(recorder: conftest.Recorder) -> None: (span,) = recorder.ended data = span.data - assert isinstance(data, ai.experimental_telemetry.EmbedSpanData) + assert isinstance(data, ai.telemetry.EmbedSpanData) assert data.model == "mock-embed-model" assert data.provider == "mock-embed" assert data.input_count == 2 @@ -63,4 +63,4 @@ async def test_embed_span(recorder: conftest.Recorder) -> None: assert span.error is None (started,) = recorder.started - assert isinstance(started.data, ai.experimental_telemetry.EmbedSpanData) + assert isinstance(started.data, ai.telemetry.EmbedSpanData) diff --git a/tests/ops/test_images.py b/tests/ops/test_images.py index 90ca28df..138b7a2e 100644 --- a/tests/ops/test_images.py +++ b/tests/ops/test_images.py @@ -54,7 +54,7 @@ async def test_generate_image_span_with_warnings( (span,) = recorder.ended data = span.data - assert isinstance(data, ai.experimental_telemetry.GenerateImageSpanData) + assert isinstance(data, ai.telemetry.GenerateImageSpanData) assert data.output_count == 1 assert data.warnings is not None assert data.warnings[0]["kind"] == "unsupported" diff --git a/tests/ops/test_reranking.py b/tests/ops/test_reranking.py index 7f813093..821dff85 100644 --- a/tests/ops/test_reranking.py +++ b/tests/ops/test_reranking.py @@ -66,7 +66,7 @@ async def test_rerank_span(recorder: conftest.Recorder) -> None: (span,) = recorder.ended data = span.data - assert isinstance(data, ai.experimental_telemetry.RerankSpanData) + assert isinstance(data, ai.telemetry.RerankSpanData) assert data.input_count == 2 assert data.top_n == 1 assert data.output_count == 1 diff --git a/tests/ops/test_transcriptions.py b/tests/ops/test_transcriptions.py index 0be9afc1..08824017 100644 --- a/tests/ops/test_transcriptions.py +++ b/tests/ops/test_transcriptions.py @@ -54,5 +54,5 @@ async def test_transcribe_span(recorder: conftest.Recorder) -> None: (span,) = recorder.ended data = span.data - assert isinstance(data, ai.experimental_telemetry.TranscribeSpanData) + assert isinstance(data, ai.telemetry.TranscribeSpanData) assert data.duration_seconds == 1.5 diff --git a/tests/ops/test_videos.py b/tests/ops/test_videos.py index cc476381..b6342dd1 100644 --- a/tests/ops/test_videos.py +++ b/tests/ops/test_videos.py @@ -52,7 +52,7 @@ async def test_error_recorded_on_span(recorder: conftest.Recorder) -> None: (span,) = recorder.ended data = span.data - assert isinstance(data, ai.experimental_telemetry.GenerateVideoSpanData) + assert isinstance(data, ai.telemetry.GenerateVideoSpanData) assert data.output_count is None assert span.error is not None assert span.error.type == "NotImplementedError" diff --git a/tests/experimental_telemetry/__init__.py b/tests/telemetry/__init__.py similarity index 100% rename from tests/experimental_telemetry/__init__.py rename to tests/telemetry/__init__.py diff --git a/tests/telemetry/test_compat.py b/tests/telemetry/test_compat.py new file mode 100644 index 00000000..fbe1cffc --- /dev/null +++ b/tests/telemetry/test_compat.py @@ -0,0 +1,50 @@ +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize( + "legacy_import", + [ + "legacy = ai.experimental_telemetry", + "from ai import experimental_telemetry as legacy", + "import ai.experimental_telemetry as legacy", + ], +) +def test_experimental_telemetry_import_warns(legacy_import: str) -> None: + subprocess.run( + [ + sys.executable, + "-Werror::DeprecationWarning", + "-c", + f""" +import importlib +import sys +import warnings + +import ai +from ai.telemetry import DictSink + +assert 'ai.experimental_telemetry' not in sys.modules +assert 'ai.telemetry.otel' not in sys.modules + +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', DeprecationWarning) + {legacy_import} + +assert len(caught) == 1 +assert caught[0].category is DeprecationWarning +assert 'use ai.telemetry' in str(caught[0].message) +assert caught[0].filename == '' +assert legacy.DictSink is DictSink + +for submodule in ('span', 'otel', 'utils.console', 'utils.viewer'): + old = importlib.import_module(f'ai.experimental_telemetry.{{submodule}}') + assert old is importlib.import_module(f'ai.telemetry.{{submodule}}') + +assert legacy.span is ai.telemetry.span +""", + ], + check=True, + ) diff --git a/tests/experimental_telemetry/test_otel.py b/tests/telemetry/test_otel.py similarity index 88% rename from tests/experimental_telemetry/test_otel.py rename to tests/telemetry/test_otel.py index bca7d3c7..e0f403d3 100644 --- a/tests/experimental_telemetry/test_otel.py +++ b/tests/telemetry/test_otel.py @@ -14,8 +14,8 @@ from opentelemetry.trace import SpanKind import ai -from ai.experimental_telemetry import otel from ai.models.core import params as core_params +from ai.telemetry import otel from ai.types import messages as messages_ from ai.types import usage as usage_ @@ -31,18 +31,18 @@ def otel_env() -> Iterator[tuple[InMemorySpanExporter, TracerProvider]]: provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) adapter = otel.OtelAdapter(tracer_provider=provider, capture_content=True) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) yield exporter, provider - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) async def test_nesting_names_and_attributes( otel_env: tuple[InMemorySpanExporter, TracerProvider], ) -> None: exporter, _ = otel_env - async with ai.experimental_telemetry.span("outer") as sp: + async with ai.telemetry.span("outer") as sp: sp.set_attrs(foo="bar") - async with ai.experimental_telemetry.span("inner"): + async with ai.telemetry.span("inner"): pass spans = {s.name: s for s in exporter.get_finished_spans()} @@ -57,10 +57,10 @@ async def test_trace_attrs_exported_on_every_span( otel_env: tuple[InMemorySpanExporter, TracerProvider], ) -> None: exporter, _ = otel_env - async with ai.experimental_telemetry.span("outer") as sp: + async with ai.telemetry.span("outer") as sp: sp.trace_attrs["session.id"] = "s-1" sp.trace_attrs["obj"] = {"a": 1} # non-scalar: exported as repr - async with ai.experimental_telemetry.span("inner"): + async with ai.telemetry.span("inner"): pass spans = {s.name: s for s in exporter.get_finished_spans()} @@ -109,7 +109,7 @@ async def test_error_status( ) -> None: exporter, _ = otel_env with pytest.raises(ValueError, match="boom"): - async with ai.experimental_telemetry.span("failing"): + async with ai.telemetry.span("failing"): raise ValueError("boom") (span,) = exporter.get_finished_spans() @@ -163,17 +163,17 @@ def __repr__(self) -> str: return "" marker = Marker() - async with ai.experimental_telemetry.span("s") as sp: - first = ai.experimental_telemetry.SpanEvent( + async with ai.telemetry.span("s") as sp: + first = ai.telemetry.SpanEvent( name="first_token", - time_ns=ai.experimental_telemetry.now_ns(), + time_ns=ai.telemetry.now_ns(), attrs={"event_type": "TextStart"}, ) sp.events.append(first) await sp.push() - second = ai.experimental_telemetry.SpanEvent( + second = ai.telemetry.SpanEvent( name="custom", - time_ns=ai.experimental_telemetry.now_ns(), + time_ns=ai.telemetry.now_ns(), attrs={"obj": marker}, ) sp.events.append(second) @@ -211,7 +211,7 @@ async def test_raw_otel_span_nests_under_ours( ) -> None: exporter, provider = otel_env tracer = provider.get_tracer("test") - async with ai.experimental_telemetry.span("outer"): + async with ai.telemetry.span("outer"): with tracer.start_as_current_span("raw"): pass @@ -226,10 +226,8 @@ async def test_non_current_span_not_attached_to_otel_context( ) -> None: exporter, provider = otel_env tracer = provider.get_tracer("test") - async with ai.experimental_telemetry.span("outer"): - async with ai.experimental_telemetry.span( - "overlapping", set_as_current=False - ): + async with ai.telemetry.span("outer"): + async with ai.telemetry.span("overlapping", set_as_current=False): # A raw otel span opened while the non-current span is # open parents like our own spans do: under the outer # span, not under the overlapping one. @@ -255,7 +253,7 @@ async def test_our_root_nests_under_raw_otel_span( exporter, provider = otel_env tracer = provider.get_tracer("test") with tracer.start_as_current_span("raw"): - async with ai.experimental_telemetry.span("inner"): + async with ai.telemetry.span("inner"): pass spans = {s.name: s for s in exporter.get_finished_spans()} @@ -270,22 +268,20 @@ async def test_span_finished_elsewhere_exports_and_parents( exporter, _ = otel_env # "Process one": mint the span and carry it as data. No push — # the record is delivered whole once the work completes. - turn = ai.experimental_telemetry.create_span("turn") - turn.started_at = ai.experimental_telemetry.now_ns() + turn = ai.telemetry.create_span("turn") + turn.started_at = ai.telemetry.now_ns() payload = turn.model_dump(mode="json") # "Process two": live children under the restored span. - restored = ai.experimental_telemetry.Span.model_validate(payload) - async with ai.experimental_telemetry.use_span(restored): - async with ai.experimental_telemetry.span("child"): + restored = ai.telemetry.Span.model_validate(payload) + async with ai.telemetry.use_span(restored): + async with ai.telemetry.span("child"): pass # "Process three": finish the span and push the complete record. - done = ai.experimental_telemetry.Span.model_validate(payload) - done.ended_at = ai.experimental_telemetry.now_ns() - done.error = ai.experimental_telemetry.SpanError( - type="TurnError", message="nope" - ) + done = ai.telemetry.Span.model_validate(payload) + done.ended_at = ai.telemetry.now_ns() + done.error = ai.telemetry.SpanError(type="TurnError", message="nope") await done.push() spans = {s.name: s for s in exporter.get_finished_spans()} @@ -307,7 +303,7 @@ async def test_otel_ids_derived_from_framework_ids( otel_env: tuple[InMemorySpanExporter, TracerProvider], ) -> None: exporter, _ = otel_env - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: pass (span,) = exporter.get_finished_spans() @@ -323,21 +319,19 @@ async def test_subclass_can_enrich_names_and_attributes() -> None: provider.add_span_processor(SimpleSpanProcessor(exporter)) class Enriched(otel.OtelAdapter): - def span_name(self, span: ai.experimental_telemetry.Span, /) -> str: + def span_name(self, span: ai.telemetry.Span, /) -> str: return f"seal:{super().span_name(span)}" - def span_attrs( - self, span: ai.experimental_telemetry.Span, / - ) -> dict[str, Any]: + def span_attrs(self, span: ai.telemetry.Span, /) -> dict[str, Any]: return super().span_attrs(span) | {"extra": True} adapter = Enriched(tracer_provider=provider) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: sp.set_attrs(foo="bar") finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) (span,) = exporter.get_finished_spans() assert span.name == "seal:s" @@ -354,14 +348,14 @@ async def test_content_capture_off_by_default( provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) adapter = otel.OtelAdapter(tracer_provider=provider) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: mock_llm([[text_msg("hello")]]) async with ai.stream(MOCK_MODEL, [ai.user_message("hi")]) as stream: async for _ in stream: pass finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) (span,) = exporter.get_finished_spans() attrs = span.attributes @@ -382,14 +376,14 @@ async def test_content_capture_env_opt_in( provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) adapter = otel.OtelAdapter(tracer_provider=provider) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: mock_llm([[text_msg("hello")]]) async with ai.stream(MOCK_MODEL, [ai.user_message("hi")]) as stream: async for _ in stream: pass finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) (span,) = exporter.get_finished_spans() assert span.attributes is not None @@ -398,12 +392,10 @@ async def test_content_capture_env_opt_in( def _stream_span( **data_kwargs: Any, -) -> ai.experimental_telemetry.Span[Any]: +) -> ai.telemetry.Span[Any]: data_kwargs.setdefault("messages", [ai.user_message("hi")]) - data = ai.experimental_telemetry.AiStreamSpanData( - model="mock-model", **data_kwargs - ) - return ai.experimental_telemetry.Span( + data = ai.telemetry.AiStreamSpanData(model="mock-model", **data_kwargs) + return ai.telemetry.Span( name="ai_stream", data=data, id="span-1", trace_id="trace-1" ) @@ -483,12 +475,12 @@ def test_response_identity_attributes() -> None: def test_tool_span_description() -> None: - data = ai.experimental_telemetry.ToolExecutionSpanData( + data = ai.telemetry.ToolExecutionSpanData( tool_name="lookup", tool_call_id="tc-1", tool_description="Look things up.", ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="tool_execution", data=data, id="span-1", trace_id="trace-1" ) attrs = otel._attributes(sp, capture_content=False) @@ -590,7 +582,7 @@ def test_semconv_message_content_shapes() -> None: def test_generate_span_attributes() -> None: - data = ai.experimental_telemetry.AiGenerateSpanData( + data = ai.telemetry.AiGenerateSpanData( model="mock-model", messages=[ai.user_message("hi")], params=core_params.InferenceRequestParams( @@ -601,7 +593,7 @@ def test_generate_span_attributes() -> None: response_id="resp-1", response_model="mock-model-2", ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="ai_generate", data=data, id="span-1", trace_id="trace-1" ) assert otel._semconv_name(sp) == "chat mock-model" @@ -618,7 +610,7 @@ def test_generate_span_attributes() -> None: def test_embed_span_attributes() -> None: - data = ai.experimental_telemetry.EmbedSpanData( + data = ai.telemetry.EmbedSpanData( model="text-embedding-3-small", provider="openai", input_count=2, @@ -626,7 +618,7 @@ def test_embed_span_attributes() -> None: dimensions=1536, usage=usage_.Usage(input_tokens=9), ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="ops", data=data, id="span-1", trace_id="trace-1" ) assert otel._semconv_name(sp) == "embeddings text-embedding-3-small" @@ -640,11 +632,11 @@ def test_embed_span_attributes() -> None: def test_generate_image_span_operation_name() -> None: - data = ai.experimental_telemetry.GenerateImageSpanData( + data = ai.telemetry.GenerateImageSpanData( model="imagen-4", provider="google", ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="ops", data=data, id="span-1", trace_id="trace-1" ) assert otel._semconv_name(sp) == "generate_content imagen-4" @@ -654,13 +646,13 @@ def test_generate_image_span_operation_name() -> None: def test_decide_span_operation_name() -> None: - data = ai.experimental_telemetry.DecideSpanData( + data = ai.telemetry.DecideSpanData( question_count=3, model="typesafe-ai/jev", provider="ai-gateway", usage=usage_.Usage(input_tokens=9), ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="ops", data=data, id="span-1", trace_id="trace-1" ) assert otel._semconv_name(sp) == "decide typesafe-ai/jev" @@ -673,12 +665,12 @@ def test_decide_span_operation_name() -> None: def test_rerank_span_operation_name() -> None: - data = ai.experimental_telemetry.RerankSpanData( + data = ai.telemetry.RerankSpanData( input_count=1, model="rerank-v3.5", provider="cohere", ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="ops", data=data, id="span-1", trace_id="trace-1" ) assert otel._semconv_name(sp) == "rerank rerank-v3.5" @@ -687,7 +679,7 @@ def test_rerank_span_operation_name() -> None: def test_run_span_attributes() -> None: - data = ai.experimental_telemetry.RunSpanData( + data = ai.telemetry.RunSpanData( agent="MyAgent", model="anthropic/claude-x", messages=[ai.user_message("hi")], @@ -696,7 +688,7 @@ def test_run_span_attributes() -> None: output_type="MyModel", usage=usage_.Usage(input_tokens=30, output_tokens=5), ) - sp = ai.experimental_telemetry.Span( + sp = ai.telemetry.Span( name="run", data=data, id="span-1", trace_id="trace-1" ) attrs = otel._attributes(sp, capture_content=True) diff --git a/tests/experimental_telemetry/test_telemetry.py b/tests/telemetry/test_telemetry.py similarity index 68% rename from tests/experimental_telemetry/test_telemetry.py rename to tests/telemetry/test_telemetry.py index d1375d77..7c603c4c 100644 --- a/tests/experimental_telemetry/test_telemetry.py +++ b/tests/telemetry/test_telemetry.py @@ -21,10 +21,10 @@ async def _add_event( - sp: ai.experimental_telemetry.Span, + sp: ai.telemetry.Span, name: str, **attrs: Any, -) -> ai.experimental_telemetry.SpanEvent: +) -> ai.telemetry.SpanEvent: """The event pattern: append, then push for live delivery.""" event = sp.add_event(name, **attrs) await sp.push() @@ -33,40 +33,39 @@ async def _add_event( @contextlib.asynccontextmanager async def _registered( - adapter: ai.experimental_telemetry.AdapterProtocol - | ai.experimental_telemetry.AdapterCallable, + adapter: ai.telemetry.AdapterProtocol | ai.telemetry.AdapterCallable, ) -> AsyncIterator[None]: - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: yield finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) class _NoopAdapter: """Raw-hook adapter base: tests override the hooks they care about.""" - async def on_span_start(self, span: ai.experimental_telemetry.Span) -> None: + async def on_span_start(self, span: ai.telemetry.Span) -> None: pass async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: pass - async def on_span_end(self, span: ai.experimental_telemetry.Span) -> None: + async def on_span_end(self, span: ai.telemetry.Span) -> None: pass async def test_nesting_and_ids(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("outer") as outer: - assert ai.experimental_telemetry.current_span() is outer - async with ai.experimental_telemetry.span("inner") as inner: + async with ai.telemetry.span("outer") as outer: + assert ai.telemetry.current_span() is outer + async with ai.telemetry.span("inner") as inner: assert inner.parent_id == outer.id assert inner.trace_id == outer.trace_id - assert ai.experimental_telemetry.current_span() is None + assert ai.telemetry.current_span() is None assert outer.parent_id is None assert [s.name for s in recorder.started] == ["outer", "inner"] assert [s.name for s in recorder.ended] == ["inner", "outer"] @@ -75,10 +74,10 @@ async def test_nesting_and_ids(recorder: Recorder) -> None: async def test_task_inherits_current_span(recorder: Recorder) -> None: async def work() -> None: - async with ai.experimental_telemetry.span("child"): + async with ai.telemetry.span("child"): pass - async with ai.experimental_telemetry.span("parent") as parent: + async with ai.telemetry.span("parent") as parent: await asyncio.create_task(work()) child = next(s for s in recorder.ended if s.name == "child") @@ -88,76 +87,72 @@ async def work() -> None: async def test_error_recorded_and_reraised(recorder: Recorder) -> None: with pytest.raises(ValueError, match="boom"): - async with ai.experimental_telemetry.span("failing"): + async with ai.telemetry.span("failing"): raise ValueError("boom") (span,) = recorder.ended # The error is serializable data, not a live exception. - assert span.error == ai.experimental_telemetry.SpanError( + assert span.error == ai.telemetry.SpanError( type="ValueError", message="boom" ) assert span.ended_at is not None def test_embed_span_roundtrips() -> None: - data = ai.experimental_telemetry.EmbedSpanData( + data = ai.telemetry.EmbedSpanData( model="m", provider="p", input_count=1, output_count=1, dimensions=8, ) - span = ai.experimental_telemetry.Span( + span = ai.telemetry.Span( name="embed", data=data, id="span-1", trace_id="trace-1" ) - restored = ai.experimental_telemetry.Span.model_validate( - span.model_dump(mode="json") - ) + restored = ai.telemetry.Span.model_validate(span.model_dump(mode="json")) - assert isinstance(restored.data, ai.experimental_telemetry.EmbedSpanData) + assert isinstance(restored.data, ai.telemetry.EmbedSpanData) assert restored.data.dimensions == 8 async def test_set_attrs(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("s") as span: + async with ai.telemetry.span("s") as span: span.set_attrs(a=1) span.set_attrs(b=2) (ended,) = recorder.ended - assert isinstance(ended.data, ai.experimental_telemetry.CustomSpanData) + assert isinstance(ended.data, ai.telemetry.CustomSpanData) assert ended.data.attrs == {"a": 1, "b": 2} async def test_set_attrs_rejects_framework_spans() -> None: - async with ai.experimental_telemetry.span( - ai.experimental_telemetry.LoopTurnSpanData() - ) as span: + async with ai.telemetry.span(ai.telemetry.LoopTurnSpanData()) as span: with pytest.raises(TypeError): span.set_attrs(a=1) async def test_replay_flag(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("s", replay=True): + async with ai.telemetry.span("s", replay=True): pass assert recorder.ended[0].replay async def test_not_set_as_current(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("outer") as outer: - async with ai.experimental_telemetry.span( + async with ai.telemetry.span("outer") as outer: + async with ai.telemetry.span( "overlapping", set_as_current=False ) as overlapping: # The span exists and parents normally... assert overlapping.parent_id == outer.id # ...but is not current: work opened meanwhile parents to # the outer span instead. - assert ai.experimental_telemetry.current_span() is outer - async with ai.experimental_telemetry.span("child") as child: + assert ai.telemetry.current_span() is outer + async with ai.telemetry.span("child") as child: assert child.parent_id == outer.id async def test_out_of_order_close_raises(recorder: Recorder) -> None: - cm_a = ai.experimental_telemetry.span("a") - cm_b = ai.experimental_telemetry.span("b") + cm_a = ai.telemetry.span("a") + cm_b = ai.telemetry.span("b") await cm_a.__aenter__() await cm_b.__aenter__() with pytest.raises(RuntimeError, match="closed out of order"): @@ -169,18 +164,16 @@ async def test_out_of_order_close_raises(recorder: Recorder) -> None: async def test_failing_adapter_is_isolated(recorder: Recorder) -> None: class Broken(_NoopAdapter): - async def on_span_start( - self, span: ai.experimental_telemetry.Span - ) -> None: + async def on_span_start(self, span: ai.telemetry.Span) -> None: raise RuntimeError("adapter bug") broken = Broken() - ai.experimental_telemetry.register(broken) + ai.telemetry.register(broken) try: - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass finally: - ai.experimental_telemetry.unregister(broken) + ai.telemetry.unregister(broken) # The broken adapter neither killed the span nor the other adapter. assert [s.name for s in recorder.ended] == ["s"] @@ -189,18 +182,16 @@ async def test_async_adapter_methods_awaited() -> None: ended: list[str] = [] class AsyncAdapter(_NoopAdapter): - async def on_span_end( - self, span: ai.experimental_telemetry.Span - ) -> None: + async def on_span_end(self, span: ai.telemetry.Span) -> None: ended.append(span.name) adapter = AsyncAdapter() - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) assert ended == ["s"] @@ -208,20 +199,16 @@ async def on_span_end( async def test_explicit_span_parent(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("elsewhere") as elsewhere: + async with ai.telemetry.span("elsewhere") as elsewhere: pass - async with ai.experimental_telemetry.span("ambient"): - async with ai.experimental_telemetry.span( - "child", parent=elsewhere - ) as child: + async with ai.telemetry.span("ambient"): + async with ai.telemetry.span("child", parent=elsewhere) as child: # The explicit parent wins over the ambient one... assert child.parent_id == elsewhere.id assert child.trace_id == elsewhere.trace_id # ...and only changes where the span hangs: it is still # current inside the block. - async with ai.experimental_telemetry.span( - "grandchild" - ) as grandchild: + async with ai.telemetry.span("grandchild") as grandchild: assert grandchild.parent_id == child.id @@ -229,35 +216,33 @@ async def test_restored_span_parent_continues_trace( recorder: Recorder, ) -> None: # "Process one": the span itself is the serializable position. - async with ai.experimental_telemetry.span("origin") as origin: + async with ai.telemetry.span("origin") as origin: payload = origin.model_dump(mode="json") # "Process two": restore and continue the same trace. - restored = ai.experimental_telemetry.Span.model_validate(payload) - async with ai.experimental_telemetry.span( - "pickup", parent=restored - ) as pickup: + restored = ai.telemetry.Span.model_validate(payload) + async with ai.telemetry.span("pickup", parent=restored) as pickup: assert pickup.trace_id == origin.trace_id assert pickup.parent_id == origin.id - async with ai.experimental_telemetry.span("nested") as nested: + async with ai.telemetry.span("nested") as nested: assert nested.trace_id == origin.trace_id assert nested.parent_id == pickup.id async def test_trace_attrs_propagate_to_children(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("root") as root: + async with ai.telemetry.span("root") as root: root.trace_attrs["session.id"] = "s-1" - async with ai.experimental_telemetry.span("child") as child: + async with ai.telemetry.span("child") as child: # A copy of the parent's attrs at creation time... assert child.trace_attrs == {"session.id": "s-1"} - async with ai.experimental_telemetry.span("grandchild") as gc: + async with ai.telemetry.span("grandchild") as gc: assert gc.trace_attrs == {"session.id": "s-1"} # ...independent of the parent's dict. child.trace_attrs["extra"] = True assert root.trace_attrs == {"session.id": "s-1"} # Later mutations reach spans created afterwards, not before. root.trace_attrs["user.id"] = "u-1" - async with ai.experimental_telemetry.span("sibling") as sibling: + async with ai.telemetry.span("sibling") as sibling: assert sibling.trace_attrs == { "session.id": "s-1", "user.id": "u-1", @@ -267,16 +252,14 @@ async def test_trace_attrs_propagate_to_children(recorder: Recorder) -> None: async def test_trace_attrs_survive_restore(recorder: Recorder) -> None: # "Process one": trace_attrs travel inside the serialized span. - async with ai.experimental_telemetry.span("origin") as origin: + async with ai.telemetry.span("origin") as origin: origin.trace_attrs["session.id"] = "s-1" payload = origin.model_dump(mode="json") # "Process two": children of the restored span inherit them. - restored = ai.experimental_telemetry.Span.model_validate(payload) + restored = ai.telemetry.Span.model_validate(payload) assert restored.trace_attrs == {"session.id": "s-1"} - async with ai.experimental_telemetry.span( - "pickup", parent=restored - ) as pickup: + async with ai.telemetry.span("pickup", parent=restored) as pickup: assert pickup.trace_attrs == {"session.id": "s-1"} @@ -284,30 +267,30 @@ async def test_use_span_parents_without_lifecycle( recorder: Recorder, ) -> None: # ``use_span`` is pure context plumbing: no pushes, no timestamps. - outer = ai.experimental_telemetry.create_span("outer") - async with ai.experimental_telemetry.use_span(outer): - assert ai.experimental_telemetry.current_span() is outer - async with ai.experimental_telemetry.span("child") as child: + outer = ai.telemetry.create_span("outer") + async with ai.telemetry.use_span(outer): + assert ai.telemetry.current_span() is outer + async with ai.telemetry.span("child") as child: assert child.parent_id == outer.id assert child.trace_id == outer.trace_id - assert ai.experimental_telemetry.current_span() is None + assert ai.telemetry.current_span() is None # Only the child was reported; outer was never pushed. assert [s.name for s in recorder.started] == ["child"] async def test_span_round_trips_typed_data() -> None: - data = ai.experimental_telemetry.ToolExecutionSpanData( + data = ai.telemetry.ToolExecutionSpanData( tool_name="lookup", tool_call_id="tc-1", args={"x": 1} ) - async with ai.experimental_telemetry.span(data) as sp: + async with ai.telemetry.span(data) as sp: sp.data.result = "ok" payload = sp.model_dump(mode="json") # A bare restore rebuilds the framework data type (matched by the # ``kind`` tag serialized in the data), so adapters dispatch on # re-pushed spans like live ones. - restored = ai.experimental_telemetry.Span.model_validate(payload) - assert restored.data == ai.experimental_telemetry.ToolExecutionSpanData( + restored = ai.telemetry.Span.model_validate(payload) + assert restored.data == ai.telemetry.ToolExecutionSpanData( tool_name="lookup", tool_call_id="tc-1", args={"x": 1}, @@ -320,26 +303,18 @@ async def test_span_round_trips_typed_data() -> None: async def test_restored_user_span_data() -> None: # A user span made with span("name") restores typed... - async with ai.experimental_telemetry.span("turn") as sp: + async with ai.telemetry.span("turn") as sp: sp.set_attrs(session="s1") - restored = ai.experimental_telemetry.Span.model_validate( - sp.model_dump(mode="json") - ) - assert restored.data == ai.experimental_telemetry.CustomSpanData( - attrs={"session": "s1"} - ) + restored = ai.telemetry.Span.model_validate(sp.model_dump(mode="json")) + assert restored.data == ai.telemetry.CustomSpanData(attrs={"session": "s1"}) restored.set_attrs(extra=1) # ...even when its name collides with a framework kind: the type # travels in the data, not the span name. - async with ai.experimental_telemetry.span("loop_turn") as sp2: + async with ai.telemetry.span("loop_turn") as sp2: sp2.set_attrs(foo=1) - collided = ai.experimental_telemetry.Span.model_validate( - sp2.model_dump(mode="json") - ) - assert collided.data == ai.experimental_telemetry.CustomSpanData( - attrs={"foo": 1} - ) + collided = ai.telemetry.Span.model_validate(sp2.model_dump(mode="json")) + assert collided.data == ai.telemetry.CustomSpanData(attrs={"foo": 1}) # ...while a user-defined span data type stays a dict on a bare # restore (its type isn't in the framework's tagged union); @@ -348,17 +323,13 @@ class RetrievalSpanData(pydantic.BaseModel): kind: Literal["retrieval"] = "retrieval" query: str - async with ai.experimental_telemetry.span( - RetrievalSpanData(query="q") - ) as sp3: + async with ai.telemetry.span(RetrievalSpanData(query="q")) as sp3: pass payload = sp3.model_dump(mode="json") - bare = ai.experimental_telemetry.Span.model_validate(payload) + bare = ai.telemetry.Span.model_validate(payload) assert isinstance(bare.data, dict) assert bare.data == {"kind": "retrieval", "query": "q"} - typed = ai.experimental_telemetry.Span[RetrievalSpanData].model_validate( - payload - ) + typed = ai.telemetry.Span[RetrievalSpanData].model_validate(payload) assert typed.data == RetrievalSpanData(query="q") # A plain dataclass with a ``kind`` ClassVar also works as span @@ -369,12 +340,10 @@ class LegacySpanData: kind: ClassVar[str] = "legacy" - async with ai.experimental_telemetry.span(LegacySpanData("q")) as sp4: + async with ai.telemetry.span(LegacySpanData("q")) as sp4: pass assert sp4.name == "legacy" - bare = ai.experimental_telemetry.Span.model_validate( - sp4.model_dump(mode="json") - ) + bare = ai.telemetry.Span.model_validate(sp4.model_dump(mode="json")) assert isinstance(bare.data, dict) assert bare.data == {"query": "q"} @@ -383,7 +352,7 @@ class LegacySpanData: async def test_create_span_reports_nothing(recorder: Recorder) -> None: - sp = ai.experimental_telemetry.create_span("quiet") + sp = ai.telemetry.create_span("quiet") assert sp.started_at is None assert sp.ended_at is None # Even a push reports nothing before the span started. @@ -395,14 +364,14 @@ async def test_create_span_reports_nothing(recorder: Recorder) -> None: async def test_push_lifecycle_split_across_pushes( recorder: Recorder, ) -> None: - sp = ai.experimental_telemetry.create_span("turn") + sp = ai.telemetry.create_span("turn") sp.set_attrs(session="s1") - sp.started_at = ai.experimental_telemetry.now_ns() + sp.started_at = ai.telemetry.now_ns() await sp.push() assert [s.name for s in recorder.started] == ["turn"] assert recorder.ended == [] - sp.ended_at = ai.experimental_telemetry.now_ns() + sp.ended_at = ai.telemetry.now_ns() await sp.push() (ended,) = recorder.ended assert ended.name == "turn" @@ -410,20 +379,18 @@ async def test_push_lifecycle_split_across_pushes( async def test_push_snapshots_are_frozen(recorder: Recorder) -> None: - sink = ai.experimental_telemetry.DictSink() - data = ai.experimental_telemetry.ToolExecutionSpanData( + sink = ai.telemetry.DictSink() + data = ai.telemetry.ToolExecutionSpanData( tool_name="t", tool_call_id="tc", args={"x": 1} ) - async with ai.experimental_telemetry.use_sink(sink): - sp = ai.experimental_telemetry.create_span(data) - sp.started_at = ai.experimental_telemetry.now_ns() + async with ai.telemetry.use_sink(sink): + sp = ai.telemetry.create_span(data) + sp.started_at = ai.telemetry.now_ns() await sp.push() # Mutations after a push don't leak into the snapshot. sp.data.args = {"x": 2} snapshot = sink.spans[sp.id] - assert isinstance( - snapshot.data, ai.experimental_telemetry.ToolExecutionSpanData - ) + assert isinstance(snapshot.data, ai.telemetry.ToolExecutionSpanData) assert snapshot.data.args == {"x": 1} @@ -431,41 +398,37 @@ async def test_finished_span_delivered_whole(recorder: Recorder) -> None: order: list[str] = [] class Adapter: - async def on_span_start( - self, span: ai.experimental_telemetry.Span - ) -> None: + async def on_span_start(self, span: ai.telemetry.Span) -> None: order.append(f"start {span.name}") async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: order.append(f"event {event.name}") - async def on_span_end( - self, span: ai.experimental_telemetry.Span - ) -> None: + async def on_span_end(self, span: ai.telemetry.Span) -> None: order.append(f"end {span.name} error={span.error is not None}") # A span that lived elsewhere arrives as one complete record... - sp = ai.experimental_telemetry.create_span("done-elsewhere") - sp.started_at = ai.experimental_telemetry.now_ns() + sp = ai.telemetry.create_span("done-elsewhere") + sp.started_at = ai.telemetry.now_ns() sp.events.append( - ai.experimental_telemetry.SpanEvent( + ai.telemetry.SpanEvent( name="milestone", - time_ns=ai.experimental_telemetry.now_ns(), + time_ns=ai.telemetry.now_ns(), attrs={}, ) ) - sp.ended_at = ai.experimental_telemetry.now_ns() - sp.error = ai.experimental_telemetry.SpanError(type="E", message="m") + sp.ended_at = ai.telemetry.now_ns() + sp.error = ai.telemetry.SpanError(type="E", message="m") payload = sp.model_dump(mode="json") adapter = Adapter() async with _registered(adapter): # ...and one push fires the full callback sequence. - await ai.experimental_telemetry.Span.model_validate(payload).push() + await ai.telemetry.Span.model_validate(payload).push() assert order == [ "start done-elsewhere", @@ -475,22 +438,18 @@ async def on_span_end( async def test_adapter_view_updated_in_place() -> None: - starts: list[ai.experimental_telemetry.Span] = [] - ends: list[ai.experimental_telemetry.Span] = [] + starts: list[ai.telemetry.Span] = [] + ends: list[ai.telemetry.Span] = [] class Holder(_NoopAdapter): - async def on_span_start( - self, span: ai.experimental_telemetry.Span - ) -> None: + async def on_span_start(self, span: ai.telemetry.Span) -> None: starts.append(span) - async def on_span_end( - self, span: ai.experimental_telemetry.Span - ) -> None: + async def on_span_end(self, span: ai.telemetry.Span) -> None: ends.append(span) async with _registered(Holder()): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: sp.set_attrs(a=1) # The adapter holds one object across callbacks; by span end it @@ -498,13 +457,13 @@ async def on_span_end( (start_view,) = starts (end_view,) = ends assert start_view is end_view - assert isinstance(end_view.data, ai.experimental_telemetry.CustomSpanData) + assert isinstance(end_view.data, ai.telemetry.CustomSpanData) assert end_view.data.attrs == {"a": 1} assert end_view.ended_at is not None async def test_repush_after_end_redelivers(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: pass assert [s.name for s in recorder.ended] == ["s"] # Re-pushing a completed span re-delivers it whole (the durable @@ -519,13 +478,13 @@ async def test_repush_after_end_redelivers(recorder: Recorder) -> None: async def test_use_sink_reroutes_pushes(recorder: Recorder) -> None: - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): - async with ai.experimental_telemetry.span("inside") as sp: + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): + async with ai.telemetry.span("inside") as sp: sp.events.append( - ai.experimental_telemetry.SpanEvent( + ai.telemetry.SpanEvent( name="milestone", - time_ns=ai.experimental_telemetry.now_ns(), + time_ns=ai.telemetry.now_ns(), attrs={}, ) ) @@ -538,25 +497,25 @@ async def test_use_sink_reroutes_pushes(recorder: Recorder) -> None: assert snapshot.ended_at is not None assert [e.name for e in snapshot.events] == ["milestone"] # Outside the context pushes reach adapters again. - async with ai.experimental_telemetry.span("outside"): + async with ai.telemetry.span("outside"): pass assert [s.name for s in recorder.ended] == ["outside"] async def test_use_sink_accepts_none(recorder: Recorder) -> None: - async with ai.experimental_telemetry.use_sink(None): - async with ai.experimental_telemetry.span("inside"): + async with ai.telemetry.use_sink(None): + async with ai.telemetry.span("inside"): pass assert [s.name for s in recorder.ended] == ["inside"] async def test_use_sink_none_preserves_outer_sink(recorder: Recorder) -> None: - sink = ai.experimental_telemetry.DictSink() + sink = ai.telemetry.DictSink() async with ( - ai.experimental_telemetry.use_sink(sink), - ai.experimental_telemetry.use_sink(None), + ai.telemetry.use_sink(sink), + ai.telemetry.use_sink(None), ): - async with ai.experimental_telemetry.span("inside"): + async with ai.telemetry.span("inside"): pass assert [s.name for s in sink.finished_spans] == ["inside"] assert recorder.ended == [] @@ -566,15 +525,15 @@ async def test_dict_sink_ships_to_adapters_exactly_as_pushed( recorder: Recorder, ) -> None: # The durable-body pattern: collect inside, re-push from a "step". - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): - async with ai.experimental_telemetry.span("outer"): - async with ai.experimental_telemetry.span("inner"): + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): + async with ai.telemetry.span("outer"): + async with ai.telemetry.span("inner"): pass payload = [s.model_dump(mode="json") for s in sink.spans.values()] for item in payload: - await ai.experimental_telemetry.Span.model_validate(item).push() + await ai.telemetry.Span.model_validate(item).push() assert {s.name for s in recorder.ended} == {"outer", "inner"} inner = next(s for s in recorder.ended if s.name == "inner") @@ -585,11 +544,11 @@ async def test_dict_sink_ships_to_adapters_exactly_as_pushed( async def test_push_never_raises(recorder: Recorder) -> None: class BrokenSink: - async def on_push(self, span: ai.experimental_telemetry.Span) -> None: + async def on_push(self, span: ai.telemetry.Span) -> None: raise RuntimeError("sink bug") - async with ai.experimental_telemetry.use_sink(BrokenSink()): - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.use_sink(BrokenSink()): + async with ai.telemetry.span("s"): pass # both pushes hit the broken sink; neither raises @@ -599,8 +558,8 @@ async def on_push(self, span: ai.experimental_telemetry.Span) -> None: async def test_ids_deterministic_under_use_random() -> None: async def run() -> tuple[str, str, str]: async with ai.messages.use_random(random.Random(7)): - async with ai.experimental_telemetry.span("outer") as outer: - async with ai.experimental_telemetry.span("inner") as inner: + async with ai.telemetry.span("outer") as outer: + async with ai.telemetry.span("inner") as inner: pass return outer.trace_id, outer.id, inner.id @@ -613,13 +572,13 @@ async def run() -> tuple[str, str, str]: async def test_adapter_class_driven() -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter class Vendor: def __init__(self) -> None: self.log: list[str] = [] # vendor-specific state async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: if span.name == "boring": return # opt out before the first yield @@ -630,10 +589,10 @@ async def __call__( vendor = Vendor() async with _registered(vendor): - async with ai.experimental_telemetry.span("boring"): + async with ai.telemetry.span("boring"): pass - async with ai.experimental_telemetry.span("outer") as outer: - async with ai.experimental_telemetry.span("inner"): + async with ai.telemetry.span("outer") as outer: + async with ai.telemetry.span("inner"): pass await _add_event(outer, "milestone") @@ -649,12 +608,12 @@ async def __call__( async def test_adapter_class_keeps_identity() -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter class Vendor: """Vendor bridge.""" async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass @@ -662,32 +621,32 @@ async def __call__( assert Vendor.__name__ == "Vendor" assert Vendor.__qualname__.endswith("Vendor") assert Vendor.__doc__ == "Vendor bridge." - assert isinstance(Vendor(), ai.experimental_telemetry.AdapterMixin) + assert isinstance(Vendor(), ai.telemetry.AdapterMixin) async def test_adapter_decorated_class_supports_subclassing() -> None: log: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter class Base: - def label(self, span: ai.experimental_telemetry.Span) -> str: + def label(self, span: ai.telemetry.Span) -> str: return span.name async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass log.append(f"end {self.label(span)}") class Enriched(Base): - def label(self, span: ai.experimental_telemetry.Span) -> str: + def label(self, span: ai.telemetry.Span) -> str: return f"enriched:{span.name}" # A subclass of a decorated class inherits the driver; no second # decoration needed. async with _registered(Enriched()): - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass assert log == ["end enriched:s"] @@ -698,45 +657,45 @@ class NoCall: pass class PlainCall: - async def __call__(self, span: ai.experimental_telemetry.Span) -> None: + async def __call__(self, span: ai.telemetry.Span) -> None: pass bad_classes: list[Any] = [NoCall, PlainCall] for bad in bad_classes: with pytest.raises(TypeError, match="async generator"): - ai.experimental_telemetry.adapter(bad) + ai.telemetry.adapter(bad) async def test_adapter_rejects_double_decoration() -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter class Vendor: async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass with pytest.raises(TypeError, match="already an adapter"): - ai.experimental_telemetry.adapter(Vendor) + ai.telemetry.adapter(Vendor) async def test_register_rejects_undecorated_class() -> None: class Vendor: async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass with pytest.raises(TypeError, match="@adapter"): - ai.experimental_telemetry.register(Vendor()) + ai.telemetry.register(Vendor()) async def test_adapter_defaults_are_noops(recorder: Recorder) -> None: # A bare AdapterMixin (its defaults) is valid: no per-span # frames. - async with _registered(ai.experimental_telemetry.AdapterMixin()): - async with ai.experimental_telemetry.span("s") as sp: + async with _registered(ai.telemetry.AdapterMixin()): + async with ai.telemetry.span("s") as sp: await _add_event(sp, "e") assert [s.name for s in recorder.ended] == ["s"] @@ -744,18 +703,16 @@ async def test_adapter_defaults_are_noops(recorder: Recorder) -> None: async def test_hook_override_composes_with_super() -> None: log: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter class Both: async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass log.append(f"frame end {span.name}") - async def on_span_end( - self, span: ai.experimental_telemetry.Span, / - ) -> None: + async def on_span_end(self, span: ai.telemetry.Span, /) -> None: log.append(f"hook end {span.name}") # Overriding a hook replaces the driver for that phase; # super() plugs it back in (the mixin sits after this @@ -765,21 +722,21 @@ async def on_span_end( ) async with _registered(Both()): - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass assert log == ["hook end s", "frame end s"] async def test_decorated_class_state_cannot_collide_with_driver() -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter class Clashy: def __init__(self) -> None: self._live = "mine" # same name the driver mangles away self.ended: list[str] = [] async def __call__( - self, span: ai.experimental_telemetry.Span + self, span: ai.telemetry.Span ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass @@ -787,21 +744,21 @@ async def __call__( clashy = Clashy() async with _registered(clashy): - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass assert clashy.ended == ["s"] assert clashy._live == "mine" async def test_adapter_function_returns_adapter() -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def vendor( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass - assert isinstance(vendor, ai.experimental_telemetry.AdapterMixin) + assert isinstance(vendor, ai.telemetry.AdapterMixin) assert "adapter" in repr(vendor) @@ -811,21 +768,21 @@ async def vendor( async def test_adapter_locals_across_yield() -> None: events: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: name = span.name # local state, no bookkeeping dict events.append(f"start {name}") while (yield) is not None: pass assert span.ended_at is not None - assert isinstance(span.data, ai.experimental_telemetry.CustomSpanData) + assert isinstance(span.data, ai.telemetry.CustomSpanData) events.append(f"end {name} a={span.data.attrs.get('a')}") async with _registered(adapter): - async with ai.experimental_telemetry.span("outer") as outer: - async with ai.experimental_telemetry.span("inner") as inner: + async with ai.telemetry.span("outer") as outer: + async with ai.telemetry.span("inner") as inner: inner.set_attrs(a=1) outer.set_attrs(a=2) @@ -840,11 +797,11 @@ async def adapter( async def test_adapter_reads_error_after_loop() -> None: - seen: list[ai.experimental_telemetry.SpanError | None] = [] + seen: list[ai.telemetry.SpanError | None] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass @@ -856,18 +813,16 @@ async def adapter( error = ValueError("boom") async with _registered(adapter): with pytest.raises(ValueError) as excinfo: - async with ai.experimental_telemetry.span("failing"): + async with ai.telemetry.span("failing"): raise error # The app still gets the original exception... assert excinfo.value is error # ...and the bridge saw its serializable record. assert seen == [ - ai.experimental_telemetry.SpanError( - type="ValueError", message="boom" - ) + ai.telemetry.SpanError(type="ValueError", message="boom") ] - async with ai.experimental_telemetry.span("fine"): + async with ai.telemetry.span("fine"): pass assert seen[-1] is None @@ -875,9 +830,9 @@ async def adapter( async def test_adapter_opt_out_before_yield(recorder: Recorder) -> None: ended: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: if span.name == "boring": return @@ -886,9 +841,9 @@ async def adapter( ended.append(span.name) async with _registered(adapter): - async with ai.experimental_telemetry.span("boring"): + async with ai.telemetry.span("boring"): pass - async with ai.experimental_telemetry.span("interesting"): + async with ai.telemetry.span("interesting"): pass assert ended == ["interesting"] @@ -896,25 +851,25 @@ async def adapter( async def test_adapter_failures_isolated(recorder: Recorder) -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def broken_before( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: raise RuntimeError("pre-yield bug") while (yield) is not None: pass - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def broken_after( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass raise RuntimeError("post-loop bug") - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def yields_after_end( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: while (yield) is not None: pass @@ -923,7 +878,7 @@ async def yields_after_end( async with _registered(broken_before): async with _registered(broken_after): async with _registered(yields_after_end): - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass # None of the broken generators killed the span or other adapters. @@ -931,31 +886,31 @@ async def yields_after_end( async def test_adapter_rejects_plain_functions() -> None: - async def not_a_generator(span: ai.experimental_telemetry.Span) -> None: + async def not_a_generator(span: ai.telemetry.Span) -> None: pass fn: Any = not_a_generator with pytest.raises(TypeError, match="async generator function"): - ai.experimental_telemetry.adapter(fn) + ai.telemetry.adapter(fn) async def test_adapter_events_live_loop() -> None: seen: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: vendor = f"vendor:{span.name}" # frame-local state while (ev := (yield)) is not None: # Delivered live, while the span is still open. assert span.ended_at is None seen.append(f"{vendor} event {ev.name}") - assert isinstance(span.data, ai.experimental_telemetry.CustomSpanData) + assert isinstance(span.data, ai.telemetry.CustomSpanData) seen.append(f"{vendor} end a={span.data.attrs.get('a')}") async with _registered(adapter): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: await _add_event(sp, "one") await _add_event(sp, "two") sp.set_attrs(a=1) @@ -970,9 +925,9 @@ async def adapter( async def test_adapter_error_reaches_loop() -> None: seen: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: while (ev := (yield)) is not None: seen.append(ev.name) @@ -981,7 +936,7 @@ async def adapter( async with _registered(adapter): with pytest.raises(ValueError, match="boom"): - async with ai.experimental_telemetry.span("failing") as sp: + async with ai.telemetry.span("failing") as sp: await _add_event(sp, "one") raise ValueError("boom") @@ -995,9 +950,9 @@ async def test_adapter_early_finish_opts_out( seen: list[str] = [] ended: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: ev = yield # handle one event, then finish assert ev is not None @@ -1005,7 +960,7 @@ async def adapter( ended.append(span.name) async with _registered(adapter): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: await _add_event(sp, "one") await _add_event(sp, "two") @@ -1019,15 +974,15 @@ async def adapter( async def test_adapter_raising_event_handler_isolated( recorder: Recorder, ) -> None: - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: while (yield) is not None: raise RuntimeError("event bug") async with _registered(adapter): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: await _add_event(sp, "one") await _add_event(sp, "two") # generator already dead: skipped @@ -1037,9 +992,9 @@ async def adapter( async def test_adapter_drain_loop() -> None: order: list[str] = [] - @ai.experimental_telemetry.adapter + @ai.telemetry.adapter async def adapter( - span: ai.experimental_telemetry.Span, + span: ai.telemetry.Span, ) -> AsyncGenerator[None, Any]: order.append("start") while (yield) is not None: @@ -1048,7 +1003,7 @@ async def adapter( order.append(f"end events={[e.name for e in span.events]}") async with _registered(adapter): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: await _add_event(sp, "one") order.append("event added") @@ -1059,7 +1014,7 @@ async def adapter( async def test_events_appended_and_stamped(recorder: Recorder) -> None: - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: first = await _add_event(sp, "first", a=1) second = await _add_event(sp, "second") @@ -1082,19 +1037,19 @@ async def test_unpushed_events_delivered_by_end_push( class EventAdapter(_NoopAdapter): async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: seen.append(event.name) async with _registered(EventAdapter()): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: # Appended but never pushed: the context manager's end # push carries it — nothing is lost, just delivered late. sp.events.append( - ai.experimental_telemetry.SpanEvent( + ai.telemetry.SpanEvent( name="quiet", - time_ns=ai.experimental_telemetry.now_ns(), + time_ns=ai.telemetry.now_ns(), attrs={}, ) ) @@ -1112,21 +1067,21 @@ async def test_span_event_dispatched_live_to_multiple_adapters( class FirstAdapter(_NoopAdapter): async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: seen.append(("first", event.name, span.ended_at is None)) class SecondAdapter(_NoopAdapter): async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: seen.append(("second", event.name, span.ended_at is None)) async with _registered(FirstAdapter()), _registered(SecondAdapter()): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: await _add_event(sp, "milestone") # Both handlers saw the event while the span was still live; the @@ -1146,22 +1101,22 @@ async def test_span_event_raising_handler_isolated( class Broken(_NoopAdapter): async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: raise RuntimeError("adapter bug") class Fine(_NoopAdapter): async def on_span_event( self, - span: ai.experimental_telemetry.Span, - event: ai.experimental_telemetry.SpanEvent, + span: ai.telemetry.Span, + event: ai.telemetry.SpanEvent, ) -> None: seen.append(event.name) # Broken registers first: it must not stop dispatch to Fine. async with _registered(Broken()), _registered(Fine()): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: event = await _add_event(sp, "milestone") assert seen == ["milestone"] @@ -1173,13 +1128,13 @@ async def on_span_event( async def test_span_without_adapters_is_noop() -> None: - async with ai.experimental_telemetry.span("outer") as sp: + async with ai.telemetry.span("outer") as sp: assert sp.id == "" assert sp.trace_id == "" assert sp.parent_id is None assert sp.started_at is None # Never current: nothing to parent under. - assert ai.experimental_telemetry.current_span() is None + assert ai.telemetry.current_span() is None # Attribute writes still work; they are just never observed. sp.set_attrs(b=2) event = sp.add_event("milestone", c=3) @@ -1192,7 +1147,7 @@ async def test_span_without_adapters_is_noop() -> None: async def test_noop_span_reraises_without_recording() -> None: with pytest.raises(ValueError, match="boom"): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: raise ValueError("boom") assert sp.error is None @@ -1206,12 +1161,10 @@ def forbidden() -> int: monkeypatch.setattr(time, "time_ns", forbidden) # The durable-execution contract: unused telemetry makes no # non-deterministic calls, so no use_time setup is needed. - async with ai.experimental_telemetry.span("outer") as sp: + async with ai.telemetry.span("outer") as sp: sp.add_event("milestone") await sp.push() - async with ai.experimental_telemetry.span( - ai.experimental_telemetry.LoopTurnSpanData() - ): + async with ai.telemetry.span(ai.telemetry.LoopTurnSpanData()): pass @@ -1219,7 +1172,7 @@ async def test_no_rng_draws_without_adapters() -> None: async def id_after_spans(spans: int) -> str: async with ai.messages.use_random(random.Random(7)): for _ in range(spans): - async with ai.experimental_telemetry.span("s"): + async with ai.telemetry.span("s"): pass return ai.messages.generate_id() @@ -1229,9 +1182,9 @@ async def id_after_spans(spans: int) -> str: async def test_routed_sink_keeps_spans_live_without_adapters() -> None: - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): - async with ai.experimental_telemetry.span("s") as sp: + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): + async with ai.telemetry.span("s") as sp: pass assert sp.id assert [s.name for s in sink.finished_spans] == ["s"] @@ -1252,7 +1205,7 @@ def time_ns() -> int: async def _stamps() -> tuple[int | None, int, int | None]: - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: event = await _add_event(sp, "milestone") return sp.started_at, event.time_ns, sp.ended_at @@ -1260,15 +1213,15 @@ async def _stamps() -> tuple[int | None, int, int | None]: async def test_use_time_overrides_and_restores(recorder: Recorder) -> None: # A ticking clock gives a deterministic timestamp sequence; the # override drives started_at, event stamps, and ended_at alike. - async with ai.experimental_telemetry.use_time(_ticking_clock(1_000)): + async with ai.telemetry.use_time(_ticking_clock(1_000)): first = await _stamps() - async with ai.experimental_telemetry.use_time(_ticking_clock(1_000)): + async with ai.telemetry.use_time(_ticking_clock(1_000)): second = await _stamps() assert first == second == (1_010, 1_020, 1_030) # Restored on exit -- back to the wall clock. - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.span("s") as sp: pass assert sp.started_at is not None assert sp.started_at > 1_030 @@ -1279,7 +1232,7 @@ async def test_use_time_decorator_handles_async_functions( ) -> None: # Works as a decorator on an async fn; the clock is shared across # calls, so the second call continues where the first left off. - @ai.experimental_telemetry.use_time(_ticking_clock(1_000)) + @ai.telemetry.use_time(_ticking_clock(1_000)) async def run() -> tuple[int | None, int, int | None]: return await _stamps() @@ -1289,8 +1242,8 @@ async def run() -> tuple[int | None, int, int | None]: async def test_use_time_accepts_time_time_ns(recorder: Recorder) -> None: before = time.time_ns() - async with ai.experimental_telemetry.use_time(time.time_ns): - async with ai.experimental_telemetry.span("s") as sp: + async with ai.telemetry.use_time(time.time_ns): + async with ai.telemetry.span("s") as sp: pass assert sp.started_at is not None assert sp.ended_at is not None @@ -1303,42 +1256,40 @@ async def test_use_time_accepts_time_time_ns(recorder: Recorder) -> None: async def test_use_span_accepts_none(recorder: Recorder) -> None: # "no span" is a normal state in gated instrumentation; None means # no reparenting, no separate code path at the callsite. - async with ai.experimental_telemetry.use_span(None): - assert ai.experimental_telemetry.current_span() is None - async with ai.experimental_telemetry.span("child") as child: + async with ai.telemetry.use_span(None): + assert ai.telemetry.current_span() is None + async with ai.telemetry.span("child") as child: pass assert child.parent_id is None async def test_is_enabled_reflects_sinks_and_adapters() -> None: # No adapters registered here (no recorder fixture), default sink. - assert not ai.experimental_telemetry.is_enabled() - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): - assert ai.experimental_telemetry.is_enabled() - assert not ai.experimental_telemetry.is_enabled() + assert not ai.telemetry.is_enabled() + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): + assert ai.telemetry.is_enabled() + assert not ai.telemetry.is_enabled() async def test_is_enabled_with_adapter_registered(recorder: Recorder) -> None: - assert ai.experimental_telemetry.is_enabled() + assert ai.telemetry.is_enabled() async def test_stamp_start_and_end(recorder: Recorder) -> None: clock = _ticking_clock(100) - async with ai.experimental_telemetry.use_time(clock): - sp = ai.experimental_telemetry.create_span("turn").stamp_start() + async with ai.telemetry.use_time(clock): + sp = ai.telemetry.create_span("turn").stamp_start() assert sp.started_at == 110 assert sp.stamp_end(error=ValueError("boom")) is sp assert sp.ended_at == 120 - assert sp.error == ai.experimental_telemetry.SpanError( - type="ValueError", message="boom" - ) + assert sp.error == ai.telemetry.SpanError(type="ValueError", message="boom") # Stamps only write fields; nothing was pushed. assert recorder.started == [] # A SpanError passes through as-is; error=None keeps an existing one. - err = ai.experimental_telemetry.SpanError(type="TurnError", message="m") - sp2 = ai.experimental_telemetry.create_span("t").stamp_start() + err = ai.telemetry.SpanError(type="TurnError", message="m") + sp2 = ai.telemetry.create_span("t").stamp_start() sp2.stamp_end(error=err) assert sp2.error is err sp2.stamp_end() @@ -1346,9 +1297,9 @@ async def test_stamp_start_and_end(recorder: Recorder) -> None: async def test_add_event_appends_without_pushing() -> None: - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): - async with ai.experimental_telemetry.span("s") as sp: + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): + async with ai.telemetry.span("s") as sp: event = sp.add_event("cache_hit", {"cache.key": "k"}, size=3) assert event.attrs == {"cache.key": "k", "size": 3} assert sp.events == [event] @@ -1362,23 +1313,23 @@ async def test_add_event_appends_without_pushing() -> None: async def test_dict_sink_finished_spans_and_push_all( recorder: Recorder, ) -> None: - sink = ai.experimental_telemetry.DictSink() - async with ai.experimental_telemetry.use_sink(sink): - async with ai.experimental_telemetry.span("done"): + sink = ai.telemetry.DictSink() + async with ai.telemetry.use_sink(sink): + async with ai.telemetry.span("done"): pass - dangling = ai.experimental_telemetry.create_span("open").stamp_start() + dangling = ai.telemetry.create_span("open").stamp_start() await dangling.push() # Only complete records are safe to ship. assert [s.name for s in sink.finished_spans] == ["done"] # The ship step: re-deliver dumped payloads where the adapters are. payload = [s.model_dump(mode="json") for s in sink.finished_spans] - await ai.experimental_telemetry.push_all(payload) + await ai.telemetry.push_all(payload) assert [s.name for s in recorder.started] == ["done"] assert [s.name for s in recorder.ended] == ["done"] # Live spans are accepted alongside dumped ones. - await ai.experimental_telemetry.push_all([dangling.stamp_end()]) + await ai.telemetry.push_all([dangling.stamp_end()]) assert [s.name for s in recorder.ended] == ["done", "open"] @@ -1387,7 +1338,7 @@ async def test_attribute_mappings_for_dotted_names( ) -> None: # Viewer attribute names ("session.id") aren't valid keywords, so # set_attrs() merges a positional mapping with keywords. - async with ai.experimental_telemetry.span("generate_title") as sp: + async with ai.telemetry.span("generate_title") as sp: sp.set_attrs({"session.id": "s1"}, model="haiku") sp.set_attrs({"output.value": "t"}, plain=1) assert sp.data.attrs == { diff --git a/tests/experimental_telemetry/utils/__init__.py b/tests/telemetry/utils/__init__.py similarity index 100% rename from tests/experimental_telemetry/utils/__init__.py rename to tests/telemetry/utils/__init__.py diff --git a/tests/experimental_telemetry/utils/test_console.py b/tests/telemetry/utils/test_console.py similarity index 67% rename from tests/experimental_telemetry/utils/test_console.py rename to tests/telemetry/utils/test_console.py index bad9f6f9..d6c10c37 100644 --- a/tests/experimental_telemetry/utils/test_console.py +++ b/tests/telemetry/utils/test_console.py @@ -5,20 +5,20 @@ import io import ai -from ai.experimental_telemetry.utils import console +from ai.telemetry.utils import console from ai.types import usage async def test_console_prints_tree() -> None: out = io.StringIO() adapter = console.ConsoleAdapter(out=out) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: - async with ai.experimental_telemetry.span("outer"): - async with ai.experimental_telemetry.span("inner") as sp: + async with ai.telemetry.span("outer"): + async with ai.telemetry.span("inner") as sp: sp.set_attrs(k=1) finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) text = out.getvalue() assert "▸ outer" in text @@ -32,19 +32,19 @@ async def test_console_prints_tree() -> None: async def test_console_prints_live_span_events() -> None: out = io.StringIO() adapter = console.ConsoleAdapter(out=out) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: - async with ai.experimental_telemetry.span("outer") as sp: + async with ai.telemetry.span("outer") as sp: sp.events.append( - ai.experimental_telemetry.SpanEvent( + ai.telemetry.SpanEvent( name="first_token", - time_ns=ai.experimental_telemetry.now_ns(), + time_ns=ai.telemetry.now_ns(), attrs={"event_type": "TextStart"}, ) ) await sp.push() finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) text = out.getvalue() assert "· first_token +" in text @@ -54,16 +54,16 @@ async def test_console_prints_live_span_events() -> None: async def test_console_labels_ops_span() -> None: out = io.StringIO() adapter = console.ConsoleAdapter(out=out) - data = ai.experimental_telemetry.EmbedSpanData( + data = ai.telemetry.EmbedSpanData( model="text-embedding-3-small", input_count=1, ) - ai.experimental_telemetry.register(adapter) + ai.telemetry.register(adapter) try: - async with ai.experimental_telemetry.span(data) as sp: + async with ai.telemetry.span(data) as sp: sp.data.usage = usage.Usage(input_tokens=9) finally: - ai.experimental_telemetry.unregister(adapter) + ai.telemetry.unregister(adapter) text = out.getvalue() assert "▸ embed text-embedding-3-small" in text @@ -73,17 +73,17 @@ async def test_console_labels_ops_span() -> None: async def test_console_tree_uses_response_complete_duration() -> None: out = io.StringIO() adapter = console.ConsoleAdapter(out=out) - span = ai.experimental_telemetry.Span( + span = ai.telemetry.Span( name="ai_stream", - data=ai.experimental_telemetry.AiStreamSpanData(model="m", messages=[]), + data=ai.telemetry.AiStreamSpanData(model="m", messages=[]), id="span-1", trace_id="trace-1", parent_id=None, started_at=0, ended_at=5_000_000_000, events=[ - ai.experimental_telemetry.SpanEvent( - name=ai.experimental_telemetry.RESPONSE_COMPLETE, + ai.telemetry.SpanEvent( + name=ai.telemetry.RESPONSE_COMPLETE, time_ns=1_500_000_000, attrs={}, )