diff --git a/python/frameworks/ag2/CHANGELOG.md b/python/frameworks/ag2/CHANGELOG.md new file mode 100644 index 000000000..229942050 --- /dev/null +++ b/python/frameworks/ag2/CHANGELOG.md @@ -0,0 +1,106 @@ +# Changelog + +## [0.1.0] - unreleased + +- Privacy follows the traceAI `TraceConfig` convention (re-verification 2026-10-16): + content is captured by default and hidden with `FI_HIDE_*` / `TraceConfig`; + `capture_content` now defaults to `True` (AG2's own default) instead of a + second, package-specific switch. In both modes local paths, stack traces + and raw exception text are no longer exported: tracebacks in tool messages + are cut, absolute paths become `[path]`, an ERROR status carries only the + exception class name, and exception events keep `exception.type`. + +- New package for AG2 1.x (PyPI `ag2`, import `ag2`). `setup()` attaches AG2's + own `TelemetryMiddleware` and registers the + Future AGI exporter; `AG2SpanProcessor` sets `gen_ai.span.kind` from + `gen_ai.operation.name` and aliases AG2's cache/thinking usage keys + (originals kept). Does not wrap `Agent` and never imports `autogen`. + Refs: TH-8236. +- Cache tokens are aliased to `gen_ai.usage.cache_read_tokens` / + `gen_ai.usage.cache_write_tokens`, the keys Future AGI reads (were the dotted + `gen_ai.usage.cache_read.input_tokens` / `cache_creation.input_tokens`, which + it does not read). A `record_usage` span that repeats a chat span's usage + moves them to `ag2.usage.*` with the token counts. +- With `capture_content=True`, the captured chat messages, tool arguments and + result, and human input are also copied to `input.value` / `output.value`, + the only keys Future AGI fills a span's input and output from. Content a + `TraceConfig` flag hides is not copied. +- Token totals: Future AGI sums the promoted `gen_ai.usage.input_tokens` / + `output_tokens` / `total_tokens` over every span of a trace. The processor + moves those keys to `ag2.usage.*` (values kept) on `record_usage model_call` + spans, which repeat the chat span's tokens, and on `record_usage subtask` + rollups whose sub-agent (`ag2.usage.label`) is itself instrumented in the + same trace. `aggregation` and `compaction` usage keeps its promoted tokens: + AG2 calls the model outside the middleware for those, so the usage span is + the only copy. The trace total now equals AG2's `UsageReport` total, except + for Anthropic with prompt caching (next entry). +- Anthropic input tokens include cached tokens. AG2's Anthropic mapper reports + `prompt_tokens` without cache reads and writes (its OpenAI mapper includes + them), so with `AnthropicConfig`'s default prompt caching a chat span showed + 4 input tokens for a 9,750-token prompt. On `anthropic` spans whose tokens + count, `cache_read_input_tokens` + `cache_creation_input_tokens` are added + to `gen_ai.usage.input_tokens` (and `total_tokens` recomputed when present). + The trace total then exceeds AG2's `UsageReport` by the cached tokens, which + that report leaves out for Anthropic; a duplicate `record_usage` span keeps + AG2's numbers under `ag2.usage.*`. +- With `capture_content=True`, a single chat completion goes on `output.value` + as its message object (Observe's chat view shows it as the assistant reply; + it shows nothing for a one-item list), and `invoke_agent` spans get the first + chat input and last chat output of their turn, so the trace list shows the + question and the answer. Content a `TraceConfig` flag hides is not copied. +- Requires `fi-instrumentation-otel>=1.1.0` (was `>=1.0.0`): published 1.0.0 + crashes in `register()` with `opentelemetry-exporter-otlp-proto-http` 1.45.0 + (`AttributeError: 'HTTPSpanExporter' object has no attribute '_headers'`). +- A later `setup(..., config=...)` on a provider that already has an + `AG2SpanProcessor` now replaces that processor's `TraceConfig` (most recent + call wins, warning logged when it changes) instead of being ignored; + `config=None` keeps the installed config. +- traceAI context helpers (`using_session`, `using_user`, `using_metadata`, + `using_attributes`, ...) now reach AG2 spans: `AG2SpanProcessor.on_start` + copies `get_attributes_from_context()` without overriding keys AG2 sets, + except `session.id`, where the context wins over `span_attributes`. +- `TraceConfig` text flags now act on AG2 content. AG2 records messages as one + JSON string per span, which `TraceConfig.mask`'s `hide_input_text` / + `hide_output_text` rules never matched, so those flags used to leave content + in place. They now drop the whole input-side / output-side content + attributes (messages, system instructions, tool arguments/results, + human-input text), as `hide_inputs` / `hide_outputs` do; + `hide_input_messages` also drops `gen_ai.system_instructions`. The README + lists which flags apply; `hide_input_images` and `hide_embedding_vectors` + have nothing to act on. +- A later `trace_provider.add_span_processor(...)` no longer removes + `AG2SpanProcessor`. fi's provider shuts down and clears every processor on + the first such call after `register()`; `install_span_processor` now wraps + the provider instance's `add_span_processor` so the AG2 processor sits out + that reset and is put back first. Installing on a provider that uses + `ConcurrentMultiSpanProcessor` logs a warning (normalization before export + is not guaranteed there). +- Normalized spans keep the SDK's `BoundedAttributes` instead of a plain + dict, so `dropped_attributes_count` is no longer reset to 0 and the span's + attribute limits still apply (a key the processor adds to a full span + evicts the oldest one and is counted). +- `setup()` without `tracer_provider` now registers a provider once and + reuses it on later calls without one (until it is shut down) instead of + calling `register()` again. Before, `setup(planner)` then `setup(worker)` + gave each agent its own provider and `AG2SpanProcessor`, so the planner's + processor never saw the worker's `invoke_agent` span and the worker's + `record_usage subtask` rollup was counted on top of the worker's chat spans. + A different `project_name` on a later call is ignored with a warning. +- traceAI context keys (`session.id`, `user.id`, `metadata`, ...) are no + longer the first keys evicted when the processor's added keys overflow + `max_span_attributes`. `on_start` writes them before AG2 sets anything, so + they were the oldest keys on the span; the processor now moves them behind + AG2's keys before re-bounding, and AG2's oldest keys are evicted instead. +- Disclosed in the README (Limits): PRD J4 is not met. A middleware that + runs outside `TelemetryMiddleware` and short-circuits leaves no `chat` span + (no span at all when it answers `on_turn`); AG2's own halt does the same + with `Agent(assembly=...)` policies, and with per-call `Plugin` policies + gives a `chat` span with status `UNSET` and no reason attribute. No capture + is added; tests pin these span shapes. +- README accuracy: the tested-version line now names only what the test + suite ran (ag2 1.0.0, 1.0.3 and 1.1.2 on Python 3.11; ag2 1.1.2 on Python + 3.10, 3.12 and 3.13); `hide_llm_invocation_parameters` is listed as having + nothing to act on (AG2 sets no `gen_ai.request.parameters`, now checked by + a drift guard); Limits also cover the by-name sub-task match, the + `create_telemetry_middleware()` `agent_name` / `tracer_provider` defaults + and the concurrent first `add_span_processor` race. diff --git a/python/frameworks/ag2/README.md b/python/frameworks/ag2/README.md new file mode 100644 index 000000000..de149df7d --- /dev/null +++ b/python/frameworks/ag2/README.md @@ -0,0 +1,274 @@ +# traceAI-ag2 + +Send AG2 1.x traces to Future AGI through AG2's own `TelemetryMiddleware`. +Alpha, fixture-validated. No live vendor run is claimed. + +AG2 1.x is the PyPI package `ag2`, imported as `ag2`. It is not Microsoft AutoGen +(`autogen-agentchat`) and it is not AG2 Classic (`autogen` 0.14.x). + +| You installed | Import | Package | +|---|---|---| +| `ag2` 1.x | `ag2` | `traceAI-ag2` (this package) | +| `autogen` 0.14.x | `autogen` | AG2 Classic integration (TH-8237) | +| `autogen-agentchat` | `autogen_agentchat` | `traceAI-autogen` | + +Requires `ag2>=1.0.0` (the first stable 1.x that ships `TelemetryMiddleware`), +`fi-instrumentation-otel>=1.1.0` and Python 3.10 to 3.13. The offline test +suite runs against ag2 1.0.0, 1.0.3 and 1.1.2 on Python 3.11, and against +ag2 1.1.2 on Python 3.10, 3.12 and 3.13. `ag2` is +Apache-2.0 and is a dependency only; nothing from it is vendored. + +## What it does + +AG2 1.x already emits OpenTelemetry GenAI spans from `TelemetryMiddleware`. +This package does not wrap `Agent`. It: + +1. registers the Future AGI exporter through `fi_instrumentation.register()` + (or uses the provider you pass), +2. attaches `TelemetryMiddleware` to your agents through AG2's public + `Agent.add_middleware` (content is captured by default, like every traceAI + instrumentor; hide it with `FI_HIDE_*` / `TraceConfig`), +3. installs `AG2SpanProcessor`, which, on spans from AG2's + `opentelemetry.instrumentation.ag2` scope only: + - sets `gen_ai.span.kind` from `gen_ai.operation.name` + (`chat` -> `LLM`, `execute_tool` -> `TOOL`, `invoke_agent` -> `AGENT`), + - copies AG2's cache and thinking usage keys onto the keys Future AGI reads + and keeps the originals: + + | AG2 key | Alias | + |---|---| + | `gen_ai.usage.cache_creation_input_tokens` | `gen_ai.usage.cache_write_tokens` | + | `gen_ai.usage.cache_read_input_tokens` | `gen_ai.usage.cache_read_tokens` | + | `gen_ai.usage.thinking_tokens` | `gen_ai.usage.reasoning.output_tokens` | + + Future AGI reads the cache counts from `gen_ai.usage.cache_read_tokens` / + `gen_ai.usage.cache_write_tokens`, not from the dotted OpenTelemetry + spellings (`gen_ai.usage.cache_read.input_tokens`). On a `record_usage` + span that repeats a chat span's usage (see below), they move to + `ag2.usage.*` with the token counts. + + - applies `TraceConfig` (`FI_HIDE_INPUTS`, `FI_HIDE_OUTPUTS`, ...), and + removes local paths, stack traces and raw error text (see Privacy). + +## Set up + +```bash +pip install traceAI-ag2 "ag2[openai]" # the openai extra is only for the OpenAIConfig example below +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export FI_PROJECT_NAME="my-chatbot" +``` + +AG2 attaches middleware per agent, so pass your agents to `setup`: + +```python +from ag2 import Agent +from ag2.config import OpenAIConfig +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_ag2 import setup + +trace_provider = register(project_type=ProjectType.OBSERVE, project_name="my-chatbot") + +agent = Agent("assistant", "You are helpful.", config=OpenAIConfig(model="gpt-4o-mini")) +setup(agent, tracer_provider=trace_provider) # content captured; FI_HIDE_* hides it + +reply = await agent.ask("Hello") # inside an async function +trace_provider.force_flush() # short scripts: flush before exit +``` + +This snippet is illustrative and was not run against a live model; the +offline example below is what the tests execute. + +`setup(agent)` without `tracer_provider` calls +`register(project_type=ProjectType.OBSERVE, project_name=project_name)` itself +and returns the provider. Later `setup()` calls without `tracer_provider` +reuse that provider until it is shut down, so `setup(planner)` followed by +`setup(worker)` gives both agents one provider and one `AG2SpanProcessor`. A +different `project_name` on a later call is ignored with a warning; pass +`tracer_provider=` to send agents to another project. + +Agents created later, or per-call middleware: + +```python +from traceai_ag2 import create_telemetry_middleware + +mw = create_telemetry_middleware(tracer_provider=trace_provider, agent_name="worker") +worker = Agent("worker", config=cfg, middleware=[mw]) +# or: await agent.ask("...", middleware=[mw]) +``` + +Call `setup(tracer_provider=trace_provider)` once in that case too, so the span +processor is installed. `setup` is idempotent: an agent that already has a +`TelemetryMiddleware` is skipped. One `AG2SpanProcessor` serves the provider: +a later `setup(..., config=TraceConfig(...))` replaces its `TraceConfig` (the +most recent call wins and a warning is logged when it changes), while +`config=None` keeps the current one. + +Span processors you add after `setup()` with +`trace_provider.add_span_processor(...)` are fine: `setup` wraps that method on +the provider instance so `AG2SpanProcessor` stays first and is not shut down +when fi's provider replaces its default exporter. + +Sessions and users come from traceAI's context helpers, as with the other +traceAI integrations: + +```python +from fi_instrumentation import using_attributes, using_session + +with using_session("chat-123"): + reply = await agent.ask("Hello") # every AG2 span gets session.id="chat-123" + +with using_attributes(session_id="chat-123", user_id="user-7", metadata={"tier": "pro"}): + reply = await agent.ask("Hello") +``` + +`provider_name` is never invented. Pass it if you want it on every span; +otherwise AG2 fills it from the model response when the client reports one. + +An offline example with a scripted model (no model API key) is in +`examples/offline_weather_agent.py`. + +## Privacy + +This package follows the traceAI `TraceConfig` convention: prompts, replies, +tool arguments and tool results are captured by default, and hidden with the +standard variables (or `TraceConfig(...)`): + +```bash +export FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true # or FI_HIDE_INPUT_MESSAGES, FI_HIDE_OUTPUT_MESSAGES, FI_HIDE_INPUT_TEXT, ... +``` + +There is no second switch. `capture_content` is AG2's own `TelemetryMiddleware` +argument and defaults to `True`, as upstream; `setup(..., capture_content=False)` +stops AG2 itself from recording content, but the supported way to hide it is +`FI_HIDE_*`. + +Never exported, in either mode: local file paths, user names, stack traces and +raw exception text. Tracebacks that AG2 puts into a tool message (a failing +tool's error is sent back to the model and recorded in the next chat span's +input) are cut to `[stack trace removed]`, absolute local paths become +`[path]`, a failing span's status description is the exception class name +(for example `RuntimeError`) instead of `str(exc)`, and exception events keep +`exception.type` only. + +`TraceConfig` (or the matching `FI_HIDE_*` variables) removes it in the span +processor. AG2 records message content as one JSON +string per span (`gen_ai.input.messages`, `gen_ai.output.messages`), not as +the per-message keys that `TraceConfig.mask`'s text and image rules match, so +the processor drops whole attributes: + +| `TraceConfig` flag | Removed from AG2 spans | +|---|---| +| `hide_inputs`, `hide_input_text` | `gen_ai.input.messages`, `gen_ai.system_instructions`, `gen_ai.tool.call.arguments`, `ag2.human_input.prompt` | +| `hide_outputs`, `hide_output_text` | `gen_ai.output.messages`, `gen_ai.tool.call.result`, `ag2.human_input.response` | +| `hide_input_messages` | `gen_ai.input.messages`, `gen_ai.system_instructions` | +| `hide_output_messages` | `gen_ai.output.messages` | +| `hide_llm_invocation_parameters` | Nothing: AG2 sets no invocation-parameter attribute (`gen_ai.request.parameters`) | +| `hide_input_images`, `hide_embedding_vectors` | Nothing: AG2 records only text parts (binary inputs are omitted) and no embeddings | + +The text flags cannot redact just the text inside the JSON, so they remove the +whole message attribute, which is the privacy-safe choice. A removed key is +not copied to `input.value` / `output.value` either. + +## Conformance + +| Field | What you get | When it is omitted | +|---|---|---| +| Span kind | `AGENT`, `LLM`, `TOOL` from `gen_ai.operation.name` | `await_human_input` and `record_usage` spans get no kind (see below) | +| Model | `gen_ai.request.model`, `gen_ai.response.model` pass through | When AG2 did not set them | +| Provider | `gen_ai.provider.name` from `provider_name` or the model response | When neither is known | +| Tokens | `gen_ai.usage.input_tokens` / `output_tokens` pass through, except on `anthropic` spans, where cache read and write tokens are added to `input_tokens` (and `total_tokens` recomputed when present; see the note below). Cache tokens are aliased to `gen_ai.usage.cache_read_tokens` / `cache_write_tokens` and thinking tokens to `gen_ai.usage.reasoning.output_tokens`, originals kept | When the usage field is zero/absent. Cost is not emitted | +| Session | Not emitted by AG2. `session.id` from traceAI's `using_session` / `using_attributes` context, which wins over a static value; or set it yourself: `setup(..., span_attributes={"session.id": "..."})` | When neither is set | +| User, metadata, tags | `user.id`, `metadata`, `tag.tags` (and the other traceAI context keys) from `using_user` / `using_metadata` / `using_tags` / `using_attributes`; a key AG2 or `span_attributes` already set is not overridden | When the context does not set them | +| Tools | `gen_ai.tool.name`, `gen_ai.tool.call.id`, `gen_ai.tool.type`; arguments/result unless hidden by `FI_HIDE_*` | | +| Input / output | `input.value` / `output.value` (what Observe shows as a span's input and output), copied from the captured chat messages, tool arguments and result, or human input. A single chat completion goes on `output.value` as its message object, which Observe's chat view shows as the assistant reply. `invoke_agent` gets the first chat input and the last chat output of its turn, so the trace list shows the question and the answer | When a `TraceConfig` flag hides the source, and always on `record_usage` spans (AG2 records no content there) | +| Errors | OTel status `ERROR` on failing tool / model / agent spans | | + +Notes: + +- AG2 also emits `await_human_input {agent}` spans. That operation is not in + the Microsoft Agent Framework kind table, so no kind is guessed for it. +- From ag2 1.0.3, AG2 also emits a `record_usage {kind}` span per usage event + (`ag2.usage.kind` = `model_call`, `aggregation`, `compaction` or `subtask`). + It has no operation name and gets no span kind; the aliases are applied. + Future AGI totals tokens by summing the promoted + `gen_ai.usage.input_tokens` / `output_tokens` / `total_tokens` over every + span in a trace, so the processor moves those three keys to + `ag2.usage.input_tokens` / `output_tokens` / `total_tokens` (values kept on + the span) exactly where a chat span already counts the same spend: + + | `ag2.usage.kind` | Same tokens already on a chat span? | Promoted tokens | + |---|---|---| + | `model_call` | Yes: the chat span of that LLM call | Moved to `ag2.usage.*` | + | `aggregation` | No: memory aggregation calls the model client directly, outside the middleware | Kept | + | `compaction` | No: history compaction calls the model client directly | Kept | + | `subtask` | Only if the sub-agent named in `ag2.usage.label` is itself instrumented and its `invoke_agent` span ended earlier in the same trace | Moved when it is, kept otherwise | + + With that, the trace total equals AG2's own `UsageReport` total; the tests + check this with real AG2 for every kind above. Exception: with an + Anthropic model and prompt caching (`AnthropicConfig` caches by default), + AG2's Anthropic mapper reports `prompt_tokens` without the cached tokens, + while its OpenAI mapper and the GenAI conventions count them as input. The + processor adds `cache_read_input_tokens` and `cache_creation_input_tokens` + to `gen_ai.usage.input_tokens` on `anthropic` spans whose tokens count, so + the trace total is the real prompt size and exceeds `UsageReport` by the + cached tokens. A `record_usage` span that repeats a chat span keeps AG2's + own numbers under `ag2.usage.*`. For the `subtask` match, the + sub-agent's `TelemetryMiddleware` must use the agent's own name (`setup` + does; `create_telemetry_middleware(agent_name=...)` must pass `agent.name`). + On ag2 1.0.0 to 1.0.2 there are no `record_usage` spans, so aggregation, + compaction and uninstrumented sub-agent spend does not reach the trace. +- Network trace propagation (`ag2.otel.traceparent`) is upstream. Apart from + that usage move, the processor never removes attributes it did not hide by + request, except under a span attribute limit: the processor keeps the + span's `BoundedAttributes` (limits and `dropped_attributes_count` carry + over), so on a span already at `max_span_attributes` (128 by default) a key + it adds evicts the oldest one and the drop is counted, as the SDK does. + traceAI context keys (`session.id`, `user.id`, `metadata`, ...) are moved + behind AG2's keys first, so AG2's oldest keys are evicted before them. + +## Limits + +- PRD J4 is not met: a middleware short-circuit shows neither `ERROR` nor a + typed reason, and this package adds no capture for it. `setup()` adds + `TelemetryMiddleware` as the innermost middleware, so a middleware + registered before it (for example in `Agent(middleware=[...])`) that + answers `on_llm_call` without calling `call_next` leaves no `chat` span for + that call (the `invoke_agent` span ends with status `UNSET`), and one that + answers `on_turn` leaves no span at all. AG2's own halt (the `HaltEvent` + that `AlertPolicy` sends on a FATAL alert) does the same when the policies + come from `Agent(assembly=[...])`. With per-call `Plugin` policies + (`agent.ask(..., plugins=[...])`, ag2 1.1.2) the halted call gets a `chat` + span with status `UNSET` and no attribute carrying the halt reason. +- The `record_usage subtask` dedupe matches the sub-agent by name within a + trace, so if an instrumented agent and an uninstrumented sub-agent with the + same name both run in one trace, the uninstrumented one's rollup loses its + promoted tokens and the trace undercounts; give such agents distinct names. +- `create_telemetry_middleware()` defaults `agent_name=None`, which AG2 + records as `unknown`, so a sub-agent instrumented that way never matches + its rollup's `ag2.usage.label` and its spend is counted twice (pass + `agent_name=agent.name`), and `tracer_provider=None` sends spans to the + OpenTelemetry global provider, which `register()` does not set by default + (pass the provider from `register()` or `setup()`). +- Two threads making the first `add_span_processor` call on a fresh + `register()` provider at the same time can race with fi's unlocked + exporter reset and drop `AG2SpanProcessor`; add span processors from one + thread, or before calling `setup()`. + +## Troubleshooting + +- No traces: `register()` needs `project_name` or `FI_PROJECT_NAME`; the + collector drops a batch whose resource has no `project_name`. +- Duplicate spans: do not attach a second `TelemetryMiddleware`, and do not + also enable `traceai-autogen` in an `ag2` process. +- Export failures are logged and never fail the agent. +- Spans arrive without `gen_ai.span.kind`: the provider runs its processors + concurrently (`TracerProvider(active_span_processor=ConcurrentMultiSpanProcessor())`; + `setup` logs a warning), so the exporter can read a span before + `AG2SpanProcessor` normalizes it. Use the default synchronous processor. +- Content is missing: a `FI_HIDE_*` variable or `TraceConfig` hide flag is set. +- Content you wanted hidden still appears: you constructed `TelemetryMiddleware` + yourself on a provider without `AG2SpanProcessor`; use `setup()`. +- This package never imports `autogen`. If your code imports `autogen`, you + are on AG2 Classic or Microsoft AutoGen; use that integration instead. diff --git a/python/frameworks/ag2/examples/offline_weather_agent.py b/python/frameworks/ag2/examples/offline_weather_agent.py new file mode 100644 index 000000000..985f05529 --- /dev/null +++ b/python/frameworks/ag2/examples/offline_weather_agent.py @@ -0,0 +1,60 @@ +"""Offline AG2 1.x example: a scripted model, traces exported to Future AGI. + +Runs without any model API key: ``ag2.testing.TestConfig`` plays the model. +Swap ``config=`` for a real ``ag2.config`` model config (for example +``OpenAIConfig(model=...)``) in your application. + + export FI_API_KEY=... FI_SECRET_KEY=... FI_PROJECT_NAME=ag2-demo + python examples/offline_weather_agent.py +""" + +import asyncio +import os + +from ag2 import Agent +from ag2.events import ModelMessage, ModelResponse, ToolCallEvent +from ag2.testing import TestConfig +from ag2.usage import Usage +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +from traceai_ag2 import setup + + +def get_weather(city: str) -> str: + """Return the weather for a city.""" + return f"sunny in {city}" + + +async def main() -> None: + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name=os.getenv("FI_PROJECT_NAME", "ag2-offline-example"), + ) + + agent = Agent( + "weather_bot", + "You answer weather questions.", + config=TestConfig( + ToolCallEvent("get_weather", arguments='{"city": "Paris"}', id="call_weather_1"), + ModelResponse( + ModelMessage("It is sunny in Paris."), + usage=Usage(prompt_tokens=11, completion_tokens=7, cache_read_input_tokens=3), + model="fake-model", + finish_reason="stop", + ), + ), + tools=[get_weather], + ) + + # Attaches AG2's TelemetryMiddleware with capture_content=False. + setup(agent, tracer_provider=trace_provider) + + reply = await agent.ask("What's the weather in Paris?") + print(reply.body) + + trace_provider.force_flush() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/frameworks/ag2/pyproject.toml b/python/frameworks/ag2/pyproject.toml new file mode 100644 index 000000000..e1046036c --- /dev/null +++ b/python/frameworks/ag2/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name = "traceAI-ag2" +version = "0.1.0" +description = "Future AGI tracing for AG2 1.x (PyPI ag2) through its native TelemetryMiddleware" +authors = ["Future AGI "] +readme = "README.md" +license = "Apache-2.0" +packages = [ + { include = "traceai_ag2" } +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +# Peer dependency: never vendored. TelemetryMiddleware (capture_content, +# agent_name, provider_name, model_name, span_attributes) and +# Agent.add_middleware ship from ag2 1.0.0 (first stable 1.x). +ag2 = ">=1.0.0" +# GEN_AI_SPAN_KIND / GEN_AI_OPERATION_NAME first appear in fi_types at 1.0.0, +# but published 1.0.0 crashes in register() with +# opentelemetry-exporter-otlp-proto-http 1.45.0 (otel.py:289 AttributeError: +# 'HTTPSpanExporter' object has no attribute '_headers'). 1.1.0 is the first +# release this package's suite passes on. +fi-instrumentation-otel = ">=1.1.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=8.0.0" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/ag2/tests/__init__.py b/python/frameworks/ag2/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/ag2/tests/_support.py b/python/frameworks/ag2/tests/_support.py new file mode 100644 index 000000000..272d3ec2e --- /dev/null +++ b/python/frameworks/ag2/tests/_support.py @@ -0,0 +1,188 @@ +"""Shared fixtures for traceai-ag2 tests: a scripted AG2 model and span helpers. + +The model is AG2's own ``ag2.testing.TestConfig``: no network, no API key. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, Dict, List, Optional + +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +USER_PROMPT = "What's the weather in Paris?" +FINAL_ANSWER = "It is sunny in Paris." +TOOL_RESULT = "sunny in Paris" +TOOL_CALL_ID = "call_weather_1" +MODEL = "fake-model" +PROVIDER = "fake-provider" + +# Every attribute AG2 sets only when capture_content=True, plus any key the +# processor would have to drop for TraceConfig. +CONTENT_KEYS = ( + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + "ag2.human_input.prompt", + "ag2.human_input.response", +) +CONTENT_STRINGS = (USER_PROMPT, FINAL_ANSWER, TOOL_RESULT, '"city"') + + +def emits_record_usage() -> bool: + """``record_usage {kind}`` spans exist from ag2 1.0.3 (telemetry.py:420 at 1.1.2).""" + from ag2.middleware.builtin import telemetry + + return hasattr(telemetry._TelemetryMiddlewareInstance, "record_usage_span") + + +def expected_one_tool_kinds() -> Dict[str, Optional[str]]: + """Span name -> expected ``gen_ai.span.kind`` for the weather run.""" + kinds: Dict[str, Optional[str]] = { + "invoke_agent weather_bot": "AGENT", + "chat": "LLM", + "execute_tool get_weather": "TOOL", + f"chat {MODEL}": "LLM", + } + if emits_record_usage(): + kinds["record_usage model_call"] = None + return kinds + + +def get_weather(city: str) -> str: + """Return the weather for a city.""" + return f"sunny in {city}" + + +def weather_config(*, with_provider: bool = True) -> Any: + """One tool call, then a final answer carrying every usage field AG2 reads.""" + from ag2.events import ModelMessage, ModelResponse, ToolCallEvent + from ag2.testing import TestConfig + from ag2.usage import Usage + + final_kwargs: Dict[str, Any] = { + "usage": Usage( + prompt_tokens=11, + completion_tokens=7, + total_tokens=18, + cache_read_input_tokens=3, + cache_creation_input_tokens=2, + thinking_tokens=5, + ), + "model": MODEL, + "finish_reason": "stop", + } + if with_provider: + final_kwargs["provider"] = PROVIDER + return TestConfig( + ToolCallEvent("get_weather", arguments='{"city": "Paris"}', id=TOOL_CALL_ID), + ModelResponse(ModelMessage(FINAL_ANSWER), **final_kwargs), + ) + + +def weather_agent(name: str = "weather_bot", *, with_provider: bool = True) -> Any: + from ag2 import Agent + + return Agent( + name, + "You answer weather questions.", + config=weather_config(with_provider=with_provider), + tools=[get_weather], + ) + + +def ask(agent: Any, prompt: str = USER_PROMPT, **kwargs: Any) -> Any: + async def _run() -> Any: + return await agent.ask(prompt, **kwargs) + + return asyncio.run(_run()) + + +def memory_provider() -> "tuple[TracerProvider, InMemorySpanExporter]": + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def fake_register(monkeypatch: Any) -> "tuple[InMemorySpanExporter, List[Optional[str]]]": + """Patch ``fi_instrumentation.register`` to build a new in-memory provider per call. + + Every provider exports to one shared exporter, so spans split across + providers still land in one place. ``calls`` records the ``project_name`` + of each call. Any provider an earlier test's ``setup()`` registered is + forgotten for this test. + """ + import fi_instrumentation + from opentelemetry.sdk.resources import Resource + + import traceai_ag2._setup as setup_module + + exporter = InMemorySpanExporter() + calls: List[Optional[str]] = [] + + def register(*, project_type: Any = None, project_name: Optional[str] = None, **_: Any) -> TracerProvider: + calls.append(project_name) + provider = TracerProvider(resource=Resource.create({"project_name": project_name or ""}), shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider + + monkeypatch.setattr(fi_instrumentation, "register", register) + monkeypatch.setattr(setup_module, "_registered_provider", None, raising=False) + return exporter, calls + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) + + +def by_name(spans: List[ReadableSpan]) -> Dict[str, ReadableSpan]: + out: Dict[str, ReadableSpan] = {} + for span in spans: + assert span.name not in out, f"duplicate span name {span.name!r}" + out[span.name] = span + return out + + +def find(spans: List[ReadableSpan], prefix: str) -> List[ReadableSpan]: + return [s for s in spans if s.name.startswith(prefix)] + + +def otlp_value(value: Dict[str, Any]) -> Any: + """Decode one OTLP JSON ``AnyValue`` (as MessageToDict renders it).""" + if "stringValue" in value: + return value["stringValue"] + if "intValue" in value: + return int(value["intValue"]) + if "boolValue" in value: + return bool(value["boolValue"]) + if "doubleValue" in value: + return float(value["doubleValue"]) + if "arrayValue" in value: + return [otlp_value(v) for v in value["arrayValue"].get("values", [])] + return value + + +def otlp_attrs(span: Dict[str, Any]) -> Dict[str, Any]: + return {a["key"]: otlp_value(a.get("value", {})) for a in span.get("attributes", [])} + + +def all_string_values(attributes: Dict[str, Any]) -> List[str]: + out: List[str] = [] + for value in attributes.values(): + if isinstance(value, str): + out.append(value) + elif isinstance(value, (list, tuple)): + out.extend(str(v) for v in value) + return out + + +def assert_no_content(attributes: Dict[str, Any], where: Optional[str] = None) -> None: + for key in CONTENT_KEYS: + assert key not in attributes, f"{key} present on {where}" + for text in all_string_values(attributes): + for needle in CONTENT_STRINGS: + assert needle not in text, f"content {needle!r} leaked on {where}: {text!r}" diff --git a/python/frameworks/ag2/tests/test_contract_harness.py b/python/frameworks/ag2/tests/test_contract_harness.py new file mode 100644 index 000000000..a370059fb --- /dev/null +++ b/python/frameworks/ag2/tests/test_contract_harness.py @@ -0,0 +1,166 @@ +"""Shared-harness contract tests (TH-8339 Receiver). + +Real path: AG2 ``Agent`` + AG2 ``TelemetryMiddleware`` (attached by +``traceai_ag2.setup``) -> ``AG2SpanProcessor`` -> ``fi_instrumentation`` +``BatchSpanProcessor`` -> real OTLP/HTTP protobuf exporter -> loopback +``harness.Receiver``. The model is AG2's scripted ``TestConfig``; nothing +leaves 127.0.0.1 and no vendor API is called. + +The Receiver drops resource attributes and headers; those are asserted in +``test_resource_headers.py``. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +_PYTHON_DIR = Path(__file__).resolve().parents[3] +_PACKAGE_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_PYTHON_DIR / "tests")) + +from harness import Receiver, run # noqa: E402 + +from ._support import ( # noqa: E402 + MODEL, + PROVIDER, + TOOL_CALL_ID, + TOOL_RESULT, + USER_PROMPT, + ask, + assert_no_content, + expected_one_tool_kinds, + otlp_attrs, + weather_agent, +) + +_STATUS_ERROR = {"STATUS_CODE_ERROR", 2} + + +@pytest.fixture() +def fi_env(monkeypatch): + def _apply(origin: str) -> None: + monkeypatch.setenv("FI_BASE_URL", origin) + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + monkeypatch.delenv(name, raising=False) + + return _apply + + +def test_one_tool_run_reaches_collector_path_with_normalized_gen_ai_spans(fi_env): + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + from traceai_ag2 import setup + + with Receiver() as receiver: + fi_env(receiver.origin) + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="ag2-contract", + verbose=False, + ) + try: + agent = weather_agent() + setup(agent, tracer_provider=trace_provider) # content captured by default (TraceConfig) + reply = ask(agent) + assert reply.body + assert trace_provider.force_flush(timeout_millis=10_000) + spans = receiver.spans() + finally: + trace_provider.shutdown() + + by_name = {s["name"]: otlp_attrs(s) for s in spans} + expected_kinds = expected_one_tool_kinds() + assert sorted(by_name) == sorted(expected_kinds), sorted(by_name) + + # span kind, from gen_ai.operation.name + assert {n: a.get("gen_ai.span.kind") for n, a in by_name.items()} == expected_kinds + assert by_name["invoke_agent weather_bot"]["gen_ai.operation.name"] == "invoke_agent" + assert by_name["invoke_agent weather_bot"]["gen_ai.agent.name"] == "weather_bot" + assert by_name["execute_tool get_weather"]["gen_ai.tool.name"] == "get_weather" + assert by_name["execute_tool get_weather"]["gen_ai.tool.call.id"] == TOOL_CALL_ID + + # model, provider, tokens (originals and dotted aliases) + chat = by_name[f"chat {MODEL}"] + assert chat["gen_ai.request.model"] == MODEL + assert chat["gen_ai.response.model"] == MODEL + assert chat["gen_ai.provider.name"] == PROVIDER + assert chat["gen_ai.response.finish_reasons"] == ["stop"] + expected_usage = { + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.cache_creation_input_tokens": 2, + "gen_ai.usage.cache_write_tokens": 2, + "gen_ai.usage.cache_read_input_tokens": 3, + "gen_ai.usage.cache_read_tokens": 3, + "gen_ai.usage.thinking_tokens": 5, + "gen_ai.usage.reasoning.output_tokens": 5, + } + usage_spans = [chat] + for source in usage_spans: + assert {k: source.get(k) for k in expected_usage} == expected_usage + if "record_usage model_call" in by_name: # ag2 >= 1.0.3 + # The duplicate of the chat span's tokens is kept under ag2.usage.* + # so the trace total (sum of promoted gen_ai.usage.* columns) counts once. + record = by_name["record_usage model_call"] + assert "gen_ai.usage.input_tokens" not in record + assert "gen_ai.usage.output_tokens" not in record + assert record["ag2.usage.input_tokens"] == 11 + assert record["ag2.usage.output_tokens"] == 7 + promoted_input = sum( + a.get("gen_ai.usage.input_tokens", 0) for a in by_name.values() + ) + assert promoted_input == 11, "trace-level input tokens must count the model call once" + + # session: AG2 emits none; content: captured by default (TraceConfig) + for name, attributes in by_name.items(): + assert "session.id" not in attributes, name + assert USER_PROMPT in chat["gen_ai.input.messages"] + assert by_name["execute_tool get_weather"]["gen_ai.tool.call.result"] == TOOL_RESULT + assert by_name["execute_tool get_weather"]["output.value"] == TOOL_RESULT + + for span in spans: + assert span.get("status", {}).get("code") not in _STATUS_ERROR, span["name"] + + +def test_short_script_flushes_spans_before_exit(): + """The offline example, run as a separate process via harness.run.""" + with Receiver() as receiver: + env = dict(os.environ) + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": "placeholder-api-key", + "FI_SECRET_KEY": "placeholder-secret-key", + "FI_PROJECT_NAME": "ag2-contract-script", + "FI_HIDE_INPUTS": "true", + "FI_HIDE_OUTPUTS": "true", + "PYTHONPATH": os.pathsep.join( + [str(_PACKAGE_DIR), str(_PYTHON_DIR), env.get("PYTHONPATH", "")] + ), + } + ) + result = run( + [sys.executable, str(_PACKAGE_DIR / "examples" / "offline_weather_agent.py")], + env=env, + stdin=None, + timeout=120, + ) + spans = receiver.spans() + + assert not result.timed_out, result.stderr.decode(errors="replace") + assert result.returncode == 0, result.stderr.decode(errors="replace") + assert b"It is sunny in Paris." in result.stdout + by_name = {s["name"]: otlp_attrs(s) for s in spans} + assert by_name["invoke_agent weather_bot"]["gen_ai.span.kind"] == "AGENT" + assert by_name["execute_tool get_weather"]["gen_ai.span.kind"] == "TOOL" + assert by_name["chat fake-model"]["gen_ai.span.kind"] == "LLM" + assert by_name["chat fake-model"]["gen_ai.usage.cache_read_tokens"] == 3 + for name, attributes in by_name.items(): + assert_no_content(attributes, name) diff --git a/python/frameworks/ag2/tests/test_import_isolation.py b/python/frameworks/ag2/tests/test_import_isolation.py new file mode 100644 index 000000000..338cb6e1b --- /dev/null +++ b/python/frameworks/ag2/tests/test_import_isolation.py @@ -0,0 +1,59 @@ +"""AC-08: traceai-ag2 must never import ``autogen`` (AG2 Classic / AutoGen).""" + +from __future__ import annotations + +import ast +import os +import subprocess +import sys +from pathlib import Path + +_PACKAGE_DIR = Path(__file__).resolve().parents[1] +_PYTHON_DIR = Path(__file__).resolve().parents[3] + +_FORBIDDEN = ("autogen", "autogen_agentchat", "autogen_core", "autogen_ext") + + +def test_import_and_setup_do_not_import_autogen(): + code = ( + "import sys\n" + "import traceai_ag2\n" + "from opentelemetry.sdk.trace import TracerProvider\n" + "from ag2 import Agent\n" + "from ag2.testing import TestConfig\n" + "traceai_ag2.setup(Agent('a', config=TestConfig('hi')), tracer_provider=TracerProvider())\n" + f"bad = sorted(m for m in sys.modules if m.split('.')[0] in {_FORBIDDEN!r})\n" + "print('FORBIDDEN', bad)\n" + "sys.exit(1 if bad else 0)\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(_PACKAGE_DIR), str(_PYTHON_DIR), env.get("PYTHONPATH", "")]) + result = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, timeout=120) + assert result.returncode == 0, (result.stdout + result.stderr).decode(errors="replace") + assert b"FORBIDDEN []" in result.stdout + + +def test_package_source_has_no_autogen_import(): + for path in sorted((_PACKAGE_DIR / "traceai_ag2").rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + roots = [alias.name.split(".")[0] for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + roots = [node.module.split(".")[0]] + else: + continue + for root in roots: + assert root not in _FORBIDDEN, f"{path}:{node.lineno} imports {root}" + + +def test_dependencies_exclude_autogen(): + text = (_PACKAGE_DIR / "pyproject.toml").read_text(encoding="utf-8") + deps = text.split("[tool.poetry.dependencies]", 1)[1].split("\n[", 1)[0] + declared = { + line.split("=", 1)[0].strip().lower() + for line in deps.splitlines() + if "=" in line and not line.lstrip().startswith("#") + } + assert "ag2" in declared + assert not declared & {"autogen", "autogen-agentchat", "pyautogen", "autogen-core"} diff --git a/python/frameworks/ag2/tests/test_processor.py b/python/frameworks/ag2/tests/test_processor.py new file mode 100644 index 000000000..4aa4dc22a --- /dev/null +++ b/python/frameworks/ag2/tests/test_processor.py @@ -0,0 +1,861 @@ +"""Unit tests for AG2SpanProcessor on synthetic spans (AC-02, AC-03).""" + +from __future__ import annotations + +import logging + +import pytest +from fi_instrumentation.instrumentation.config import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_ag2 import ( + AG2_INSTRUMENTATION_SCOPE, + OPERATION_TO_SPAN_KIND, + USAGE_KEY_ALIASES, + AG2SpanProcessor, + install_span_processor, + normalize_attributes, +) + +from ._support import attrs, memory_provider + + +def _emit(provider: TracerProvider, attributes: dict, scope: str = AG2_INSTRUMENTATION_SCOPE, name: str = "s"): + tracer = provider.get_tracer(scope) + with tracer.start_as_current_span(name, attributes=attributes): + pass + + +@pytest.fixture() +def pipeline(): + provider, exporter = memory_provider() + assert install_span_processor(provider, config=TraceConfig()) is True + return provider, exporter + + +# --- AC-02: kind map ------------------------------------------------------- + + +@pytest.mark.parametrize( + "operation, kind", + [("chat", "LLM"), ("execute_tool", "TOOL"), ("invoke_agent", "AGENT")], +) +def test_operation_maps_to_span_kind(pipeline, operation, kind): + provider, exporter = pipeline + _emit(provider, {"gen_ai.operation.name": operation}) + (span,) = exporter.get_finished_spans() + assert attrs(span)["gen_ai.span.kind"] == kind + + +@pytest.mark.parametrize( + "attributes", + [ + {"gen_ai.operation.name": "await_human_input"}, # read, but not in the MAF table + {"ag2.span.type": "usage", "ag2.usage.kind": "model_call"}, # record_usage: no operation + {"gen_ai.operation.name": "embeddings"}, # MAF string AG2 does not emit + {}, + ], +) +def test_unmapped_operations_get_no_kind(pipeline, attributes): + provider, exporter = pipeline + _emit(provider, attributes) + (span,) = exporter.get_finished_spans() + assert "gen_ai.span.kind" not in attrs(span) + + +def test_kind_table_is_only_the_strings_read_from_ag2(): + assert dict(OPERATION_TO_SPAN_KIND) == {"chat": "LLM", "execute_tool": "TOOL", "invoke_agent": "AGENT"} + + +def test_existing_span_kind_is_not_overwritten(pipeline): + provider, exporter = pipeline + _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.span.kind": "CHAIN"}) + (span,) = exporter.get_finished_spans() + assert attrs(span)["gen_ai.span.kind"] == "CHAIN" + + +def test_foreign_scope_is_untouched(pipeline): + provider, exporter = pipeline + original = { + "gen_ai.operation.name": "chat", + "gen_ai.usage.cache_read_input_tokens": 4, + } + _emit(provider, original, scope="some.other.instrumentation") + (span,) = exporter.get_finished_spans() + assert attrs(span) == original + + +# --- AC-03: usage aliases -------------------------------------------------- + + +def test_usage_aliases_copy_and_keep_originals(pipeline): + provider, exporter = pipeline + source = { + "gen_ai.operation.name": "chat", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.cache_creation_input_tokens": 2, + "gen_ai.usage.cache_read_input_tokens": 3, + "gen_ai.usage.thinking_tokens": 5, + } + _emit(provider, source) + (span,) = exporter.get_finished_spans() + got = attrs(span) + for key, value in source.items(): + assert got[key] == value, key # originals kept, input/output unchanged + assert got["gen_ai.usage.cache_write_tokens"] == 2 + assert got["gen_ai.usage.cache_read_tokens"] == 3 + assert got["gen_ai.usage.reasoning.output_tokens"] == 5 + assert set(got) == set(source) | { + "gen_ai.span.kind", + "gen_ai.usage.cache_write_tokens", + "gen_ai.usage.cache_read_tokens", + "gen_ai.usage.reasoning.output_tokens", + } + + +def test_alias_table_matches_spec(): + assert dict(USAGE_KEY_ALIASES) == { + "gen_ai.usage.cache_creation_input_tokens": "gen_ai.usage.cache_write_tokens", + "gen_ai.usage.cache_read_input_tokens": "gen_ai.usage.cache_read_tokens", + "gen_ai.usage.thinking_tokens": "gen_ai.usage.reasoning.output_tokens", + } + + +def test_alias_absent_when_source_absent(pipeline): + provider, exporter = pipeline + _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.usage.input_tokens": 1}) + (span,) = exporter.get_finished_spans() + got = attrs(span) + for target in USAGE_KEY_ALIASES.values(): + assert target not in got + + +def test_alias_does_not_overwrite_existing_target(): + got = normalize_attributes( + { + "gen_ai.usage.thinking_tokens": 5, + "gen_ai.usage.reasoning.output_tokens": 9, + } + ) + assert got["gen_ai.usage.reasoning.output_tokens"] == 9 + assert got["gen_ai.usage.thinking_tokens"] == 5 + + +def test_aliases_apply_to_record_usage_spans(pipeline): + provider, exporter = pipeline + _emit( + provider, + {"ag2.span.type": "usage", "gen_ai.usage.cache_read_input_tokens": 3}, + name="record_usage model_call", + ) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert got["gen_ai.usage.cache_read_tokens"] == 3 + assert "gen_ai.span.kind" not in got + + +def test_cache_tokens_use_the_keys_future_agi_reads(): + # Future AGI's attribute registry reads gen_ai.usage.cache_read_tokens / + # cache_write_tokens (tracer/utils/semantic_conventions.py CACHE_READ_TOKENS, + # CACHE_WRITE_TOKENS); the dotted semconv spellings are not read. + got = normalize_attributes( + { + "gen_ai.operation.name": "chat", + "gen_ai.usage.cache_creation_input_tokens": 2, + "gen_ai.usage.cache_read_input_tokens": 3, + } + ) + assert (got["gen_ai.usage.cache_write_tokens"], got["gen_ai.usage.cache_read_tokens"]) == (2, 3) + assert "gen_ai.usage.cache_read.input_tokens" not in got + assert "gen_ai.usage.cache_creation.input_tokens" not in got + + +def test_duplicate_usage_span_does_not_repeat_cache_tokens(): + # record_usage model_call repeats the chat span's cache counts too. + got = normalize_attributes( + { + "ag2.span.type": "usage", + "ag2.usage.kind": "model_call", + "gen_ai.usage.cache_creation_input_tokens": 2, + "gen_ai.usage.cache_read_input_tokens": 3, + } + ) + assert "gen_ai.usage.cache_read_tokens" not in got + assert "gen_ai.usage.cache_write_tokens" not in got + assert (got["ag2.usage.cache_write_tokens"], got["ag2.usage.cache_read_tokens"]) == (2, 3) + + +def test_out_of_band_usage_span_keeps_cache_tokens(): + got = normalize_attributes( + {"ag2.span.type": "usage", "ag2.usage.kind": "aggregation", "gen_ai.usage.cache_read_input_tokens": 3} + ) + assert got["gen_ai.usage.cache_read_tokens"] == 3 + + +def test_model_call_usage_span_does_not_double_count_trace_tokens(pipeline): + """record_usage model_call repeats the chat span's tokens (telemetry.py:437-440 vs 501-504). + + fi-collector promotes gen_ai.usage.* on any span and Observe sums + total_tokens over every span in a trace, so the duplicate moves to ag2.usage.*. + """ + provider, exporter = pipeline + _emit( + provider, + { + "ag2.span.type": "usage", + "ag2.usage.kind": "model_call", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "ag2.usage.total_tokens": 18, + }, + name="record_usage model_call", + ) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert "gen_ai.usage.input_tokens" not in got + assert "gen_ai.usage.output_tokens" not in got + assert got["ag2.usage.input_tokens"] == 11 + assert got["ag2.usage.output_tokens"] == 7 + assert got["ag2.usage.total_tokens"] == 18 + + +def test_subtask_usage_span_keeps_promoted_tokens(pipeline): + """A subtask rollup with no instrumented worker in the trace is the only copy.""" + provider, exporter = pipeline + _emit( + provider, + { + "ag2.span.type": "usage", + "ag2.usage.kind": "subtask", + "ag2.usage.label": "worker", + "gen_ai.usage.input_tokens": 4, + }, + name="record_usage subtask", + ) + (span,) = exporter.get_finished_spans() + assert attrs(span)["gen_ai.usage.input_tokens"] == 4 + + +@pytest.mark.parametrize("kind", ["aggregation", "compaction"]) +def test_out_of_band_usage_spans_keep_promoted_tokens(pipeline, kind): + """aggregate.py / compact.py call the model client outside on_llm_call, so + no chat span carries these tokens.""" + provider, exporter = pipeline + _emit( + provider, + { + "ag2.span.type": "usage", + "ag2.usage.kind": kind, + "gen_ai.usage.input_tokens": 100, + "gen_ai.usage.output_tokens": 50, + }, + name=f"record_usage {kind}", + ) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert got["gen_ai.usage.input_tokens"] == 100 + assert got["gen_ai.usage.output_tokens"] == 50 + assert "ag2.usage.input_tokens" not in got + + +def _subtask_after_worker(provider, worker_name: str, label: str) -> None: + tracer = provider.get_tracer(AG2_INSTRUMENTATION_SCOPE) + with tracer.start_as_current_span("invoke_agent planner", attributes={"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "planner"}): + with tracer.start_as_current_span( + f"invoke_agent {worker_name}", + attributes={"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": worker_name}, + ): + pass + with tracer.start_as_current_span( + "record_usage subtask", + attributes={ + "ag2.span.type": "usage", + "ag2.usage.kind": "subtask", + "ag2.usage.label": label, + "gen_ai.usage.input_tokens": 40, + "gen_ai.usage.output_tokens": 4, + }, + ): + pass + + +def test_subtask_usage_span_demoted_when_its_agent_ran_instrumented_in_the_trace(pipeline): + provider, exporter = pipeline + _subtask_after_worker(provider, worker_name="worker", label="worker") + (rollup,) = [s for s in exporter.get_finished_spans() if s.name == "record_usage subtask"] + got = attrs(rollup) + assert "gen_ai.usage.input_tokens" not in got and "gen_ai.usage.output_tokens" not in got + assert (got["ag2.usage.input_tokens"], got["ag2.usage.output_tokens"]) == (40, 4) + + +def test_subtask_usage_span_kept_when_label_is_another_agent(pipeline): + provider, exporter = pipeline + _subtask_after_worker(provider, worker_name="other_worker", label="worker") + (rollup,) = [s for s in exporter.get_finished_spans() if s.name == "record_usage subtask"] + assert attrs(rollup)["gen_ai.usage.input_tokens"] == 40 + + +def test_agent_name_map_is_bounded(): + processor = AG2SpanProcessor(config=TraceConfig(), max_tracked_traces=2) + provider = TracerProvider() + provider.add_span_processor(processor) + for name in ("a", "b", "c"): + _emit(provider, {"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": name}) + assert len(processor._agents_by_trace) == 2 + assert [set(v) for v in processor._agents_by_trace.values()] == [{"b"}, {"c"}] + + +def test_pending_context_sessions_are_bounded_and_released_on_end(): + from fi_instrumentation import using_session + + processor = AG2SpanProcessor(config=TraceConfig(), max_tracked_traces=1) + provider = TracerProvider() + provider.add_span_processor(processor) + tracer = provider.get_tracer(AG2_INSTRUMENTATION_SCOPE) + with using_session("s"): + open_spans = [tracer.start_span(f"s{i}") for i in range(40)] + assert len(processor._session_by_span) == processor._max_pending_spans == 16 + for span in open_spans: + span.end() + assert len(processor._session_by_span) == 0 + + +def _limited_pipeline(max_span_attributes: int, config: TraceConfig = TraceConfig(), max_attribute_length=None): + from opentelemetry.sdk.trace import SpanLimits + + exporter = InMemorySpanExporter() + provider = TracerProvider( + span_limits=SpanLimits(max_span_attributes=max_span_attributes, max_attribute_length=max_attribute_length), + shutdown_on_exit=False, + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + assert install_span_processor(provider, config=config) is True + return provider, exporter + + +def test_normalizing_keeps_the_sdk_dropped_attribute_count(): + """ReadableSpan.dropped_attributes (exported as dropped_attributes_count) + reads the BoundedAttributes counter; replacing the attributes must keep it.""" + from opentelemetry.attributes import BoundedAttributes + + provider, exporter = _limited_pipeline(2, TraceConfig(hide_inputs=True), max_attribute_length=64) + # The SDK keeps the two newest keys and counts "a" as dropped. + _emit(provider, {"a": 1, "b": 2, "gen_ai.input.messages": "secret"}) + (span,) = exporter.get_finished_spans() + assert attrs(span) == {"b": 2} + assert span.dropped_attributes == 1 + assert isinstance(span._attributes, BoundedAttributes) + assert (span._attributes.maxlen, span._attributes.max_value_len) == (2, 64) + with pytest.raises(TypeError): + span._attributes["late"] = "write" # ended spans stay immutable + + +def test_keys_the_processor_adds_count_against_span_limits(): + provider, exporter = _limited_pipeline(2) + _emit(provider, {"x": 1, "gen_ai.operation.name": "chat"}) # full, nothing dropped yet + (span,) = exporter.get_finished_spans() + # Adding gen_ai.span.kind evicts the oldest key, as the SDK would, and + # the eviction is counted. + assert attrs(span) == {"gen_ai.operation.name": "chat", "gen_ai.span.kind": "LLM"} + assert span.dropped_attributes == 1 + + +def test_processor_only_adds_keys_by_default(pipeline): + """Nothing AG2 set is stripped, including propagation-related keys.""" + provider, exporter = pipeline + source = { + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "a", + "ag2.otel.traceparent": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + "ag2.span.type": "agent", + } + _emit(provider, source) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert {k: got[k] for k in source} == source + + +# --- TraceConfig second gate ---------------------------------------------- + + +def test_trace_config_hides_content_when_capture_was_on(): + content = { + "gen_ai.operation.name": "execute_tool", + "gen_ai.input.messages": '[{"role":"user","content":"hi"}]', + "gen_ai.output.messages": '[{"role":"assistant","content":"yo"}]', + "gen_ai.tool.call.arguments": '{"city":"Paris"}', + "gen_ai.tool.call.result": "sunny", + "ag2.human_input.prompt": "ok?", + "ag2.human_input.response": "yes", + "gen_ai.tool.name": "get_weather", + } + got = normalize_attributes(content, TraceConfig(hide_inputs=True, hide_outputs=True)) + assert got == { + "gen_ai.operation.name": "execute_tool", + "gen_ai.span.kind": "TOOL", + "gen_ai.tool.name": "get_weather", + } + kept = normalize_attributes(content, TraceConfig(hide_inputs=False, hide_outputs=False)) + for key, value in content.items(): + assert kept[key] == value + + +# AG2 records message content as JSON strings, not as the flattened +# ``gen_ai.input.messages.{i}.message.content`` keys TraceConfig.mask's text +# and image rules match, so those flags drop the whole JSON attribute. +_AG2_CONTENT = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "m", + "gen_ai.system_instructions": '[{"type": "text", "content": "be terse"}]', + "gen_ai.input.messages": '[{"content": "my secret prompt", "role": "user"}]', + "gen_ai.output.messages": '[{"content": "secret answer", "role": "assistant"}]', + "gen_ai.tool.call.arguments": '{"city": "Paris"}', + "gen_ai.tool.call.result": "sunny", + "ag2.human_input.prompt": "ok?", + "ag2.human_input.response": "yes", +} +_INPUT_TEXT_KEYS = { + "gen_ai.system_instructions", + "gen_ai.input.messages", + "gen_ai.tool.call.arguments", + "ag2.human_input.prompt", +} +_OUTPUT_TEXT_KEYS = {"gen_ai.output.messages", "gen_ai.tool.call.result", "ag2.human_input.response"} + + +@pytest.mark.parametrize( + "flags, dropped", + [ + ({"hide_input_text": True}, _INPUT_TEXT_KEYS), + ({"hide_output_text": True}, _OUTPUT_TEXT_KEYS), + ({"hide_inputs": True}, _INPUT_TEXT_KEYS), + ({"hide_outputs": True}, _OUTPUT_TEXT_KEYS), + ({"hide_input_messages": True}, {"gen_ai.input.messages", "gen_ai.system_instructions"}), + ({"hide_output_messages": True}, {"gen_ai.output.messages"}), + ({"hide_input_text": True, "hide_output_text": True}, _INPUT_TEXT_KEYS | _OUTPUT_TEXT_KEYS), + ], + ids=lambda v: "+".join(sorted(v)) if isinstance(v, dict) else None, +) +def test_trace_config_flags_on_ag2_json_content(flags, dropped): + got = normalize_attributes(_AG2_CONTENT, TraceConfig(**flags)) + assert not dropped & set(got), sorted(dropped & set(got)) + for key in set(_AG2_CONTENT) - dropped: + assert got[key] == _AG2_CONTENT[key], key + + +@pytest.mark.parametrize("flag", ["hide_input_images", "hide_embedding_vectors"]) +def test_flags_with_nothing_to_act_on_leave_ag2_content(flag): + """AG2 records only text parts (binary inputs are omitted) and no + embeddings, so these flags change nothing.""" + got = normalize_attributes(_AG2_CONTENT, TraceConfig(**{flag: True})) + for key, value in _AG2_CONTENT.items(): + assert got[key] == value, key + + +def test_trace_config_reads_env(monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + processor = AG2SpanProcessor() + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(processor) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.input.messages": "[]"}) + (span,) = exporter.get_finished_spans() + assert "gen_ai.input.messages" not in attrs(span) + + +# --- input.value / output.value -------------------------------------------- +# Future AGI fills a span's input and output only from input.value / +# output.value, so captured AG2 content is copied there. + + +@pytest.mark.parametrize( + "source, expected", + [ + ( + { + "gen_ai.operation.name": "chat", + "gen_ai.input.messages": '[{"content": "hi", "role": "user"}]', + "gen_ai.output.messages": '[{"content": "yo", "role": "assistant"}]', + }, + { + "input.value": '[{"content": "hi", "role": "user"}]', + "input.mime_type": "application/json", + # One completion: the message object, which Observe's chat view + # shows as the assistant reply (it shows nothing for a list). + "output.value": '{"content": "yo", "role": "assistant"}', + "output.mime_type": "application/json", + }, + ), + ( + { + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.call.arguments": '{"city": "Paris"}', + "gen_ai.tool.call.result": "sunny", + }, + { + "input.value": '{"city": "Paris"}', + "input.mime_type": "application/json", + "output.value": "sunny", + "output.mime_type": "text/plain", + }, + ), + ( + { + "gen_ai.operation.name": "await_human_input", + "ag2.human_input.prompt": "ok?", + "ag2.human_input.response": "yes", + }, + { + "input.value": "ok?", + "input.mime_type": "text/plain", + "output.value": "yes", + "output.mime_type": "text/plain", + }, + ), + ], + ids=["chat", "execute_tool", "await_human_input"], +) +def test_captured_content_becomes_span_input_and_output(pipeline, source, expected): + provider, exporter = pipeline + _emit(provider, source) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert {k: got.get(k) for k in expected} == expected + for key, value in source.items(): + assert got[key] == value + + +def test_no_input_or_output_without_captured_content(pipeline): + provider, exporter = pipeline + _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.request.model": "m"}) + (span,) = exporter.get_finished_spans() + assert not {"input.value", "output.value", "input.mime_type", "output.mime_type"} & set(attrs(span)) + + +@pytest.mark.parametrize( + "flags, gone", + [ + ({"hide_inputs": True}, {"input.value", "input.mime_type"}), + ({"hide_input_text": True}, {"input.value", "input.mime_type"}), + ({"hide_input_messages": True}, {"input.value", "input.mime_type"}), + ({"hide_outputs": True}, {"output.value", "output.mime_type"}), + ({"hide_output_text": True}, {"output.value", "output.mime_type"}), + ({"hide_output_messages": True}, {"output.value", "output.mime_type"}), + ], + ids=lambda v: "+".join(sorted(v)) if isinstance(v, dict) else None, +) +def test_hidden_content_is_not_copied_to_input_or_output(flags, gone): + chat = {k: v for k, v in _AG2_CONTENT.items() if not k.startswith(("gen_ai.tool", "ag2.human"))} + got = normalize_attributes(chat, TraceConfig(**flags)) + assert not gone & set(got), sorted(gone & set(got)) + kept = {"input.value", "output.value"} - gone + assert kept <= set(got), "the side that is not hidden is still copied" + + +def test_existing_input_and_output_values_are_kept(pipeline): + provider, exporter = pipeline + _emit( + provider, + { + "gen_ai.operation.name": "chat", + "gen_ai.input.messages": "[]", + "input.value": "mine", + "output.value": "also mine", + "gen_ai.output.messages": "[]", + }, + ) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert (got["input.value"], got["output.value"]) == ("mine", "also mine") + + +def test_chat_with_several_completions_keeps_the_list(): + messages = '[{"content": "a", "role": "assistant"}, {"content": "b", "role": "assistant"}]' + got = normalize_attributes({"gen_ai.operation.name": "chat", "gen_ai.output.messages": messages}) + assert got["output.value"] == messages + + +# --- invoke_agent input / output ------------------------------------------- +# AG2 records no content on invoke_agent; the first chat input and the last +# chat output of the agent's turn are copied there so Observe's trace list +# shows the question and the answer. + + +def _agent_turn(provider, chats, agent_attrs=None): + tracer = provider.get_tracer(AG2_INSTRUMENTATION_SCOPE) + with tracer.start_as_current_span( + "invoke_agent a", attributes={"gen_ai.operation.name": "invoke_agent", **(agent_attrs or {})} + ): + for chat in chats: + with tracer.start_as_current_span("chat m", attributes={"gen_ai.operation.name": "chat", **chat}): + pass + + +_TURN = [ + { + "gen_ai.input.messages": '[{"content": "Q?", "role": "user"}]', + "gen_ai.output.messages": '[{"content": null, "role": "assistant", "tool_calls": []}]', + }, + { + "gen_ai.input.messages": '[{"content": "Q?", "role": "user"}, {"content": "tool says", "role": "tool"}]', + "gen_ai.output.messages": '[{"content": "A.", "role": "assistant"}]', + }, +] + + +def _agent(exporter): + (agent,) = [s for s in exporter.get_finished_spans() if s.name == "invoke_agent a"] + return attrs(agent) + + +def test_agent_span_gets_first_chat_input_and_last_chat_output(pipeline): + provider, exporter = pipeline + _agent_turn(provider, _TURN) + got = _agent(exporter) + assert got["input.value"] == '[{"content": "Q?", "role": "user"}]' + assert got["output.value"] == '{"content": "A.", "role": "assistant"}' + assert got["input.mime_type"] == got["output.mime_type"] == "application/json" + + +def test_agent_span_has_no_io_without_captured_content(pipeline): + provider, exporter = pipeline + _agent_turn(provider, [{"gen_ai.request.model": "m"}, {"gen_ai.request.model": "m"}]) + got = _agent(exporter) + assert not {"input.value", "output.value", "input.mime_type", "output.mime_type"} & set(got) + + +def test_agent_span_io_follows_hide_flags(): + provider, exporter = memory_provider() + install_span_processor(provider, config=TraceConfig(hide_inputs=True)) + _agent_turn(provider, _TURN) + got = _agent(exporter) + assert "input.value" not in got and "Q?" not in str(got) + assert got["output.value"] == '{"content": "A.", "role": "assistant"}' + + +def test_agent_span_keeps_its_own_io(pipeline): + provider, exporter = pipeline + _agent_turn(provider, _TURN, agent_attrs={"input.value": "mine", "output.value": "also mine"}) + got = _agent(exporter) + assert (got["input.value"], got["output.value"]) == ("mine", "also mine") + + +def test_agent_io_state_is_released(pipeline): + provider, exporter = pipeline + _agent_turn(provider, _TURN) + (processor,) = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2SpanProcessor)] + assert not processor._agent_io + + +# --- Anthropic input tokens -------------------------------------------------- +# AG2's Anthropic mapper reports prompt_tokens without cached tokens (its OpenAI +# mapper includes them). gen_ai.usage.input_tokens means total input, so on an +# anthropic span that counts, the cache read and write tokens are added. + + +def test_anthropic_input_tokens_include_cached_tokens(): + got = normalize_attributes( + { + "gen_ai.operation.name": "chat", + "gen_ai.provider.name": "anthropic", + "gen_ai.usage.input_tokens": 4, + "gen_ai.usage.output_tokens": 97, + "gen_ai.usage.total_tokens": 101, + "gen_ai.usage.cache_read_input_tokens": 9747, + "gen_ai.usage.cache_creation_input_tokens": 2, + } + ) + assert got["gen_ai.usage.input_tokens"] == 9753 + assert got["gen_ai.usage.total_tokens"] == 9850 + assert (got["gen_ai.usage.cache_read_tokens"], got["gen_ai.usage.cache_write_tokens"]) == (9747, 2) + + +def test_anthropic_without_total_gets_no_total_invented(): + got = normalize_attributes( + { + "gen_ai.operation.name": "chat", + "gen_ai.provider.name": "anthropic", + "gen_ai.usage.input_tokens": 4, + "gen_ai.usage.output_tokens": 97, + "gen_ai.usage.cache_read_input_tokens": 9747, + } + ) + assert got["gen_ai.usage.input_tokens"] == 9751 + assert "gen_ai.usage.total_tokens" not in got + + +@pytest.mark.parametrize("provider_name", ["openai", None]) +def test_other_providers_input_tokens_unchanged(provider_name): + source = { + "gen_ai.operation.name": "chat", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.cache_read_input_tokens": 3, + } + if provider_name: + source["gen_ai.provider.name"] = provider_name + assert normalize_attributes(source)["gen_ai.usage.input_tokens"] == 11 + + +def test_anthropic_duplicate_usage_span_stays_off_the_counted_keys(): + got = normalize_attributes( + { + "ag2.span.type": "usage", + "ag2.usage.kind": "model_call", + "gen_ai.provider.name": "anthropic", + "gen_ai.usage.input_tokens": 4, + "gen_ai.usage.cache_read_input_tokens": 9747, + } + ) + assert "gen_ai.usage.input_tokens" not in got + assert got["ag2.usage.input_tokens"] == 4 # AG2's own number, kept as reported + + +def test_anthropic_counted_usage_span_includes_cached_tokens(): + got = normalize_attributes( + { + "ag2.span.type": "usage", + "ag2.usage.kind": "aggregation", + "gen_ai.provider.name": "anthropic", + "gen_ai.usage.input_tokens": 4, + "gen_ai.usage.cache_read_input_tokens": 100, + } + ) + assert got["gen_ai.usage.input_tokens"] == 104 + + +# --- installation ---------------------------------------------------------- + + +def test_install_is_idempotent_and_prepends(): + provider, exporter = memory_provider() + assert install_span_processor(provider) is True + assert install_span_processor(provider) is False + chain = provider._active_span_processor._span_processors + assert isinstance(chain[0], AG2SpanProcessor) + assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 + assert isinstance(chain[1], SimpleSpanProcessor) + + +def test_install_keeps_fi_register_exporter(): + """register() marks its exporter as a replaceable default; install must not drop it.""" + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + exporter = InMemorySpanExporter() + provider = register( + project_type=ProjectType.OBSERVE, + project_name="ag2-unit", + batch=False, + span_exporter=exporter, + verbose=False, + ) + try: + assert install_span_processor(provider) is True + _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.usage.thinking_tokens": 2}) + (span,) = exporter.get_finished_spans() + got = attrs(span) + assert got["gen_ai.span.kind"] == "LLM" + assert got["gen_ai.usage.reasoning.output_tokens"] == 2 + finally: + provider.shutdown() + + +def _fi_provider(exporter): + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + return register( + project_type=ProjectType.OBSERVE, + project_name="ag2-unit", + batch=False, + span_exporter=exporter, + verbose=False, + ) + + +def test_later_add_span_processor_on_fi_provider_keeps_ag2_processor_first(): + """fi's provider shuts down and clears every processor on the first + add_span_processor() after register() (otel.py 329-340); the user's new + exporter must still receive normalized AG2 spans.""" + provider = _fi_provider(InMemorySpanExporter()) + try: + assert install_span_processor(provider) is True + replacement = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(replacement)) + + chain = provider._active_span_processor._span_processors + assert isinstance(chain[0], AG2SpanProcessor), chain + assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 + assert [type(p) for p in chain[1:]] == [SimpleSpanProcessor] + + _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.usage.thinking_tokens": 2}) + (span,) = replacement.get_finished_spans() + got = attrs(span) + assert got["gen_ai.span.kind"] == "LLM" + assert got["gen_ai.usage.reasoning.output_tokens"] == 2 + finally: + provider.shutdown() + + +def test_later_add_span_processor_on_sdk_provider_keeps_ag2_processor_first(): + provider, first = memory_provider() + assert install_span_processor(provider) is True + assert install_span_processor(provider) is False # no second wrapper either + second = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(second)) + provider.add_span_processor(SimpleSpanProcessor(InMemorySpanExporter())) + + chain = provider._active_span_processor._span_processors + assert isinstance(chain[0], AG2SpanProcessor), chain + assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 + assert len(chain) == 4 + _emit(provider, {"gen_ai.operation.name": "chat"}) + for exporter in (first, second): + (span,) = exporter.get_finished_spans() + assert attrs(span)["gen_ai.span.kind"] == "LLM" + + +def test_provider_shutdown_still_shuts_the_processor_down(): + provider, _ = memory_provider() + install_span_processor(provider) + (processor,) = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2SpanProcessor)] + provider.shutdown() + assert processor._shutdown is True + + +def test_install_warns_when_processors_run_concurrently(caplog): + """A ConcurrentMultiSpanProcessor runs on_end of every processor in + parallel, so the exporter may read a span before it is normalized.""" + from opentelemetry.sdk.trace import ConcurrentMultiSpanProcessor + + provider = TracerProvider(active_span_processor=ConcurrentMultiSpanProcessor(), shutdown_on_exit=False) + try: + with caplog.at_level(logging.WARNING, logger="traceai_ag2"): + assert install_span_processor(provider) is True + assert any("ConcurrentMultiSpanProcessor" in r.getMessage() for r in caplog.records), caplog.text + finally: + provider.shutdown() + + +def test_install_without_sdk_provider_returns_false(): + class _NoChain: + pass + + assert install_span_processor(_NoChain()) is False + + +def test_shutdown_processor_stops_mapping(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + processor = AG2SpanProcessor(config=TraceConfig()) + provider.add_span_processor(processor) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + processor.shutdown() + _emit(provider, {"gen_ai.operation.name": "chat"}) + (span,) = exporter.get_finished_spans() + assert "gen_ai.span.kind" not in attrs(span) + assert processor.force_flush() is True diff --git a/python/frameworks/ag2/tests/test_resource_headers.py b/python/frameworks/ag2/tests/test_resource_headers.py new file mode 100644 index 000000000..e0b9e75ff --- /dev/null +++ b/python/frameworks/ag2/tests/test_resource_headers.py @@ -0,0 +1,134 @@ +"""Resource attributes and headers on the wire (the harness Receiver drops both). + +``setup()`` without a provider calls ``fi_instrumentation.register`` itself; +this checks what that sends: path ``/tracer/v1/traces``, ``X-Api-Key`` / +``X-Secret-Key`` headers, and resource ``project_name`` + +``project_type=observe``. Loopback only; placeholder keys. +""" + +from __future__ import annotations + +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List + +import pytest + +from traceai_ag2 import setup + +from ._support import ask, weather_agent + + +class _CaptureServer: + def __init__(self) -> None: + self.requests: List[Dict[str, Any]] = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + owner.requests.append( + { + "path": self.path, + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": self.rfile.read(length), + } + ) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = f"http://127.0.0.1:{self._server.server_port}" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +def _resource_attributes(body: bytes) -> List[Dict[str, Any]]: + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + + request = ExportTraceServiceRequest() + request.ParseFromString(body) + out = [] + for resource_spans in request.resource_spans: + out.append({kv.key: kv.value.string_value for kv in resource_spans.resource.attributes}) + return out + + +@pytest.fixture() +def capture(monkeypatch): + server = _CaptureServer() + monkeypatch.setenv("FI_BASE_URL", server.origin) + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + monkeypatch.delenv("FI_PROJECT_NAME", raising=False) + yield server + server.close() + + +def test_setup_registers_observe_project_with_fi_headers(capture): + agent = weather_agent() + provider = setup(agent, project_name="ag2-resource-test") + try: + assert provider.resource.attributes["project_name"] == "ag2-resource-test" + assert provider.resource.attributes["project_type"] == "observe" + ask(agent) + assert provider.force_flush(timeout_millis=10_000) + finally: + provider.shutdown() + + assert capture.requests, "no OTLP request reached the loopback collector" + for request in capture.requests: + assert request["path"] == "/tracer/v1/traces" + headers = request["headers"] + assert headers["x-api-key"] == "placeholder-api-key" + assert headers["x-secret-key"] == "placeholder-secret-key" + assert "authorization" not in headers + assert headers["content-type"] == "application/x-protobuf" + for resource in _resource_attributes(request["body"]): + assert resource["project_name"] == "ag2-resource-test" + assert resource["project_type"] == "observe" + assert "openinference.project.name" not in resource + + +def test_setup_project_name_falls_back_to_env(capture, monkeypatch): + monkeypatch.setenv("FI_PROJECT_NAME", "ag2-from-env") + provider = setup(weather_agent()) + try: + assert provider.resource.attributes["project_name"] == "ag2-from-env" + assert provider.resource.attributes["project_type"] == "observe" + finally: + provider.shutdown() + + +def test_agent_returns_when_collector_is_down(monkeypatch): + """Export failure is logged by the exporter and never raised into the agent.""" + import socket + + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + with socket.socket() as sock: # reserve then release a port nothing listens on + sock.bind(("127.0.0.1", 0)) + dead_port = sock.getsockname()[1] + monkeypatch.setenv("FI_BASE_URL", f"http://127.0.0.1:{dead_port}") + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + + provider = register(project_type=ProjectType.OBSERVE, project_name="ag2-down", timeout=1, verbose=False) + try: + agent = weather_agent() + setup(agent, tracer_provider=provider) + reply = ask(agent) + assert reply.body == "It is sunny in Paris." + provider.force_flush(timeout_millis=3_000) # returns; does not raise + finally: + provider.shutdown() diff --git a/python/frameworks/ag2/tests/test_setup.py b/python/frameworks/ag2/tests/test_setup.py new file mode 100644 index 000000000..3d9198ae9 --- /dev/null +++ b/python/frameworks/ag2/tests/test_setup.py @@ -0,0 +1,851 @@ +"""Unit tests for setup() against real AG2 with its scripted TestConfig model. + +Covers AC-04 (traceparent relay, without the network hub), AC-05, AC-06, +AC-07 and the "does not wrap Agent" rule. +""" + +from __future__ import annotations + +import asyncio +import inspect +import logging + +import pytest +from fi_instrumentation.instrumentation.config import TraceConfig +from opentelemetry.trace import StatusCode + +import traceai_ag2 +from traceai_ag2 import AG2SpanProcessor, create_telemetry_middleware, setup + +from ._support import ( + FINAL_ANSWER, + MODEL, + PROVIDER, + TOOL_CALL_ID, + TOOL_RESULT, + USER_PROMPT, + ask, + assert_no_content, + attrs, + expected_one_tool_kinds, + fake_register, + find, + get_weather, + memory_provider, + weather_agent, + weather_config, +) + + +def _telemetry_middlewares(agent): + from ag2.middleware.builtin.telemetry import TelemetryMiddleware + + try: + entries = [e.middleware for e in agent.middleware] # ag2 >= 1.0.2 + except AttributeError: + entries = list(agent._middleware) # ag2 1.0.0 / 1.0.1 + return [m for m in entries if isinstance(m, TelemetryMiddleware)] + + +# --- upstream drift guards ------------------------------------------------- + + +def test_scope_and_strings_match_installed_ag2(): + from ag2 import _telemetry_consts + from ag2.middleware.builtin import telemetry + + assert traceai_ag2.AG2_INSTRUMENTATION_SCOPE == _telemetry_consts.OTEL_INSTRUMENTING_MODULE + source = inspect.getsource(telemetry) + for operation in ("invoke_agent", "chat", "execute_tool", "await_human_input"): + assert f'"gen_ai.operation.name", "{operation}"' in source + for key in traceai_ag2.USAGE_KEY_ALIASES: + assert f'"{key}"' in source + params = inspect.signature(telemetry.TelemetryMiddleware.__init__).parameters + assert params["capture_content"].default is True # upstream default; the helper keeps it + # README: hide_llm_invocation_parameters has nothing to act on because AG2 + # sets no invocation-parameter attribute. + assert "gen_ai.request.parameters" not in source + inspect.getsource(_telemetry_consts) + + +# --- privacy: traceAI TraceConfig convention (content on by default) ------- + + +def test_setup_defaults_capture_content_on_and_spans_carry_content(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, config=TraceConfig()) + + (mw,) = _telemetry_middlewares(agent) + assert mw._capture_content is True + + reply = ask(agent) + assert reply.body == FINAL_ANSWER + spans = exporter.get_finished_spans() + (tool,) = find(spans, "execute_tool") + assert attrs(tool)["gen_ai.tool.call.result"] == TOOL_RESULT + (final_chat,) = find(spans, f"chat {MODEL}") + assert USER_PROMPT in attrs(final_chat)["gen_ai.input.messages"] + + +def test_default_content_is_hidden_by_trace_config_alone(): + """No second switch: TraceConfig hide flags remove content from a default setup().""" + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + ask(agent) + spans = exporter.get_finished_spans() + assert spans + for span in spans: + assert_no_content(attrs(span), span.name) + + +def test_default_content_is_hidden_by_fi_hide_env(monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider) # config=None -> TraceConfig() reads FI_HIDE_* + ask(agent) + spans = exporter.get_finished_spans() + assert spans + for span in spans: + assert_no_content(attrs(span), span.name) + + +def test_create_telemetry_middleware_defaults_on(): + mw = create_telemetry_middleware() + assert mw._capture_content is True + + +def test_capture_content_false_passthrough_has_no_content(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, capture_content=False) + ask(agent) + for span in exporter.get_finished_spans(): + assert_no_content(attrs(span), span.name) + + +def test_capture_content_opt_in_forwards_true(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, capture_content=True, config=TraceConfig()) + ask(agent) + spans = exporter.get_finished_spans() + (tool,) = find(spans, "execute_tool") + assert "Paris" in attrs(tool)["gen_ai.tool.call.arguments"] + assert attrs(tool)["gen_ai.tool.call.result"] == TOOL_RESULT + (final_chat,) = find(spans, f"chat {MODEL}") + assert USER_PROMPT in attrs(final_chat)["gen_ai.input.messages"] + assert FINAL_ANSWER in attrs(final_chat)["gen_ai.output.messages"] + + +def test_trace_config_is_a_second_gate_when_content_is_on(): + provider, exporter = memory_provider() + agent = weather_agent() + setup( + agent, + tracer_provider=provider, + capture_content=True, + config=TraceConfig(hide_inputs=True, hide_outputs=True), + ) + ask(agent) + for span in exporter.get_finished_spans(): + assert_no_content(attrs(span), span.name) + + +def test_later_setup_config_replaces_installed_processor_config(caplog): + """README path: setup(tracer_provider=...) first, then setup(agent, config=...). + The most recent explicit config wins and the change is logged.""" + provider, exporter = memory_provider() + setup(tracer_provider=provider) + agent = weather_agent() + with caplog.at_level(logging.WARNING, logger="traceai_ag2"): + setup( + agent, + tracer_provider=provider, + capture_content=True, + config=TraceConfig(hide_inputs=True, hide_outputs=True), + ) + assert any("TraceConfig" in r.getMessage() for r in caplog.records), caplog.text + chain = provider._active_span_processor._span_processors + assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 + ask(agent) + spans = exporter.get_finished_spans() + assert spans + for span in spans: + assert_no_content(attrs(span), span.name) + + +def test_later_setup_without_config_keeps_installed_config(caplog): + """config=None means "no change": a later setup() never re-exposes content.""" + provider, exporter = memory_provider() + setup(tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + agent = weather_agent() + with caplog.at_level(logging.WARNING, logger="traceai_ag2"): + setup(agent, tracer_provider=provider, capture_content=True) + setup(agent, tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + assert not [r for r in caplog.records if "TraceConfig" in r.getMessage()], caplog.text + ask(agent) + for span in exporter.get_finished_spans(): + assert_no_content(attrs(span), span.name) + + +# --- span shape, kinds, model, usage ---------------------------------------- + + +def test_text_hide_flags_remove_ag2_message_json(): + """hide_input_text / hide_output_text act on AG2's real JSON-string content.""" + provider, exporter = memory_provider() + agent = weather_agent() + setup( + agent, + tracer_provider=provider, + capture_content=True, + config=TraceConfig(hide_input_text=True, hide_output_text=True), + ) + ask(agent) + spans = exporter.get_finished_spans() + assert find(spans, "chat") + for span in spans: + assert_no_content(attrs(span), span.name) + + +def test_hide_input_text_keeps_outputs(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, capture_content=True, config=TraceConfig(hide_input_text=True)) + ask(agent) + spans = exporter.get_finished_spans() + (final_chat,) = find(spans, f"chat {MODEL}") + assert "gen_ai.input.messages" not in attrs(final_chat) + assert FINAL_ANSWER in attrs(final_chat)["gen_ai.output.messages"] + (tool,) = find(spans, "execute_tool") + assert "gen_ai.tool.call.arguments" not in attrs(tool) + assert attrs(tool)["gen_ai.tool.call.result"] == TOOL_RESULT + + +def test_ag2_never_records_image_bytes(): + """Why hide_input_images has nothing to act on: AG2 serialises only text + parts into gen_ai.input.messages, even with capture_content=True.""" + from ag2 import Agent + from ag2.events import BinaryInput, TextInput + from ag2.testing import TestConfig + + try: + from ag2.events import BinaryType + + image = BinaryInput(b"\x89PNG-SECRET-PIXELS", type=BinaryType.IMAGE, media_type="image/png") + except Exception as exc: # pragma: no cover - older BinaryInput signature + pytest.skip(f"cannot build an image BinaryInput on this ag2: {exc!r}") + provider, exporter = memory_provider() + agent = Agent("img_bot", config=TestConfig("seen")) + setup(agent, tracer_provider=provider, capture_content=True) + + async def _run(): + return await agent.ask(TextInput("describe"), image) + + asyncio.run(_run()) + (chat,) = find(exporter.get_finished_spans(), "chat") + assert "describe" in attrs(chat)["gen_ai.input.messages"] + for span in exporter.get_finished_spans(): + for value in attrs(span).values(): + assert "SECRET-PIXELS" not in str(value), span.name + + +def test_one_tool_run_span_shape(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider) + ask(agent) + spans = exporter.get_finished_spans() + kinds = {s.name: attrs(s).get("gen_ai.span.kind") for s in spans} + assert kinds == expected_one_tool_kinds() + + (tool,) = find(spans, "execute_tool") + assert attrs(tool)["gen_ai.tool.name"] == "get_weather" + assert attrs(tool)["gen_ai.tool.call.id"] == TOOL_CALL_ID + + (final_chat,) = find(spans, f"chat {MODEL}") + a = attrs(final_chat) + assert a["gen_ai.request.model"] == MODEL + assert a["gen_ai.response.model"] == MODEL + assert a["gen_ai.provider.name"] == PROVIDER + assert a["gen_ai.usage.input_tokens"] == 11 + assert a["gen_ai.usage.output_tokens"] == 7 + assert a["gen_ai.usage.cache_creation_input_tokens"] == 2 + assert a["gen_ai.usage.cache_write_tokens"] == 2 + assert a["gen_ai.usage.cache_read_input_tokens"] == 3 + assert a["gen_ai.usage.cache_read_tokens"] == 3 + assert a["gen_ai.usage.thinking_tokens"] == 5 + assert a["gen_ai.usage.reasoning.output_tokens"] == 5 + + root = [s for s in spans if s.parent is None] + assert [s.name for s in root] == ["invoke_agent weather_bot"] + assert len({s.context.trace_id for s in spans}) == 1 + + +def test_provider_name_is_not_invented(): + provider, exporter = memory_provider() + agent = weather_agent(with_provider=False) + setup(agent, tracer_provider=provider) + ask(agent) + for span in exporter.get_finished_spans(): + assert "gen_ai.provider.name" not in attrs(span), span.name + + +def test_provider_name_is_forwarded_when_given(): + provider, exporter = memory_provider() + agent = weather_agent(with_provider=False) + setup(agent, tracer_provider=provider, provider_name="openai") + ask(agent) + (root,) = find(exporter.get_finished_spans(), "invoke_agent") + assert attrs(root)["gen_ai.provider.name"] == "openai" + + +# --- AC-07 session ------------------------------------------------------------- + + +def test_session_is_absent_unless_app_sets_it(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider) + ask(agent) + for span in exporter.get_finished_spans(): + a = attrs(span) + assert "session.id" not in a + assert "gen_ai.conversation.id" not in a + + +def test_app_can_stamp_session_through_span_attributes(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, span_attributes={"session.id": "sess-42"}) + ask(agent) + spans = exporter.get_finished_spans() + assert spans + for span in spans: + assert attrs(span)["session.id"] == "sess-42", span.name + + +def _ask_inside(agent, *context_managers): + from contextlib import ExitStack + + async def _run(): + with ExitStack() as stack: + for cm in context_managers: + stack.enter_context(cm) + return await agent.ask(USER_PROMPT) + + return asyncio.run(_run()) + + +def test_using_session_sets_session_on_every_ag2_span(): + from fi_instrumentation import using_session + + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider) + _ask_inside(agent, using_session("sess-ctx-1")) + spans = exporter.get_finished_spans() + assert {s.name for s in spans} >= {"invoke_agent weather_bot", "execute_tool get_weather"} + for span in spans: + assert attrs(span).get("session.id") == "sess-ctx-1", span.name + + +def test_using_attributes_context_reaches_ag2_spans(): + from fi_instrumentation import using_attributes + + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider) + _ask_inside(agent, using_attributes(session_id="s-1", user_id="u-1", metadata={"tenant": "acme"}, tags=["t1"])) + for span in exporter.get_finished_spans(): + a = attrs(span) + assert a.get("session.id") == "s-1", span.name + assert a.get("user.id") == "u-1", span.name + assert "acme" in a.get("metadata", ""), span.name + assert list(a.get("tag.tags", ())) == ["t1"], span.name + + +def test_context_session_wins_but_other_ag2_keys_are_not_overridden(): + """session.id from using_session replaces a static span_attributes value; + any other key AG2 set itself is kept.""" + from fi_instrumentation import using_session, using_user + + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider, span_attributes={"session.id": "static", "user.id": "from-ag2"}) + _ask_inside(agent, using_session("dynamic"), using_user("from-context")) + spans = exporter.get_finished_spans() + assert spans + for span in spans: + a = attrs(span) + assert a["session.id"] == "dynamic", span.name + assert a["user.id"] == "from-ag2", span.name + + +def test_context_attributes_skip_foreign_scope_spans(): + from fi_instrumentation import using_session + + provider, exporter = memory_provider() + setup(tracer_provider=provider) + with using_session("sess-ctx-1"): + with provider.get_tracer("some.other.instrumentation").start_as_current_span("other"): + pass + (span,) = exporter.get_finished_spans() + assert "session.id" not in attrs(span) + + +def test_context_keys_are_evicted_last_when_processor_keys_fill_a_limited_span(): + """on_start writes traceAI context keys before AG2 sets any attribute, so + they are the oldest keys on the span. When the keys the processor adds + (span kind, usage aliases) overflow max_span_attributes, the oldest keys + are evicted; session.id and user.id must not be among them.""" + from fi_instrumentation import using_attributes + from opentelemetry.sdk.trace import SpanLimits, TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + def final_chat(max_span_attributes): + exporter = InMemorySpanExporter() + provider = TracerProvider( + span_limits=SpanLimits(max_span_attributes=max_span_attributes), shutdown_on_exit=False + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + agent = weather_agent() + setup(agent, tracer_provider=provider) + _ask_inside(agent, using_attributes(session_id="sess-limit", user_id="user-limit")) + (chat,) = find(exporter.get_finished_spans(), f"chat {MODEL}") + return chat + + added_by_processor = {"gen_ai.span.kind", *traceai_ag2.USAGE_KEY_ALIASES.values()} + unlimited = attrs(final_chat(128)) + assert added_by_processor <= set(unlimited) + written_by_ag2_and_context = [k for k in unlimited if k not in added_by_processor] + + # Exactly full when AG2 is done, so only the processor's keys evict. + chat = final_chat(len(written_by_ag2_and_context)) + a = attrs(chat) + assert chat.dropped_attributes == len(added_by_processor) + assert a.get("session.id") == "sess-limit" + assert a.get("user.id") == "user-limit" + assert a.get("gen_ai.usage.input_tokens") == 11 + assert a.get("gen_ai.span.kind") == "LLM" + + +# --- AC-06 errors and cancellation ------------------------------------------- + + +def _test_config_supports(parameter: str) -> bool: + from ag2.testing import TestConfig + + return parameter in inspect.signature(TestConfig).parameters + + +def test_tool_exception_sets_error_status(): + from ag2 import Agent + from ag2.events import ToolCallEvent + from ag2.testing import TestConfig + + def explode(city: str) -> str: + """Always fails.""" + raise ValueError("weather service down") + + provider, exporter = memory_provider() + script = (ToolCallEvent("explode", arguments='{"city": "Paris"}'), "Sorry, the tool failed.") + tolerant = _test_config_supports("raise_tool_errors") # ag2 >= 1.0.3 + config = TestConfig(*script, raise_tool_errors=False) if tolerant else TestConfig(*script) + agent = Agent("fragile_bot", config=config, tools=[explode]) + setup(agent, tracer_provider=provider) + if tolerant: + ask(agent) + else: # older TestClient re-raises the tool error from the next model call + with pytest.raises(ValueError): + ask(agent) + (tool,) = find(exporter.get_finished_spans(), "execute_tool explode") + assert tool.status.status_code == StatusCode.ERROR + assert attrs(tool)["gen_ai.span.kind"] == "TOOL" + + +def test_model_call_exception_sets_error_and_closes_spans(): + """The scripted client raises inside the LLM call (the tool's error, re-raised + by TestClient's default ``raise_tool_errors``); works on every ag2 1.x.""" + from ag2 import Agent + from ag2.events import ToolCallEvent + from ag2.testing import TestConfig + + def explode(city: str) -> str: + """Always fails.""" + raise ValueError("weather service down") + + provider, exporter = memory_provider() + agent = Agent( + "broken_bot", + config=TestConfig(ToolCallEvent("explode", arguments='{"city": "Paris"}'), "unreachable"), + tools=[explode], + ) + setup(agent, tracer_provider=provider) + with pytest.raises(ValueError): + ask(agent) + spans = exporter.get_finished_spans() + chats = find(spans, "chat") + assert [c.status.status_code for c in chats] == [StatusCode.UNSET, StatusCode.ERROR] + (root,) = find(spans, "invoke_agent") + assert root.status.status_code == StatusCode.ERROR + assert all(s.end_time is not None for s in spans) + + +def test_provider_exception_sets_error_and_closes_spans(): + import ag2.testing as ag2_testing + from ag2 import Agent + from ag2.testing import TestConfig + + if "BaseException" not in inspect.getsource(ag2_testing.TestClient): + pytest.skip("scripting a raised exception needs ag2.testing from ag2 >= 1.0.4") + + provider, exporter = memory_provider() + agent = Agent("broken_bot", config=TestConfig(RuntimeError("provider 500"))) + setup(agent, tracer_provider=provider) + with pytest.raises(RuntimeError): + ask(agent) + spans = exporter.get_finished_spans() + (chat,) = find(spans, "chat") + (root,) = find(spans, "invoke_agent") + assert chat.status.status_code == StatusCode.ERROR + assert root.status.status_code == StatusCode.ERROR + + +def test_cancel_closes_spans(): + from ag2 import Agent + from ag2.events import ToolCallEvent + from ag2.testing import TestConfig + + started = asyncio.Event() + + async def hang(city: str) -> str: + """Never returns.""" + started.set() + await asyncio.Event().wait() + return "unreachable" + + provider, exporter = memory_provider() + agent = Agent("hanging_bot", config=TestConfig(ToolCallEvent("hang", arguments='{"city": "Paris"}'), "done"), tools=[hang]) + setup(agent, tracer_provider=provider) + + async def run_and_cancel(): + task = asyncio.ensure_future(agent.ask("go")) + await asyncio.wait_for(started.wait(), timeout=10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(run_and_cancel()) + spans = exporter.get_finished_spans() + names = {s.name for s in spans} + assert "invoke_agent hanging_bot" in names + assert "execute_tool hang" in names + assert all(s.end_time is not None for s in spans) + + +# --- PRD J4: middleware short-circuits (documented gap, not captured) -------- + + +def _short_circuit_middlewares(): + from ag2.events import HaltEvent, ModelMessage, ModelResponse + from ag2.middleware import BaseMiddleware + + class AnswerLLMCall(BaseMiddleware): + async def on_llm_call(self, call_next, events, context): + return ModelResponse(ModelMessage("blocked")) + + class AnswerTurn(BaseMiddleware): + async def on_turn(self, call_next, event, context): + return ModelResponse(ModelMessage("blocked")) + + class EmitHalt(BaseMiddleware): + """Sends the HaltEvent a FATAL AlertPolicy alert sends, then carries on.""" + + async def on_llm_call(self, call_next, events, context): + await context.send(HaltEvent(reason="budget exceeded", source="limiter")) + return await call_next(events, context) + + return AnswerLLMCall, AnswerTurn, EmitHalt + + +def test_j4_outer_middleware_answering_the_llm_call_leaves_no_chat_span(): + from ag2 import Agent + from ag2.testing import TestConfig + + answer_llm_call, _, _ = _short_circuit_middlewares() + provider, exporter = memory_provider() + agent = Agent("limited_bot", config=TestConfig("unreachable"), middleware=[answer_llm_call]) + setup(agent, tracer_provider=provider) + assert ask(agent).body == "blocked" + spans = exporter.get_finished_spans() + assert [s.name for s in spans] == ["invoke_agent limited_bot"] + assert spans[0].status.status_code == StatusCode.UNSET + + +def test_j4_outer_middleware_answering_the_turn_leaves_no_span(): + from ag2 import Agent + from ag2.testing import TestConfig + + _, answer_turn, _ = _short_circuit_middlewares() + provider, exporter = memory_provider() + agent = Agent("limited_bot", config=TestConfig("unreachable"), middleware=[answer_turn]) + setup(agent, tracer_provider=provider) + assert ask(agent).body == "blocked" + assert exporter.get_finished_spans() == () + + +def test_j4_ag2_halt_with_agent_assembly_policies_leaves_no_chat_span(): + """Agent(assembly=...) adds AG2's halt check before setup() adds telemetry, + so the halted LLM call is answered outside the chat span.""" + from ag2 import Agent + from ag2.policies.alert import AlertPolicy + from ag2.testing import TestConfig + + _, _, emit_halt = _short_circuit_middlewares() + provider, exporter = memory_provider() + agent = Agent("halted_bot", config=TestConfig("unreachable"), middleware=[emit_halt], assembly=[AlertPolicy()]) + setup(agent, tracer_provider=provider) + assert ask(agent).body == "HALTED: budget exceeded" + spans = exporter.get_finished_spans() + assert [s.name for s in spans] == ["invoke_agent halted_bot"] + assert spans[0].status.status_code == StatusCode.UNSET + + +def test_j4_ag2_halt_with_per_call_policies_gives_a_chat_span_without_reason(): + """Per-call Plugin policies (present in ag2 1.1.2, absent in 1.0.0) put + the halt check inside the telemetry middleware: the halted call gets a + chat span, status UNSET, and no attribute carries the halt reason.""" + from ag2 import Agent + from ag2.policies.alert import AlertPolicy + from ag2.testing import TestConfig + + try: + from ag2.plugin import Plugin + except ImportError: # pragma: no cover - depends on the installed ag2 + Plugin = None + if Plugin is None or not hasattr(Plugin, "add_policy") or "plugins" not in inspect.signature(Agent.ask).parameters: + pytest.skip("installed ag2 has no per-call Plugin policies (Agent.ask(plugins=...))") + + _, _, emit_halt = _short_circuit_middlewares() + provider, exporter = memory_provider() + agent = Agent("halted_bot", config=TestConfig("unreachable"), middleware=[emit_halt]) + setup(agent, tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + assert ask(agent, plugins=[Plugin().add_policy(AlertPolicy())]).body == "HALTED: budget exceeded" + spans = exporter.get_finished_spans() + assert [s.name for s in spans] == ["chat", "invoke_agent halted_bot"] + for span in spans: + assert span.status.status_code == StatusCode.UNSET, span.name + assert not any("budget exceeded" in str(v) for v in attrs(span).values()), span.name + + +# --- AC-04 (partial): traceparent relay across two agents -------------------- + + +def test_traceparent_relay_keeps_one_trace_across_two_agents(): + """Relays the upstream key exactly as ag2/network/client/handlers.py does, + without running the network hub (not exercised here).""" + from ag2 import Agent + from ag2._telemetry_consts import TRACEPARENT_DEP_KEY + from ag2.testing import TestConfig + + provider, exporter = memory_provider() + planner = Agent("planner", config=TestConfig("plan ready")) + worker = Agent("worker", config=TestConfig("work done")) + setup(planner, worker, tracer_provider=provider) + + ask(planner, "plan it") + (planner_span,) = find(exporter.get_finished_spans(), "invoke_agent planner") + ctx = planner_span.context + traceparent = f"00-{ctx.trace_id:032x}-{ctx.span_id:016x}-01" + + ask(worker, "do it", dependencies={TRACEPARENT_DEP_KEY: traceparent}) + (worker_span,) = find(exporter.get_finished_spans(), "invoke_agent worker") + assert worker_span.context.trace_id == ctx.trace_id + assert worker_span.parent is not None and worker_span.parent.span_id == ctx.span_id + + +# --- attach semantics -------------------------------------------------------- + + +def test_setup_is_idempotent_per_agent(): + provider, exporter = memory_provider() + agent = weather_agent() + setup(agent, tracer_provider=provider) + setup(agent, tracer_provider=provider) + assert len(_telemetry_middlewares(agent)) == 1 + chain = provider._active_span_processor._span_processors + assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 + ask(agent) + assert len(find(exporter.get_finished_spans(), "invoke_agent")) == 1 + + +def test_setup_without_provider_registers_once_and_reuses_it(monkeypatch): + """README: setup is idempotent. A second setup() without tracer_provider + reuses the provider, and its one AG2SpanProcessor, from the first call + instead of registering another; a later config still replaces the + installed one.""" + exporter, calls = fake_register(monkeypatch) + planner = weather_agent("planner") + worker = weather_agent("worker") + first = setup(planner, project_name="ag2-app") + second = setup(worker, capture_content=True, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + assert second is first + assert calls == ["ag2-app"] + chain = first._active_span_processor._span_processors + assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 + ask(worker) + spans = exporter.get_finished_spans() + assert find(spans, "invoke_agent worker") + for span in spans: + assert_no_content(attrs(span), span.name) + + +def test_setup_reusing_its_provider_warns_when_project_name_differs(monkeypatch, caplog): + _, calls = fake_register(monkeypatch) + first = setup(project_name="ag2-app") + with caplog.at_level(logging.WARNING, logger="traceai_ag2"): + assert setup(project_name="other-app") is first + assert calls == ["ag2-app"] + assert any("other-app" in r.getMessage() for r in caplog.records), caplog.text + + +def test_setup_registers_again_after_its_provider_is_shut_down(monkeypatch): + """A provider the application shut down is not reused.""" + _, calls = fake_register(monkeypatch) + first = setup(project_name="ag2-app") + first.shutdown() + second = setup(project_name="ag2-app") + assert second is not first + assert calls == ["ag2-app", "ag2-app"] + + +def test_setup_names_middleware_after_each_agent(): + from ag2 import Agent + from ag2.testing import TestConfig + + provider, exporter = memory_provider() + a = Agent("alpha", config=TestConfig("a")) + b = Agent("beta", config=TestConfig("b")) + setup(a, b, tracer_provider=provider) + ask(a) + ask(b) + names = {s.name for s in exporter.get_finished_spans()} + assert {"invoke_agent alpha", "invoke_agent beta"} <= names + + +def test_manual_middleware_with_setup_installing_only_the_processor(): + """README path for agents built after setup: Agent(middleware=[...]) and ask(middleware=[...]).""" + from ag2 import Agent + + provider, exporter = memory_provider() + assert setup(tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True)) is provider + mw = create_telemetry_middleware(tracer_provider=provider, agent_name="late_bot") + agent = Agent("late_bot", config=weather_config(), tools=[get_weather], middleware=[mw]) + ask(agent) + per_call = Agent("per_call_bot", config=weather_config(), tools=[get_weather]) + ask(per_call, middleware=[create_telemetry_middleware(tracer_provider=provider, agent_name="per_call_bot")]) + + spans = exporter.get_finished_spans() + for name in ("invoke_agent late_bot", "invoke_agent per_call_bot"): + (root,) = [s for s in spans if s.name == name] + assert attrs(root)["gen_ai.span.kind"] == "AGENT" + for span in spans: + assert_no_content(attrs(span), span.name) + tools = find(spans, "execute_tool") + assert len(tools) == 2 + assert all(attrs(s)["gen_ai.span.kind"] == "TOOL" for s in tools) + + +def test_setup_does_not_wrap_agent(): + from ag2 import Agent + + before = {name: inspect.getattr_static(Agent, name) for name in ("__init__", "ask", "add_middleware")} + provider, _ = memory_provider() + setup(weather_agent(), tracer_provider=provider) + after = {name: inspect.getattr_static(Agent, name) for name in before} + assert before == after + + +def test_max_tool_result_chars_forwarded_only_when_given(): + from ag2.middleware.builtin.telemetry import TelemetryMiddleware + + if "max_tool_result_chars" not in inspect.signature(TelemetryMiddleware.__init__).parameters: + pytest.skip("installed ag2 predates max_tool_result_chars (added in 1.0.4)") + default = create_telemetry_middleware() + upstream_default = inspect.signature(TelemetryMiddleware.__init__).parameters["max_tool_result_chars"].default + assert default._max_tool_result_chars == upstream_default + assert create_telemetry_middleware(max_tool_result_chars=16)._max_tool_result_chars == 16 + + +# --- local paths, stack traces and raw error text are never exported -------- + + +def _failing_agent(): + from ag2 import Agent + from ag2.events import ModelMessage, ModelResponse, ToolCallEvent + from ag2.testing import TestConfig + from ag2.usage import Usage + + def broken(city: str) -> str: + """Always fails.""" + raise RuntimeError("boom-MARK /Users/someone/secret.txt") + + return Agent( + "broken_bot", + config=TestConfig( + ToolCallEvent("broken", arguments='{"city": "Paris"}', id="call_b1"), + ModelResponse(ModelMessage("failed"), usage=Usage(prompt_tokens=1, completion_tokens=1), model=MODEL), + ), + tools=[broken], + ) + + +@pytest.mark.parametrize("hide", [False, True]) +def test_error_text_paths_and_stack_traces_are_not_exported(hide): + provider, exporter = memory_provider() + agent = _failing_agent() + config = TraceConfig(hide_inputs=hide, hide_outputs=hide) + setup(agent, tracer_provider=provider, config=config) + try: + ask(agent) + except RuntimeError: # the scripted model does not catch the tool error + pass + spans = exporter.get_finished_spans() + (tool,) = find(spans, "execute_tool") + assert tool.status.status_code == StatusCode.ERROR + assert tool.status.description == "RuntimeError" # class name only + for span in spans: + for event in span.events: + assert set(event.attributes or {}) <= {"exception.type"}, event.attributes + blob = repr(dict(span.attributes)) + repr(span.status.description) + repr([e.attributes for e in span.events]) + for leak in ("/Users/", "secret.txt", "Traceback", "site-packages", "boom-MARK"): + assert leak not in blob, (span.name, leak) + + +def test_scrub_text_cuts_tracebacks_and_local_paths_but_keeps_urls(): + from traceai_ag2._processor import scrub_text + + assert "Users" not in scrub_text('File "/Users/me/app.py", line 3') + assert "home" not in scrub_text("open /home/bob/a.txt failed") + assert "bob" not in scrub_text(r"C:\Users\bob\a.py") + assert scrub_text("x Traceback (most recent call last):\n File \"/a/b.py\"\nValueError: q") == "x [stack trace removed]" + assert scrub_text("see https://example.com/home/page and /not/a/known/root") == "see https://example.com/home/page and /not/a/known/root" + + +def test_traceback_in_a_tool_message_is_cut_from_chat_input_and_input_value(): + import json + + from traceai_ag2 import normalize_attributes + + tb = 'Traceback (most recent call last):\n File "/Users/me/venv/x.py", line 1\nRuntimeError: boom' + messages = json.dumps([{"role": "user", "content": "hi"}, {"role": "tool", "content": tb}]) + out = normalize_attributes({"gen_ai.operation.name": "chat", "gen_ai.input.messages": messages}, TraceConfig()) + for key in ("gen_ai.input.messages", "input.value"): + assert "Traceback" not in out[key] and "/Users/" not in out[key], key + assert "[stack trace removed]" in out[key] + assert json.loads(out[key])[0]["content"] == "hi" diff --git a/python/frameworks/ag2/tests/test_usage_totals.py b/python/frameworks/ag2/tests/test_usage_totals.py new file mode 100644 index 000000000..b0f319d89 --- /dev/null +++ b/python/frameworks/ag2/tests/test_usage_totals.py @@ -0,0 +1,251 @@ +"""Trace token totals against AG2's own ``UsageReport``, with real AG2. + +fi-collector promotes ``gen_ai.usage.input_tokens`` / ``output_tokens`` / +``total_tokens`` into its token columns on any span, and Observe sums them over +every span of a trace. So the sum of those keys over a trace must equal the +total AG2 itself reports for the run. The model is AG2's scripted +``ag2.testing.TestConfig``: no network, no API key. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, Iterable, Tuple + +import pytest + +from traceai_ag2 import setup + +from ._support import attrs, emits_record_usage, fake_register, memory_provider + +PROMOTED_INPUT = "gen_ai.usage.input_tokens" +PROMOTED_OUTPUT = "gen_ai.usage.output_tokens" +PROMOTED_TOTAL = "gen_ai.usage.total_tokens" + +needs_record_usage = pytest.mark.skipif( + not emits_record_usage(), + reason="record_usage spans (the only carrier of aggregation/compaction/sub-task rollup spend) start at ag2 1.0.3", +) + + +def _ag2_version() -> Tuple[int, ...]: + from importlib.metadata import version + + parts = [] + for piece in version("ag2").split(".")[:3]: + digits = "".join(ch for ch in piece if ch.isdigit()) + parts.append(int(digits or 0)) + return tuple(parts) + + +def _response(text: str, prompt_tokens: int, completion_tokens: int) -> Any: + from ag2.events import ModelMessage, ModelResponse + from ag2.usage import Usage + + return ModelResponse( + ModelMessage(text), + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + model="fake-model", + ) + + +def _trace_sums(spans: Iterable[Any]) -> Tuple[int, int, int]: + spans = list(spans) + assert len({s.context.trace_id for s in spans}) == 1, "expected one trace" + total_in = sum(attrs(s).get(PROMOTED_INPUT, 0) for s in spans) + total_out = sum(attrs(s).get(PROMOTED_OUTPUT, 0) for s in spans) + total = sum(attrs(s).get(PROMOTED_TOTAL, 0) for s in spans) + return total_in, total_out, total + + +def _run(agent: Any, prompt: str) -> Any: + async def _go() -> Any: + reply = await agent.ask(prompt) + return await reply.usage() + + return asyncio.run(_go()) + + +def _assert_trace_matches_report(spans: Any, report: Any) -> None: + total_in, total_out, total = _trace_sums(spans) + assert (total_in, total_out) == (report.total.prompt_tokens, report.total.completion_tokens) + # AG2 never sets gen_ai.usage.total_tokens; if a later release does, it must + # not be counted twice either. + assert total in (0, report.total.prompt_tokens + report.total.completion_tokens) + + +def _knowledge(*, aggregate: bool = False, compact: bool = False) -> Any: + from ag2.knowledge import MemoryKnowledgeStore + from ag2.knowledge.config import KnowledgeConfig + from ag2.testing import TestConfig + + kwargs: dict = {"store": MemoryKnowledgeStore(), "expose_tool": False, "write_event_log": False} + if aggregate: + from ag2.aggregate import AggregateTrigger, ConversationSummaryAggregate + + kwargs["aggregate"] = ConversationSummaryAggregate(TestConfig(_response("summary", 100, 50))) + kwargs["aggregate_trigger"] = AggregateTrigger(on_end=True) + if compact: + from ag2.compact import CompactTrigger, SummarizeCompact + + kwargs["compact"] = SummarizeCompact(target=2, config=TestConfig(_response("compacted", 300, 30))) + kwargs["compact_trigger"] = CompactTrigger(max_events=2) + return KnowledgeConfig(**kwargs) + + +def _delegating_planner(worker: Any, **planner_kwargs: Any) -> Any: + from ag2 import Agent + from ag2.events import ToolCallEvent + from ag2.testing import TestConfig + from ag2.tools.subagents import subagent_tool + + return Agent( + "planner", + config=TestConfig( + ToolCallEvent("task_worker", arguments='{"objective": "x"}', id="call_task_1"), + _response("ok", 11, 7), + ), + tools=[subagent_tool(worker, description="delegate")], + **planner_kwargs, + ) + + +def _worker() -> Any: + from ag2 import Agent + from ag2.testing import TestConfig + + return Agent("worker", config=TestConfig(_response("done", 40, 4))) + + +# --- R1: memory aggregation calls the model outside on_llm_call --------------- + + +@needs_record_usage +def test_aggregation_usage_counts_once_in_trace_total(): + """aggregate.py calls the client on a throwaway stream: no chat span repeats it.""" + from ag2 import Agent + from ag2.testing import TestConfig + + provider, exporter = memory_provider() + agent = Agent("mem_bot", config=TestConfig(_response("hello", 11, 7)), knowledge=_knowledge(aggregate=True)) + setup(agent, tracer_provider=provider) + report = _run(agent, "hi") + + assert (report.total.prompt_tokens, report.total.completion_tokens) == (111, 57) + spans = exporter.get_finished_spans() + (aggregation,) = [s for s in spans if s.name == "record_usage aggregation"] + assert attrs(aggregation)[PROMOTED_INPUT] == 100 + assert attrs(aggregation)[PROMOTED_OUTPUT] == 50 + _assert_trace_matches_report(spans, report) + + +# --- R2: a sub-task rollup repeats an instrumented worker's chat spans -------- + + +@needs_record_usage +def test_subtask_rollup_keeps_tokens_when_worker_is_not_instrumented(): + """The rollup is the only copy of the worker's spend: it must stay promoted.""" + provider, exporter = memory_provider() + worker = _worker() + planner = _delegating_planner(worker) + setup(planner, tracer_provider=provider) + report = _run(planner, "plan") + + spans = exporter.get_finished_spans() + (rollup,) = [s for s in spans if s.name == "record_usage subtask"] + assert attrs(rollup)["ag2.usage.label"] == "worker" + assert attrs(rollup)[PROMOTED_INPUT] == 40 + assert attrs(rollup)[PROMOTED_OUTPUT] == 4 + _assert_trace_matches_report(spans, report) + + +def test_subtask_rollup_counts_once_when_worker_is_instrumented(): + """setup(planner, worker): the worker's chat spans already carry its tokens.""" + provider, exporter = memory_provider() + worker = _worker() + planner = _delegating_planner(worker) + setup(planner, worker, tracer_provider=provider) + report = _run(planner, "plan") + + spans = exporter.get_finished_spans() + assert [s.name for s in spans if s.name.startswith("invoke_agent")] == ["invoke_agent worker", "invoke_agent planner"] + for rollup in [s for s in spans if s.name == "record_usage subtask"]: + a = attrs(rollup) + assert PROMOTED_INPUT not in a and PROMOTED_OUTPUT not in a + assert a["ag2.usage.input_tokens"] == 40 + assert a["ag2.usage.output_tokens"] == 4 + _assert_trace_matches_report(spans, report) + + +@needs_record_usage +def test_subtask_counts_once_when_planner_and_worker_are_set_up_in_separate_calls(monkeypatch): + """setup(planner) then setup(worker), neither passing tracer_provider. Both + must share one provider and one AG2SpanProcessor; otherwise the planner's + processor never sees the worker's invoke_agent and keeps the rollup's + tokens on top of the worker's own chat spans.""" + exporter, _ = fake_register(monkeypatch) + worker = _worker() + planner = _delegating_planner(worker) + setup(planner) + setup(worker) + report = _run(planner, "plan") + _assert_trace_matches_report(exporter.get_finished_spans(), report) + + +@needs_record_usage +def test_subtask_rollup_for_an_agent_seen_in_another_trace_keeps_tokens(): + """Agent names are tracked per trace: an instrumented worker's earlier, + separate run must not demote a later rollup from an uninstrumented copy.""" + from ag2 import Agent + from ag2.testing import TestConfig + + provider, exporter = memory_provider() + instrumented_namesake = Agent("worker", config=TestConfig(_response("solo", 1, 1))) + setup(instrumented_namesake, tracer_provider=provider) + _run(instrumented_namesake, "warm up") + exporter.clear() + + planner = _delegating_planner(_worker()) + setup(planner, tracer_provider=provider) + report = _run(planner, "plan") + _assert_trace_matches_report(exporter.get_finished_spans(), report) + + +# --- every usage route in one run --------------------------------------------- + + +@needs_record_usage +@pytest.mark.parametrize("instrument_worker", [False, True], ids=["worker-uninstrumented", "worker-instrumented"]) +def test_trace_tokens_equal_usage_report_across_every_usage_route(instrument_worker): + """Main loop (chat + record_usage model_call), compaction, aggregation and a + sub-task rollup in one run: summed promoted input and output tokens equal + AG2's UsageReport. + + Compaction is left out on ag2 < 1.0.4: there ``UsageReport`` reads the + compacted history, so it drops usage events compaction removed (measured on + 1.0.3: report 111 input tokens, trace and actual spend 451). The trace is + right in that case and the report is not, so it is no ground truth. + """ + compact = _ag2_version() >= (1, 0, 4) + provider, exporter = memory_provider() + worker = _worker() + planner = _delegating_planner(worker, knowledge=_knowledge(aggregate=True, compact=compact)) + agents = (planner, worker) if instrument_worker else (planner,) + setup(*agents, tracer_provider=provider) + report = _run(planner, "plan") + + spans = exporter.get_finished_spans() + names = {s.name for s in spans} + expected_routes = ["record_usage model_call", "record_usage aggregation", "record_usage subtask"] + if compact: + expected_routes.append("record_usage compaction") + for expected in expected_routes: + assert expected in names, f"{expected} not emitted; the scenario no longer covers it" + spent_in = 11 + 40 + 100 + (300 if compact else 0) + spent_out = 7 + 4 + 50 + (30 if compact else 0) + assert (report.total.prompt_tokens, report.total.completion_tokens) == (spent_in, spent_out) + _assert_trace_matches_report(spans, report) diff --git a/python/frameworks/ag2/traceai_ag2/__init__.py b/python/frameworks/ag2/traceai_ag2/__init__.py new file mode 100644 index 000000000..cc5fc2bb5 --- /dev/null +++ b/python/frameworks/ag2/traceai_ag2/__init__.py @@ -0,0 +1,40 @@ +"""Future AGI tracing for AG2 1.x (PyPI ``ag2``, import ``ag2``). + +AG2 1.x already emits OpenTelemetry GenAI spans from its own +``TelemetryMiddleware``. This package does not wrap ``Agent``. It: + +* attaches ``TelemetryMiddleware`` (content captured by default, hidden with + ``TraceConfig`` / ``FI_HIDE_*``) + (``setup`` / ``create_telemetry_middleware``), +* registers the Future AGI exporter through ``fi_instrumentation.register()`` + when you do not pass a provider, +* installs :class:`AG2SpanProcessor`, which sets ``gen_ai.span.kind`` and + aliases AG2's cache and thinking usage keys onto the keys Future AGI reads. + +This is not Microsoft AutoGen (``autogen-agentchat``, see ``traceai-autogen``) +and not AG2 Classic (``autogen`` 0.14.x). It never imports ``autogen``. +""" + +from ._processor import ( + AG2_INSTRUMENTATION_SCOPE, + OPERATION_TO_SPAN_KIND, + USAGE_KEY_ALIASES, + AG2SpanProcessor, + normalize_attributes, + span_kind_for, +) +from ._setup import create_telemetry_middleware, install_span_processor, setup +from .version import __version__ + +__all__ = [ + "AG2SpanProcessor", + "AG2_INSTRUMENTATION_SCOPE", + "OPERATION_TO_SPAN_KIND", + "USAGE_KEY_ALIASES", + "create_telemetry_middleware", + "install_span_processor", + "normalize_attributes", + "setup", + "span_kind_for", + "__version__", +] diff --git a/python/frameworks/ag2/traceai_ag2/_processor.py b/python/frameworks/ag2/traceai_ag2/_processor.py new file mode 100644 index 000000000..a29b97cfe --- /dev/null +++ b/python/frameworks/ag2/traceai_ag2/_processor.py @@ -0,0 +1,645 @@ +"""Span processor that normalizes AG2 1.x ``TelemetryMiddleware`` spans. + +AG2 1.x (PyPI ``ag2``, import ``ag2``) already emits OpenTelemetry GenAI spans +from ``ag2.middleware.builtin.telemetry.TelemetryMiddleware``. This processor +does not create spans. On every span from AG2's instrumentation scope it only: + +* sets ``gen_ai.span.kind`` from ``gen_ai.operation.name`` for the operation + strings AG2 emits that the Microsoft Agent Framework table also maps + (``chat``, ``execute_tool``, ``invoke_agent``); +* copies AG2's cache usage keys onto the keys Future AGI reads + (``gen_ai.usage.cache_read_tokens`` / ``cache_write_tokens``) and its thinking + tokens onto ``gen_ai.usage.reasoning.output_tokens``, keeping the originals; +* on ``record_usage`` spans whose tokens a chat span in the same trace already + carries (every ``model_call``; a ``subtask`` rollup whose sub-agent is itself + instrumented in that trace), moves ``gen_ai.usage.input_tokens`` / + ``output_tokens`` / ``total_tokens`` to ``ag2.usage.*`` so the trace total + counts each model call once. ``aggregation`` and ``compaction`` usage keeps + its promoted tokens: AG2 calls the model outside the middleware for those; +* applies ``TraceConfig`` (the traceAI privacy controls). AG2 records message + content as one JSON string per span, so ``hide_inputs`` / + ``hide_input_text`` drop every input-side content key (input messages, + system instructions, tool arguments, human-input prompt), ``hide_outputs`` / + ``hide_output_text`` every output-side one (output messages, tool result, + human-input response), and ``hide_input_messages`` / + ``hide_output_messages`` the message keys only. ``hide_input_images`` and + ``hide_embedding_vectors`` have nothing to act on (AG2 records neither), + and neither do ``hide_llm_invocation_parameters`` (AG2 sets no + ``gen_ai.request.parameters``) and ``base64_image_max_length``; + ``TraceConfig.mask`` still runs on every key; +* copies captured content to ``input.value`` / ``output.value`` (chat + messages, tool arguments and result, human input), which is where Future + AGI reads a span's input and output; content the config hides is not copied. + A single chat completion goes on ``output.value`` as its message object, and + ``invoke_agent`` gets its turn's first chat input and last chat output; +* on ``anthropic`` spans whose tokens count, adds the cache read and write + tokens to ``gen_ai.usage.input_tokens`` (AG2's Anthropic mapper leaves them + out; its OpenAI mapper and the GenAI conventions include them); +* in ``on_start``, copies traceAI context attributes (``using_session``, + ``using_user``, ``using_metadata``, ``using_attributes``, ...) onto the span + without overriding keys AG2 sets, except ``session.id``, where the context + wins. + +Spans from any other instrumentation scope pass through untouched. +""" + +from __future__ import annotations + +import json +import logging +import re +import threading +from collections import OrderedDict +from typing import AbstractSet, Any, Dict, FrozenSet, Mapping, Optional, Set, Tuple + +from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes +from fi_instrumentation.instrumentation.config import TraceConfig +from fi_instrumentation.instrumentation.context_attributes import CONTEXT_ATTRIBUTES, get_attributes_from_context +from opentelemetry.attributes import BoundedAttributes +from opentelemetry.context import Context +from opentelemetry.sdk.trace import Event, ReadableSpan, Span, SpanProcessor +from opentelemetry.trace import Status, StatusCode + +logger = logging.getLogger(__name__) + +# ``ag2/_telemetry_consts.py`` line 42 at ag2 1.1.2 (same value since 1.0.0): +# the scope every AG2 tracer (agent middleware and network hub) reports. +AG2_INSTRUMENTATION_SCOPE = "opentelemetry.instrumentation.ag2" + +# Operation strings read from ``ag2/middleware/builtin/telemetry.py`` at 1.1.2: +# "invoke_agent" line 355 (span "invoke_agent {agent_name}") +# "chat" line 467 (span "chat" / "chat {model}") +# "execute_tool" line 535 (span "execute_tool {tool_name}") +# "await_human_input" line 574 (span "await_human_input {agent_name}") +# The ``record_usage {kind}`` span (line 420) sets no operation name. +# Only strings that the Microsoft Agent Framework table also maps get a kind. +# ``await_human_input`` is not in that table, so it is left without a kind +# rather than guessed. +OPERATION_TO_SPAN_KIND: Mapping[str, str] = { + "chat": FiSpanKindValues.LLM.value, + "execute_tool": FiSpanKindValues.TOOL.value, + "invoke_agent": FiSpanKindValues.AGENT.value, +} + +# AG2 spelling -> the spelling Future AGI reads. AG2 sets the left-hand keys at +# telemetry.py lines 441-446 (record_usage span) and 505-510 (chat span) at +# 1.1.2, only when the usage field is non-zero. Originals are kept. The cache +# targets are the keys Future AGI's attribute registry reads +# (tracer/utils/semantic_conventions.py CACHE_READ_TOKENS / CACHE_WRITE_TOKENS, +# also fi_instrumentation SpanAttributes.GEN_AI_USAGE_CACHE_*_TOKENS); it does +# not read the dotted semconv spellings (gen_ai.usage.cache_read.input_tokens). +_CACHE_READ_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_READ_TOKENS # gen_ai.usage.cache_read_tokens +_CACHE_WRITE_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_WRITE_TOKENS # gen_ai.usage.cache_write_tokens +USAGE_KEY_ALIASES: Mapping[str, str] = { + "gen_ai.usage.cache_creation_input_tokens": _CACHE_WRITE_TOKENS, + "gen_ai.usage.cache_read_input_tokens": _CACHE_READ_TOKENS, + "gen_ai.usage.thinking_tokens": "gen_ai.usage.reasoning.output_tokens", +} + +# Content keys ``TraceConfig.mask`` does not fully cover. AG2 sets the tool +# and human-input keys only when ``capture_content=True`` (telemetry.py lines +# 541, 555, 578, 588 at 1.1.2) and records message content as one JSON string +# per span (``gen_ai.input.messages`` line 477, ``gen_ai.output.messages`` line +# 517), never as the flattened ``gen_ai.input.messages.{i}.message.content`` +# keys that ``mask``'s ``hide_input_text`` / ``hide_output_text`` / +# ``hide_input_images`` rules match. So the text flags drop the whole JSON +# attribute. ``gen_ai.system_instructions`` is not emitted by ag2 1.0.0-1.1.2; +# it is listed so a later release that adds it is still hidden. +_INPUT_MESSAGES = SpanAttributes.GEN_AI_INPUT_MESSAGES # gen_ai.input.messages +_OUTPUT_MESSAGES = SpanAttributes.GEN_AI_OUTPUT_MESSAGES # gen_ai.output.messages +_SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions" +_TOOL_ARGUMENTS = SpanAttributes.GEN_AI_TOOL_CALL_ARGUMENTS # gen_ai.tool.call.arguments +_TOOL_RESULT = SpanAttributes.GEN_AI_TOOL_CALL_RESULT # gen_ai.tool.call.result +_HUMAN_INPUT_PROMPT = "ag2.human_input.prompt" +_HUMAN_INPUT_RESPONSE = "ag2.human_input.response" +_INPUT_MESSAGE_KEYS = (_INPUT_MESSAGES, _SYSTEM_INSTRUCTIONS) +_OUTPUT_MESSAGE_KEYS = (_OUTPUT_MESSAGES,) +_INPUT_CONTENT_KEYS = _INPUT_MESSAGE_KEYS + (_TOOL_ARGUMENTS, _HUMAN_INPUT_PROMPT) +_OUTPUT_CONTENT_KEYS = _OUTPUT_MESSAGE_KEYS + (_TOOL_RESULT, _HUMAN_INPUT_RESPONSE) + +# Future AGI fills a span's input and output only from ``input.value`` / +# ``output.value``; AG2 sets neither. Per operation, the content keys that +# become them. AG2 records no content on ``invoke_agent`` spans. +_IO_SOURCES: Mapping[str, Tuple[str, str]] = { + "chat": (_INPUT_MESSAGES, _OUTPUT_MESSAGES), + "execute_tool": (_TOOL_ARGUMENTS, _TOOL_RESULT), + "await_human_input": (_HUMAN_INPUT_PROMPT, _HUMAN_INPUT_RESPONSE), +} + + +# --- local paths and stack traces are never exported ------------------------ +# +# A failing tool's text is sent back to the model by AG2 and recorded as a +# tool message in the next chat span's input; for an exception that text is a +# full Python traceback with absolute file paths and the user's home +# directory. Span status descriptions and exception events carry +# ``str(exc)`` and ``exception.stacktrace``. None of that is exported in +# either privacy mode: tracebacks are cut, absolute local paths are replaced. +_TRACEBACK_RE = re.compile(r"Traceback \(most recent call last\):.*", re.DOTALL) +_PATH_RE = re.compile( + r"(?|:*?]+|/(?:Users|home|root|private|var|tmp|opt|usr|srv|mnt|app|workspace|etc)/[^\s\"'<>|:*?,;)\]]+)" +) +STACK_TRACE_REMOVED = "[stack trace removed]" +PATH_REMOVED = "[path]" + + +def scrub_text(value: str) -> str: + """Cut a Python traceback and replace absolute local file paths.""" + if "Traceback (most recent call last)" in value: + value = _TRACEBACK_RE.sub(STACK_TRACE_REMOVED, value) + return _PATH_RE.sub(PATH_REMOVED, value) + + +def _scrub_json(value: Any) -> Any: + if isinstance(value, str): + return scrub_text(value) + if isinstance(value, list): + return [_scrub_json(v) for v in value] + if isinstance(value, dict): + return {k: _scrub_json(v) for k, v in value.items()} + return value + + +def _scrub_content_value(value: Any) -> Any: + """Scrub a content attribute: plain text, or AG2's JSON-string messages.""" + if not isinstance(value, str): + return value + try: + parsed = json.loads(value) + except ValueError: + return scrub_text(value) + if isinstance(parsed, (dict, list)): + scrubbed = _scrub_json(parsed) + return value if scrubbed == parsed else json.dumps(scrubbed, ensure_ascii=False) + return scrub_text(value) + + +def _scrub_paths_and_traces(mapped: Dict[str, Any]) -> None: + for key in _INPUT_CONTENT_KEYS + _OUTPUT_CONTENT_KEYS: + if key in mapped: + mapped[key] = _scrub_content_value(mapped[key]) + + +def _sanitize_error_details(span: ReadableSpan) -> None: + """Drop raw exception text from the span status and exception events. + + The status keeps ``ERROR`` and gets the exception class name (from the + exception event AG2 records) as its description instead of ``str(exc)``; + exception events keep ``exception.type`` only. + """ + events = list(span.events or ()) + error_type: Optional[str] = None + new_events = [] + changed = False + for event in events: + attrs = dict(event.attributes or {}) + if event.name == "exception": + error_type = error_type or (str(attrs["exception.type"]) if attrs.get("exception.type") else None) + kept = {k: v for k, v in attrs.items() if k == "exception.type"} + if kept != attrs: + changed = True + event = Event(name=event.name, attributes=kept, timestamp=event.timestamp) + new_events.append(event) + if changed: + span._events = new_events # type: ignore[attr-defined] + status = span.status + if status is not None and status.status_code == StatusCode.ERROR and status.description != error_type: + span._status = Status(StatusCode.ERROR, error_type) # type: ignore[attr-defined] + + +def span_kind_for(attributes: Mapping[str, Any]) -> Optional[str]: + """Return the Future AGI span kind for an AG2 span, or ``None``.""" + operation = attributes.get(SpanAttributes.GEN_AI_OPERATION_NAME) + if not isinstance(operation, str): + return None + return OPERATION_TO_SPAN_KIND.get(operation) + + +def normalize_attributes( + attributes: Mapping[str, Any], + config: Optional[TraceConfig] = None, + *, + instrumented_agents: AbstractSet[str] = frozenset(), +) -> Dict[str, Any]: + """Return a new attribute dict with the kind, aliases and content gate applied. + + Never removes a key AG2 set, except where ``config`` asks to hide content + and where a duplicate usage span's promoted token keys move under + ``ag2.usage.*`` (see ``_demote_duplicate_usage``). ``instrumented_agents`` + are the agent names already seen on ``invoke_agent`` spans of the same + trace; :class:`AG2SpanProcessor` tracks them. + """ + mapped: Dict[str, Any] = dict(attributes) + + if SpanAttributes.GEN_AI_SPAN_KIND not in mapped: + kind = span_kind_for(mapped) + if kind is not None: + mapped[SpanAttributes.GEN_AI_SPAN_KIND] = kind + + for source, target in USAGE_KEY_ALIASES.items(): + if source in mapped and target not in mapped: + mapped[target] = mapped[source] + + _demote_duplicate_usage(mapped, instrumented_agents) + _count_cached_input(mapped) + + _scrub_paths_and_traces(mapped) + hidden = _hidden_content_keys(config) if config is not None else set() + _copy_content_to_io(mapped, hidden) + + if config is not None: + mapped = _apply_trace_config(mapped, config, hidden) + return mapped + + +def _mime_type(value: Any) -> str: + if isinstance(value, str): + try: + if isinstance(json.loads(value), (dict, list)): + return "application/json" + except ValueError: + pass + return "text/plain" + + +def _single_completion(value: Any) -> Any: + """A one-message output list as that message object, else ``value`` unchanged. + + Observe's chat view shows an output object ``{role, content}`` as the + assistant reply, and nothing for a list. + """ + if not isinstance(value, str): + return value + try: + parsed = json.loads(value) + except ValueError: + return value + if isinstance(parsed, list) and len(parsed) == 1 and isinstance(parsed[0], dict): + return json.dumps(parsed[0], ensure_ascii=False) + return value + + +def _copy_content_to_io(mapped: Dict[str, Any], hidden: AbstractSet[str]) -> None: + """Copy captured content to ``input.value`` / ``output.value``. + + Only content AG2 captured and ``hidden`` does not remove is copied, so a + hide flag also keeps the copy off the span. Existing values are kept. + """ + operation = mapped.get(SpanAttributes.GEN_AI_OPERATION_NAME) + sources = _IO_SOURCES.get(operation) # type: ignore[arg-type] + if sources is None: + return + targets = ( + (SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE), + (SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE), + ) + for source, (value_key, mime_key) in zip(sources, targets): + value = mapped.get(source) + if value is None or source in hidden or value_key in mapped: + continue + if operation == "chat" and value_key == SpanAttributes.OUTPUT_VALUE: + value = _single_completion(value) + mapped[value_key] = value + mapped.setdefault(mime_key, _mime_type(value)) + + +# AG2's Anthropic mapper (ag2/config/anthropic/mappers.py normalize_usage at +# 1.1.2) reports prompt_tokens as Anthropic's raw input_tokens, which leaves +# out cache reads and writes; its OpenAI mapper counts cached tokens inside +# prompt_tokens, and so do the GenAI conventions for gen_ai.usage.input_tokens. +_ANTHROPIC = "anthropic" +_CACHED_INPUT_KEYS = ("gen_ai.usage.cache_read_input_tokens", "gen_ai.usage.cache_creation_input_tokens") + + +def _token_count(value: Any) -> Optional[int]: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return int(value) + + +def _count_cached_input(mapped: Dict[str, Any]) -> None: + """Add cached tokens to an ``anthropic`` span's counted input tokens. + + Only spans that still carry ``gen_ai.usage.input_tokens`` (chat spans and + usage spans that are the only copy) change; a duplicate usage span's + ``ag2.usage.*`` keeps AG2's own numbers. The total is recomputed when the + span has one. The trace total then exceeds AG2's ``UsageReport`` by the + cached tokens, which that report leaves out for Anthropic. + """ + if mapped.get(SpanAttributes.GEN_AI_PROVIDER_NAME) != _ANTHROPIC: + return + uncached = _token_count(mapped.get("gen_ai.usage.input_tokens")) + if uncached is None: + return + cached = sum(_token_count(mapped.get(key)) or 0 for key in _CACHED_INPUT_KEYS) + if not cached: + return + mapped["gen_ai.usage.input_tokens"] = uncached + cached + output = _token_count(mapped.get("gen_ai.usage.output_tokens")) + if "gen_ai.usage.total_tokens" in mapped and output is not None: + mapped["gen_ai.usage.total_tokens"] = uncached + cached + output + + +# fi-collector promotes these keys into the token columns on any span +# (fi-collector/pkg/adapter/adapter.go inputTokenKeys/outputTokenKeys/ +# totalTokenKeys), and Observe sums total_tokens over every span of a trace. +_PROMOTED_TOKEN_KEYS = ( + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", + # Not summed today, but a duplicate usage span repeats the chat span's + # cache counts too; keep Future AGI's cache keys on one span per call. + _CACHE_READ_TOKENS, + _CACHE_WRITE_TOKENS, +) +# Usage kinds AG2 emits (``ag2/_telemetry_consts.py`` ATTR_USAGE_KIND at +# 1.1.2) and whether a chat span in the same trace already carries the tokens: +# +# * ``model_call``: yes. ``Agent`` emits it right after the LLM call that +# ``on_llm_call`` wrapped in a chat span (agent.py 1585-1595; telemetry.py +# 437-440 vs 501-504). Always demoted. +# * ``aggregation``: no. ``ag2/aggregate.py`` 97-99 and 193-195 call the model +# client directly on a throwaway ``Context(MemoryStream())`` that never goes +# through ``on_llm_call``; the ``UsageEvent`` (aggregate.py 214-216) is the +# only record of that spend. Never demoted. +# * ``compaction``: no, for the same reason (compact.py 189-205). Never demoted. +# * ``subtask``: a rollup of a sub-agent's calls (tools/subagents/run_task.py +# 63-93, ``label`` = the sub-agent's name). It repeats chat spans only when +# that sub-agent is itself instrumented, which shows as an +# ``invoke_agent {label}`` span that ended earlier in the same trace. Demoted +# only then; otherwise the rollup is the only copy and keeps its tokens. +_ALWAYS_DUPLICATE_USAGE_KINDS = frozenset({"model_call"}) +_SUBTASK_USAGE_KIND = "subtask" +_USAGE_LABEL = "ag2.usage.label" +_AGENT_NAME = "gen_ai.agent.name" +_SESSION_ID = SpanAttributes.SESSION_ID # session.id + + +def _scope_name(span: Any) -> Optional[str]: + return getattr(getattr(span, "instrumentation_scope", None), "name", None) + + +def _span_key(span: Any) -> Tuple[int, int]: + ctx = span.context + return (ctx.trace_id, ctx.span_id) if ctx is not None else (0, id(span)) + + +def _demote_duplicate_usage(mapped: Dict[str, Any], instrumented_agents: AbstractSet[str] = frozenset()) -> None: + """Move a duplicate usage span's promoted token keys under ``ag2.usage.*``. + + ``instrumented_agents`` are the agent names seen on ``invoke_agent`` spans + of the same trace. The values are kept, so nothing AG2 reported is lost, + but the trace total counts each model call once. + """ + if mapped.get("ag2.span.type") != "usage": + return + kind = mapped.get("ag2.usage.kind") + if kind in _ALWAYS_DUPLICATE_USAGE_KINDS: + pass + elif kind == _SUBTASK_USAGE_KIND and mapped.get(_USAGE_LABEL) in instrumented_agents: + pass + else: + return + for key in _PROMOTED_TOKEN_KEYS: + if key in mapped: + value = mapped.pop(key) + mapped.setdefault("ag2.usage." + key.rsplit(".", 1)[1], value) + + +def _hidden_content_keys(config: TraceConfig) -> Set[str]: + """Keys the config hides on AG2 spans, beyond what ``TraceConfig.mask`` does. + + * ``hide_inputs`` / ``hide_input_text``: every input-side text key + (input messages, system instructions, tool arguments, human prompt). + * ``hide_outputs`` / ``hide_output_text``: every output-side text key + (output messages, tool result, human response). + * ``hide_input_messages`` / ``hide_output_messages``: the message keys only. + """ + hidden: Set[str] = set() + if config.hide_inputs or config.hide_input_text: + hidden.update(_INPUT_CONTENT_KEYS) + elif config.hide_input_messages: + hidden.update(_INPUT_MESSAGE_KEYS) + if config.hide_outputs or config.hide_output_text: + hidden.update(_OUTPUT_CONTENT_KEYS) + elif config.hide_output_messages: + hidden.update(_OUTPUT_MESSAGE_KEYS) + return hidden + + +def _apply_trace_config( + attributes: Dict[str, Any], config: TraceConfig, hidden: Optional[AbstractSet[str]] = None +) -> Dict[str, Any]: + if hidden is None: + hidden = _hidden_content_keys(config) + + masked: Dict[str, Any] = {} + for key, value in attributes.items(): + if key in hidden: + continue + value = config.mask(key, value) + if value is None: + continue + masked[key] = value + return masked + + +def _move_context_keys_last(mapped: Dict[str, Any]) -> None: + """Move traceAI context keys (``session.id``, ``user.id``, ...) to the newest end. + + ``on_start`` writes them before AG2 sets any attribute, so they are the + oldest keys on the span, and ``BoundedAttributes`` evicts oldest-first + when the keys this processor adds overflow ``max_span_attributes``. + Moved last, they are evicted after every key AG2 set. + """ + for key in CONTEXT_ATTRIBUTES: + if key in mapped: + mapped[key] = mapped.pop(key) + + +def _bounded_like(previous: Any, attributes: Dict[str, Any]) -> Any: + """Return ``attributes`` in the container the span already used. + + The SDK keeps span attributes in ``BoundedAttributes``, and + ``ReadableSpan.dropped_attributes`` (exported as + ``dropped_attributes_count``) reads its ``dropped`` counter, which a plain + dict would reset to 0. The span's limits and immutability are kept and the + count is carried over. Keys this processor adds count against the same + ``max_span_attributes`` limit: when the span is full the oldest key is + evicted and counted, as the SDK does for ``set_attribute``. + """ + if not isinstance(previous, BoundedAttributes): + return attributes + bounded = BoundedAttributes( + maxlen=previous.maxlen, + attributes=attributes, + immutable=getattr(previous, "_immutable", True), + max_value_len=previous.max_value_len, + ) + bounded.dropped += previous.dropped + return bounded + + +class AG2SpanProcessor(SpanProcessor): + """Normalize AG2 ``TelemetryMiddleware`` spans before export. + + Install it ahead of the exporting processor (``install_span_processor`` + does this) so the exporter sees the added keys. + + It remembers, per trace, the agent names of ``invoke_agent`` spans that + have ended, so a ``record_usage subtask`` rollup for an instrumented + sub-agent is not counted on top of that sub-agent's own chat spans. The + map is bounded to the most recent ``max_tracked_traces`` traces. + """ + + def __init__(self, config: Optional[TraceConfig] = None, *, max_tracked_traces: int = 1024) -> None: + self._config = config if config is not None else TraceConfig() + self._shutdown = False + self._max_tracked_traces = max(1, int(max_tracked_traces)) + self._agents_by_trace: "OrderedDict[int, Set[str]]" = OrderedDict() + # Context session ids of AG2 spans that started but have not ended. + self._max_pending_spans = 16 * self._max_tracked_traces + self._session_by_span: "OrderedDict[Tuple[int, int], Any]" = OrderedDict() + # Per invoke_agent span (keyed by its trace and span id): the first + # chat input and the last chat output of its turn, as + # {"input": (start, value, mime), "output": (end, value, mime)}. + self._agent_io: "OrderedDict[Tuple[int, int], Dict[str, Tuple[int, Any, Any]]]" = OrderedDict() + self._lock = threading.Lock() + + def _note_agent(self, trace_id: int, attributes: Mapping[str, Any]) -> None: + if attributes.get(SpanAttributes.GEN_AI_OPERATION_NAME) != "invoke_agent": + return + name = attributes.get(_AGENT_NAME) + if not isinstance(name, str) or not name: + return + with self._lock: + names = self._agents_by_trace.get(trace_id) + if names is None: + names = self._agents_by_trace[trace_id] = set() + while len(self._agents_by_trace) > self._max_tracked_traces: + self._agents_by_trace.popitem(last=False) + else: + self._agents_by_trace.move_to_end(trace_id) + names.add(name) + + def _agents_in(self, trace_id: int) -> FrozenSet[str]: + with self._lock: + return frozenset(self._agents_by_trace.get(trace_id, ())) + + def update_config(self, config: TraceConfig) -> bool: + """Replace the ``TraceConfig`` applied to later spans; ``True`` if it changed.""" + with self._lock: + if config == self._config: + return False + self._config = config + return True + + def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None: + """Copy traceAI context attributes (``using_session``, ``using_user``, + ``using_metadata``, ``using_attributes``, ...) onto AG2 spans. + + A key AG2 already set is not overridden, and a key AG2 sets later wins, + except ``session.id``: a session from the context replaces one from + ``span_attributes`` (re-applied in ``on_end``). + """ + if self._shutdown or _scope_name(span) != AG2_INSTRUMENTATION_SCOPE: + return + try: + context_attributes = dict(get_attributes_from_context()) + if not context_attributes: + return + present = span.attributes or {} + for key, value in context_attributes.items(): + if key == _SESSION_ID or key not in present: + span.set_attribute(key, value) + session = context_attributes.get(_SESSION_ID) + if session is not None: + with self._lock: + self._session_by_span[_span_key(span)] = session + while len(self._session_by_span) > self._max_pending_spans: + self._session_by_span.popitem(last=False) + except Exception: # pragma: no cover - defensive + logger.debug("traceai-ag2: could not read context attributes for %r", getattr(span, "name", None), exc_info=True) + + def on_end(self, span: ReadableSpan) -> None: + if self._shutdown: + return + if _scope_name(span) != AG2_INSTRUMENTATION_SCOPE: + return + try: + with self._lock: + session = self._session_by_span.pop(_span_key(span), None) + current = dict(span.attributes or {}) + trace_id = span.context.trace_id if span.context is not None else 0 + self._note_agent(trace_id, current) + agents = self._agents_in(trace_id) if current.get("ag2.usage.kind") == _SUBTASK_USAGE_KIND else frozenset() + mapped = normalize_attributes(current, self._config, instrumented_agents=agents) + operation = mapped.get(SpanAttributes.GEN_AI_OPERATION_NAME) + if operation == "chat": + self._note_chat_io(span, mapped) + elif operation == "invoke_agent": + self._apply_agent_io(span, mapped) + if session is not None: + mapped[_SESSION_ID] = session + _move_context_keys_last(mapped) + _sanitize_error_details(span) + if mapped != current: + # ``ReadableSpan.attributes`` is a read-only view over + # ``_attributes``; every later processor and exporter in the + # chain receives this same object. + span._attributes = _bounded_like(span._attributes, mapped) # type: ignore[attr-defined] + except Exception: # pragma: no cover - defensive + logger.debug("traceai-ag2: could not normalize span %r", getattr(span, "name", None), exc_info=True) + + def _note_chat_io(self, span: ReadableSpan, mapped: Mapping[str, Any]) -> None: + """Remember a chat span's input / output for its parent ``invoke_agent``. + + AG2 records no content on ``invoke_agent``. The turn's first chat + input and last chat output make it show the question and the answer. + Only content that is on the chat span (captured, not hidden) is used. + """ + parent = span.parent + if parent is None or parent.is_remote: + return + found_input = mapped.get(SpanAttributes.INPUT_VALUE) + found_output = mapped.get(SpanAttributes.OUTPUT_VALUE) + if found_input is None and found_output is None: + return + key = (parent.trace_id, parent.span_id) + with self._lock: + entry = self._agent_io.get(key) + if entry is None: + entry = self._agent_io[key] = {} + while len(self._agent_io) > self._max_pending_spans: + self._agent_io.popitem(last=False) + if found_input is not None and span.start_time is not None: + first = entry.get("input") + if first is None or span.start_time < first[0]: + entry["input"] = (span.start_time, found_input, mapped.get(SpanAttributes.INPUT_MIME_TYPE)) + if found_output is not None and span.end_time is not None: + last = entry.get("output") + if last is None or span.end_time >= last[0]: + entry["output"] = (span.end_time, found_output, mapped.get(SpanAttributes.OUTPUT_MIME_TYPE)) + + def _apply_agent_io(self, span: ReadableSpan, mapped: Dict[str, Any]) -> None: + with self._lock: + entry = self._agent_io.pop(_span_key(span), None) + if not entry: + return + for side, value_key, mime_key in ( + ("input", SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE), + ("output", SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE), + ): + found = entry.get(side) + if found is None or value_key in mapped: + continue + _, value, mime = found + mapped[value_key] = value + if mime is not None: + mapped.setdefault(mime_key, mime) + + def shutdown(self) -> None: + self._shutdown = True + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True diff --git a/python/frameworks/ag2/traceai_ag2/_setup.py b/python/frameworks/ag2/traceai_ag2/_setup.py new file mode 100644 index 000000000..6c722d2bf --- /dev/null +++ b/python/frameworks/ag2/traceai_ag2/_setup.py @@ -0,0 +1,315 @@ +"""One-call setup for AG2 1.x tracing to Future AGI. + +AG2 has no process-wide middleware registry: ``TelemetryMiddleware`` is +attached per agent (``Agent(middleware=[...])``, ``Agent.add_middleware``) or +per call (``agent.ask(..., middleware=[...])``). ``setup`` therefore attaches +the middleware to the agents you pass it through AG2's public +``add_middleware``. It does not patch or wrap ``Agent``. +""" + +from __future__ import annotations + +import contextlib +import functools +import logging +import threading +from typing import Any, Dict, Iterable, Optional + +from fi_instrumentation.instrumentation.config import TraceConfig +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import ConcurrentMultiSpanProcessor, TracerProvider + +from ._processor import AG2SpanProcessor + +logger = logging.getLogger(__name__) + +# Set on the wrapper ``_keep_processor_first`` installs, so it wraps once. +_KEEP_FIRST_MARKER = "_traceai_ag2_keeps_processor_first" + + +class _Unset: + def __repr__(self) -> str: # pragma: no cover - cosmetic + return "" + + +_UNSET: Any = _Unset() + + +def create_telemetry_middleware( + *, + tracer_provider: Optional[TracerProvider] = None, + capture_content: bool = True, + agent_name: Optional[str] = None, + provider_name: Optional[str] = None, + model_name: Optional[str] = None, + span_attributes: Optional[Dict[str, str]] = None, + max_tool_result_chars: Any = _UNSET, +) -> Any: + """Build AG2's own ``TelemetryMiddleware``. + + Content is captured by default, like every traceAI instrumentor, and is + hidden with ``TraceConfig`` / the ``FI_HIDE_*`` variables (applied by + ``AG2SpanProcessor``). ``capture_content`` is AG2's own constructor + argument, passed through unchanged; it is not a second privacy setting. + + ``provider_name`` is never invented. Leave it ``None`` and AG2 fills it + from the model response when the client reports one. + + ``max_tool_result_chars`` is forwarded only when given; the keyword exists + from ag2 1.0.4. + + Use this for agents created after ``setup`` or for per-call middleware:: + + agent = Agent("bot", config=cfg, middleware=[create_telemetry_middleware( + tracer_provider=trace_provider, agent_name="bot")]) + """ + from ag2.middleware.builtin.telemetry import TelemetryMiddleware + + kwargs: Dict[str, Any] = { + "tracer_provider": tracer_provider, + "capture_content": bool(capture_content), + "agent_name": agent_name, + "provider_name": provider_name, + "model_name": model_name, + "span_attributes": dict(span_attributes) if span_attributes else None, + } + if max_tool_result_chars is not _UNSET: + kwargs["max_tool_result_chars"] = max_tool_result_chars + return TelemetryMiddleware(**kwargs) + + +def install_span_processor( + tracer_provider: Optional[TracerProvider] = None, + *, + config: Optional[TraceConfig] = None, +) -> bool: + """Install :class:`AG2SpanProcessor` ahead of the provider's exporters. + + Returns ``True`` when installed, ``False`` when the provider has no span + processor chain or already has an ``AG2SpanProcessor``. In the latter case + an explicit ``config`` replaces the installed processor's ``TraceConfig`` + (the most recent call wins) and a warning is logged when it changes; + ``config=None`` leaves the installed config as it is. + + The processor is prepended to the active multi-processor rather than added + with ``add_span_processor``: the provider returned by + ``fi_instrumentation.register()`` treats its exporter as a replaceable + default and shuts it down on the first ``add_span_processor`` call, and the + processor must run before the exporter reads the span. + + That same ``add_span_processor`` call, made later by the application, + would also shut down and drop this processor. So the provider instance's + ``add_span_processor`` is wrapped once: the processor is taken out of the + chain for the call (fi's reset never shuts it down) and put back first + afterwards. Spans ending during that call are not normalized. + + A provider whose active processor is a ``ConcurrentMultiSpanProcessor`` + runs every processor's ``on_end`` in parallel, so an exporter can read a + span before it is normalized; a warning is logged for it. + """ + provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + active = getattr(provider, "_active_span_processor", None) + if active is None or not hasattr(active, "_span_processors"): + logger.warning( + "traceai-ag2: tracer provider %s has no span processor chain; pass the " + "provider returned by fi_instrumentation.register() as tracer_provider=.", + type(provider).__name__, + ) + return False + + processor = AG2SpanProcessor(config=config) + with _chain_lock(active): + existing = tuple(active._span_processors) + installed = next((p for p in existing if isinstance(p, AG2SpanProcessor)), None) + if installed is None: + active._span_processors = (processor,) + existing + _keep_processor_first(provider, active) + if installed is None: + if isinstance(active, ConcurrentMultiSpanProcessor): + logger.warning( + "traceai-ag2: the tracer provider runs span processors concurrently " + "(ConcurrentMultiSpanProcessor), so exporters may read AG2 spans before " + "AG2SpanProcessor normalizes them. Use the default synchronous processor." + ) + return True + if config is not None and installed.update_config(config): + logger.warning( + "traceai-ag2: an AG2SpanProcessor was already installed on this provider; " + "its TraceConfig is now replaced by the one from the latest call: %r", + config, + ) + return False + + +def _chain_lock(active: Any) -> Any: + lock = getattr(active, "_lock", None) + # SDK multi-processors always carry a lock. + return lock if lock is not None else contextlib.nullcontext() # pragma: no branch + + +def _keep_processor_first(provider: Any, active: Any) -> None: + """Wrap ``provider.add_span_processor`` so :class:`AG2SpanProcessor` stays first. + + fi's ``TracerProvider.add_span_processor`` shuts down and clears the whole + chain on its first call after ``register()`` (``fi_instrumentation/otel.py`` + ``add_span_processor``: ``_default_processor``). The wrapper lifts the AG2 + processor out of the chain for the call, so it is neither shut down nor + dropped, then puts it back in front of whatever the call left. + """ + original = getattr(provider, "add_span_processor", None) + if original is None or getattr(original, _KEEP_FIRST_MARKER, False): + return + + @functools.wraps(original) + def add_span_processor(*args: Any, **kwargs: Any) -> Any: + with _chain_lock(active): + processors = tuple(active._span_processors) + ours = tuple(p for p in processors if isinstance(p, AG2SpanProcessor)) + if ours: + active._span_processors = tuple(p for p in processors if not isinstance(p, AG2SpanProcessor)) + try: + return original(*args, **kwargs) + finally: + if ours: + with _chain_lock(active): + rest = tuple(p for p in active._span_processors if not isinstance(p, AG2SpanProcessor)) + active._span_processors = ours[:1] + rest + + setattr(add_span_processor, _KEEP_FIRST_MARKER, True) + try: + provider.add_span_processor = add_span_processor + except (AttributeError, TypeError): # pragma: no cover - providers with __slots__ + logger.warning( + "traceai-ag2: cannot guard %s.add_span_processor; add span processors " + "before calling setup(), or AG2 spans may reach exporters un-normalized.", + type(provider).__name__, + ) + + +def _has_telemetry_middleware(agent: Any) -> bool: + from ag2.middleware.builtin.telemetry import TelemetryMiddleware + + # ``Agent.middleware`` (public, ag2 >= 1.0.2) yields entries whose + # ``.middleware`` is the factory; ag2 1.0.0/1.0.1 only have the + # ``_middleware`` list declared on ``PluginTarget``. + entries: Iterable[Any] + try: + entries = [getattr(e, "middleware", e) for e in agent.middleware] + except Exception: + entries = list(getattr(agent, "_middleware", ()) or ()) + return any(isinstance(m, TelemetryMiddleware) for m in entries) + + +# The provider ``setup()`` registered itself (no ``tracer_provider`` given). +# Later calls without one reuse it: a second ``register()`` would give the +# later agents their own provider and ``AG2SpanProcessor``, which never sees +# the earlier agents' spans, so a sub-task rollup would be counted twice. +_registered_provider: Optional[TracerProvider] = None +_register_lock = threading.Lock() + + +def _is_shut_down(provider: Any) -> bool: + """``True`` once ``provider.shutdown()`` reached its ``AG2SpanProcessor``.""" + active = getattr(provider, "_active_span_processor", None) + processors = getattr(active, "_span_processors", ()) + return any(isinstance(p, AG2SpanProcessor) and p._shutdown for p in processors) + + +def _registered_or_new_provider(project_name: Optional[str]) -> TracerProvider: + global _registered_provider + with _register_lock: + provider = _registered_provider + if provider is not None and not _is_shut_down(provider): + registered_name = getattr(getattr(provider, "resource", None), "attributes", {}).get("project_name") + if project_name is not None and project_name != registered_name: + logger.warning( + "traceai-ag2: project_name %r is ignored; setup() reuses the provider it " + "registered earlier for project %r. Pass tracer_provider= to send these " + "agents elsewhere.", + project_name, + registered_name, + ) + return provider + + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + provider = register(project_type=ProjectType.OBSERVE, project_name=project_name) + _registered_provider = provider + return provider + + +def setup( + *agents: Any, + tracer_provider: Optional[TracerProvider] = None, + project_name: Optional[str] = None, + capture_content: bool = True, + provider_name: Optional[str] = None, + model_name: Optional[str] = None, + span_attributes: Optional[Dict[str, str]] = None, + max_tool_result_chars: Any = _UNSET, + config: Optional[TraceConfig] = None, +) -> TracerProvider: + """Register the Future AGI exporter and attach AG2's ``TelemetryMiddleware``. + + Args: + *agents: AG2 ``Agent`` objects to instrument. Each gets its own + ``TelemetryMiddleware`` named after ``agent.name`` through + ``agent.add_middleware``. An agent that already has a + ``TelemetryMiddleware`` is skipped, so calling ``setup`` twice does + not duplicate spans. + tracer_provider: Provider from ``fi_instrumentation.register(...)``. + When omitted, the first such ``setup`` call runs + ``register(project_type=OBSERVE, project_name=project_name)`` + (``FI_PROJECT_NAME``, ``FI_API_KEY``, ``FI_SECRET_KEY`` and + ``FI_BASE_URL`` apply) and later calls without one reuse that + provider until it is shut down, so every agent shares one + provider and one ``AG2SpanProcessor``. + project_name: Used only when ``setup`` registers a provider; ignored, + with a warning, when it differs from the one already registered. + capture_content: Forwarded to ``TelemetryMiddleware`` (AG2's own + argument; default ``True``, the same as upstream). Hide content + with ``config=TraceConfig(...)`` or the ``FI_HIDE_*`` variables. + provider_name, model_name, span_attributes, max_tool_result_chars: + Forwarded to ``TelemetryMiddleware``. ``span_attributes`` is one + way to stamp ``session.id`` on every span (AG2 emits no session + id); traceAI's ``using_session`` context takes precedence. + config: ``TraceConfig`` applied by the span processor. Defaults to ``TraceConfig()`` (reads ``FI_HIDE_*``). + One processor serves the whole provider: when it is already + installed, an explicit ``config`` replaces its config (the most + recent call wins, with a warning when it changes) and + ``config=None`` keeps the installed one. + + Returns: + The tracer provider in use. Short scripts call ``force_flush()`` on it + before exiting. + """ + if tracer_provider is None: + tracer_provider = _registered_or_new_provider(project_name) + elif project_name is not None: + logger.warning( + "traceai-ag2: project_name is ignored when tracer_provider is passed; " + "set it on fi_instrumentation.register()." + ) + + install_span_processor(tracer_provider, config=config) + + for agent in agents: + if _has_telemetry_middleware(agent): + logger.info( + "traceai-ag2: agent %r already has a TelemetryMiddleware; not adding another.", + getattr(agent, "name", agent), + ) + continue + agent.add_middleware( + create_telemetry_middleware( + tracer_provider=tracer_provider, + capture_content=capture_content, + agent_name=getattr(agent, "name", None), + provider_name=provider_name, + model_name=model_name, + span_attributes=span_attributes, + max_tool_result_chars=max_tool_result_chars, + ) + ) + return tracer_provider diff --git a/python/frameworks/ag2/traceai_ag2/version.py b/python/frameworks/ag2/traceai_ag2/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/ag2/traceai_ag2/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out