diff --git a/python/frameworks/ag2-classic/CHANGELOG.md b/python/frameworks/ag2-classic/CHANGELOG.md new file mode 100644 index 000000000..5de6346b3 --- /dev/null +++ b/python/frameworks/ag2-classic/CHANGELOG.md @@ -0,0 +1,18 @@ +## [0.1.0] - 2026-10-04 +### Features +- New package for AG2 Classic (`autogen` 0.14.x). `setup()` enables upstream + `autogen.opentelemetry` (`instrument_llm_wrapper`, `instrument_agent`, + `instrument_pattern`) on the Future AGI tracer provider, without creating a + second provider or exporter. +- `AG2ClassicSpanProcessor` maps `ag2.span.type` to `gen_ai.span.kind`, copies + the outermost `gen_ai.conversation.id` to `session.id`, adds + `gen_ai.cost.total` (only when upstream's cost is above 0; AG2 reports 0 + for models it cannot price) and `gen_ai.usage.total_tokens` on LLM spans, + and sets ERROR status from upstream `error.type`. +- Content follows the traceAI `TraceConfig` convention: captured by default, + hidden with `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` (and the message/text + variants) or `setup(config=TraceConfig(...))`. Upstream message, tool, + human-input and code-output keys of a hidden side are removed before export. + `capture_content=True/False` is an optional override. +- Version guard: rejects non-0.14 `autogen`, Microsoft AutoGen, `ag2` 1.x and + `pyautogen` layouts with an error that names the matching package. diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md new file mode 100644 index 000000000..34daf61de --- /dev/null +++ b/python/frameworks/ag2-classic/README.md @@ -0,0 +1,232 @@ +# traceAI-ag2-classic + +Send AG2 Classic traces to Future AGI. AG2 Classic is the `autogen` distribution +at 0.14.x, imported as `autogen`, from `ag2ai/ag2classic`. It ships its own +OpenTelemetry instrumentation, `autogen.opentelemetry`. This package turns that +on against a Future AGI tracer provider and maps its span attributes to Future +AGI conventions. Alpha. Validated against fixtures and a loopback fake model, and re-verified +2026-10-16 end to end against a local Future AGI stack with a real model +(autogen 0.14.1 and 0.14.0). + +| You installed | Import | Package | +|---|---|---| +| `autogen` 0.14.x | `autogen` | `traceAI-ag2-classic` (this package) | +| `ag2` 1.x | `ag2` | `traceAI-ag2` | +| `autogen-agentchat` | `autogen_agentchat` (Microsoft AutoGen) | `traceAI-autogen` | +| `pyautogen` | older alias | not supported | + +Supported: `autogen` 0.14.0 and 0.14.1. Upstream `ag2ai/ag2classic` at +`86fa5ccb3f9424cee87edb4a9ea58f191ea958ba`, Apache-2.0, copyright AG2ai. +`autogen` is a normal dependency (`>=0.14.0,<0.15`); it is never vendored. + +Note on 0.14.0: PyPI `autogen==0.14.0` is a small alias whose only requirement +is `ag2==0.14.0`, and the `autogen` module is shipped by that `ag2` 0.14.0 +distribution. Its `autogen/opentelemetry` tree is identical to 0.14.1's, so the +version guard accepts exactly that pairing. `autogen==0.14.1` ships the module +itself and installs no `ag2`. `ag2` 1.x ships only an `ag2` module. + +## Install + +```bash +pip install traceAI-ag2-classic "autogen[openai]" +``` + +`traceAI-ag2-classic` pins `autogen` to `>=0.14.0,<0.15`. Plain `autogen` +installs no model client; the `openai` extra adds the OpenAI client the quick +start below needs (without it, creating an agent with an `llm_config` raises +`ImportError: ... 'openai' is not installed`). For another provider, use its +extra instead or as well, for example `"autogen[anthropic]"` or +`"autogen[gemini]"`. + +`autogen[openai,tracing]` also works. The `tracing` extra adds +`opentelemetry-sdk` and the OTLP gRPC exporter; this package never constructs +that exporter. + +```bash +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export FI_PROJECT_NAME="my-chatbot" +``` + +## Set up + +```python +from autogen import ConversableAgent +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_ag2_classic import setup + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="my-chatbot", +) + +assistant = ConversableAgent("assistant", llm_config={"config_list": [{"model": "gpt-4o-mini"}]}) +user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER") + +tracing = setup(tracer_provider=trace_provider, agents=[assistant, user]) +user.initiate_chat(assistant, message="Hello", max_turns=2) + +trace_provider.force_flush() # short scripts: flush before exit +``` + +Upstream instruments agents per instance, so pass every agent you want traced, +or call `tracing.instrument_agent(agent)` for agents created later. Pass a +`GroupChatManager` and its group chat members and speaker selection are +instrumented too. Pass group patterns with `patterns=[...]` or +`tracing.instrument_pattern(pattern)`. `tracing.instrument_a2a_server(server)` +is available only when upstream exports it (the `autogen[a2a]` extra); it is +never called unless you ask. + +`setup()`: + +1. Checks that `autogen` is AG2 Classic 0.14.x. It raises + `AG2ClassicCompatibilityError`, naming the packages above, when `autogen` + is missing (for example only `ag2` 1.x or Microsoft AutoGen is installed), + when the version is not 0.14.x, when `autogen.opentelemetry` is missing, or + when the `autogen` module comes from `pyautogen` or from `ag2` outside the + 0.14.0 alias pairing. +2. Puts `AG2ClassicSpanProcessor` first on the provider you pass, ahead of the + Future AGI exporter. +3. Calls upstream `instrument_llm_wrapper(capture_messages=...)` (on unless every content side is hidden), + then `instrument_agent` and `instrument_pattern` for what you pass. If one + of those raises (for example a non-agent in `agents=`), `setup()` restores + `OpenAIWrapper.create` and re-raises. + +It never creates a provider, never sets the global provider, and never adds an +exporter. One provider, the one `register()` returned. HTTP to the collector is +the default; pass `transport=Transport.GRPC` (from `fi_instrumentation.otel`) to `register()` for gRPC. Do not +also install the `[tracing]` extra's gRPC exporter as a global provider, or +spans are duplicated. + +`tracing.uninstrument()` restores `OpenAIWrapper.create`, so no new LLM spans. +If another library wrapped `OpenAIWrapper.create` after `setup()`, that wrapper +is left in place and a WARNING is logged; call `uninstrument()` again once it +is removed. +Agents stay instrumented; upstream has no per-agent undo. The processor stays on +the provider, so spans from those agents still have content removed and +aggregate usage moved off non-LLM spans. It is shared by every handle on that +provider and shuts down with `trace_provider.shutdown()`. + +## Conformance + +Every row was read from `autogen/opentelemetry/instrumentators/` at 0.14.1. +LLM, agent, tool, conversation, group-chat speaker selection, pattern and +`initiate_chats` spans are exercised by real `autogen` runs against a loopback +fake model in `tests/`; code execution, human input and remote-agent spans are +covered by mapping unit tests only. + +| Field | What you get | When it is omitted | +|---|---|---| +| Span kind | `gen_ai.span.kind` from upstream `ag2.span.type`: `llm` → `LLM`; `agent` → `AGENT`; `tool`, `code_execution` → `TOOL`; `conversation`, `multi_conversation`, `speaker_selection`, `human_input` → `CHAIN`; `handoff` → `CHAIN` (declared upstream as TODO, never emitted at 0.14.x). `ag2.span.type` and `gen_ai.operation.name` are kept. | The `a2a-execution` span from `instrument_a2a_server` has no `ag2.span.type` and gets no kind. | +| Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | +| Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans carry upstream's aggregate input/output tokens for the whole chat. Future AGI sums tokens over every span in a trace, so those move to `ag2.usage.input_tokens` / `ag2.usage.output_tokens`; each LLM call counts once. | +| Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table) when it is above 0. AG2 reports 0 for a model its price table lacks (it logs "The cost will be 0"); that 0 is not copied, so Future AGI prices the call from its tokens and its own model table. Upstream also reports `response.cost` on a cache hit (`oai/client.py:1230-1241` returns the cached response with its stored cost), so with `cache_seed` set a cached reply carries the cost of the original call and Future AGI over-reports cost for cached responses; its token counts are the original call's too. Not fixed here; upstream behaviour. | Conversation spans carry upstream's aggregate `gen_ai.usage.cost`; it moves to `ag2.usage.cost`, so no `gen_ai.usage.*` or `gen_ai.cost.total` key is left on any non-LLM span. | +| Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id. Only `conversation` spans from `initiate_chat` / `a_initiate_chat` keep their chat id in `gen_ai.conversation.id`; every other AG2 span (LLM, agent, tool, `initiate_chats`, resumed conversations) carries the session id there, as Future AGI's own tracer does. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | +| User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | +| Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) hidden by `FI_HIDE_INPUTS` (arguments) / `FI_HIDE_OUTPUTS` (result). | +| Errors | Exceptions keep OTel's ERROR status. A model call that raises keeps ERROR, and upstream sets `error.type` to the exception class (for example `BadRequestError`, `llm_wrapper.py:97-101`); the span has no usage or cost. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | +| Retrieval | Not emitted upstream. | Always. | + +## Limits + +- To tell a nested chat from the outermost one, `AG2ClassicSpanProcessor` + tracks every live span on the provider (AG2 or not) and frees each entry + when its span ends. Tracking caps at 10000 live spans. The nested-session + split (an inner chat getting its own `session.id`) returns only if that many + spans leak (start and never end) or run concurrently on the provider; spans + started past the cap are not tracked. Upstream ends every span it starts + (`with tracer.start_as_current_span(...)`), so a leak needs other code on + the same provider. + +## Privacy + +Content follows the traceAI `TraceConfig` convention, like every traceAI +instrumentor: message bodies, tool arguments and results are exported by +default, and hidden with the standard variables (no package-specific switch): + +```bash +export FI_HIDE_INPUTS=true # input side: messages in, tool arguments, human prompt +export FI_HIDE_OUTPUTS=true # output side: messages out, tool results, code output, chat summaries +# also honoured: FI_HIDE_INPUT_MESSAGES, FI_HIDE_OUTPUT_MESSAGES, FI_HIDE_INPUT_TEXT, FI_HIDE_OUTPUT_TEXT +``` + +or in code, `setup(..., config=TraceConfig(hide_inputs=True, hide_outputs=True))` +(`from fi_instrumentation import TraceConfig`). `setup(..., capture_content=False)` +hides both sides and `capture_content=True` keeps both; either overrides `config`. + +Upstream has a content flag only on `instrument_llm_wrapper` +(`capture_messages`). Its agent, conversation, tool, human-input and +code-execution spans always record message bodies, tool arguments and results, +the human prompt and answer, code output and chat summaries. This package +removes the keys of a hidden side before export. Input side: +`gen_ai.input.messages`, `gen_ai.tool.call.arguments`, `ag2.human_input.prompt` +and `input.value`. Output side: `gen_ai.output.messages`, +`gen_ai.tool.call.result`, `ag2.human_input.response`, +`ag2.code_execution.output`, `ag2.chats.summaries` and `output.value`. That is +stricter than upstream. Kept content is lifted into `input.value` / +`output.value`. Note that a conversation's output messages carry the whole chat +history, so the prompt also appears there unless `FI_HIDE_OUTPUTS` is set. + +Never exported in either mode: local paths, user names, host names, e-mail +addresses and stack traces (spans carry the exception class in `error.type` and +OpenTelemetry's status; checked on a real run). Exception messages from the +model provider or your tool do appear in the span status text, as with any +OpenTelemetry instrumentation; hide them by not putting secrets in exception +messages. + +The filtering lives in `AG2ClassicSpanProcessor`, which must run before any +exporting processor. `setup()` keeps it first on the provider: + +- `trace_provider.add_span_processor(...)` after `setup()`: Future AGI's + provider shuts down and removes every processor on the first such call after + `register()` (`fi_instrumentation/otel.py:336-339`), this one included. + `setup()` wraps that method on your provider instance, so after the call this + processor is put back first and re-enabled. Your processor sees filtered + spans. +- `tracing.uninstrument()` leaves the processor in place, because agents + instrumented earlier keep emitting spans. +- Editing `_active_span_processor._span_processors` yourself, or shutting the + processor down without going through `trace_provider.add_span_processor`, + bypasses this; call `setup()` again to put the processor back first and + re-enable it. +- A provider built with the SDK's `ConcurrentMultiSpanProcessor` runs every + processor's `on_end` in parallel, so an exporter can send a span before its + content is removed. `setup()` logs a WARNING for it; use the default + `SynchronousMultiSpanProcessor`, which `register()` uses. + +## Attribute inventory (autogen 0.14.1) + +Paths are relative to `autogen/opentelemetry/`. Scope name +`opentelemetry.instrumentation.ag2`, schema +`https://opentelemetry.io/schemas/1.11.0` (`consts.py:25-26`). + +| Span (name) | `ag2.span.type` | Keys emitted | Source | +|---|---|---|---| +| `chat {model}` | `llm` | `gen_ai.operation.name=chat`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.agent.name`, `gen_ai.request.{temperature,max_tokens,top_p,frequency_penalty,presence_penalty}`, `gen_ai.response.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.response.finish_reasons`, `gen_ai.usage.cost`, `error.type`; messages only if `capture_messages` | `instrumentators/llm_wrapper.py:74-135,153`, `utils.py:213-222` | +| `invoke_agent {agent}` | `agent` | `gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.input.messages`, `gen_ai.output.messages` (always) | `instrumentators/agent_instrumentators/reply.py:36-58,75-95` | +| `invoke_agent {agent}` (remote) | `agent` | `gen_ai.agent.remote`, `server.address` | `instrumentators/agent_instrumentators/remote.py:43-48` | +| `conversation {agent}` | `conversation` | `gen_ai.operation.name=conversation`, `gen_ai.agent.name`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.conversation.max_turns`, `gen_ai.input.messages`, `gen_ai.conversation.id`, `gen_ai.conversation.turns`, `gen_ai.output.messages`, `gen_ai.usage.cost`, `gen_ai.response.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | `instrumentators/agent_instrumentators/chat.py:33-82,100-147` | +| `conversation {agent}` (resume, run_chat) | `conversation` | `gen_ai.conversation.resumed`; run_chat: messages | `instrumentators/agent_instrumentators/chat.py:167-170,191-237` | +| `agent.initiate_chats` / `initiate_chats` | `multi_conversation` | `gen_ai.operation.name=initiate_chats`, `ag2.chats.{count,mode,recipients,ids,summaries,prerequisites}` | `instrumentators/agent_instrumentators/chat.py:253-318` | +| `execute_tool {name}` | `tool` | `gen_ai.operation.name=execute_tool`, `gen_ai.tool.name`, `gen_ai.tool.type`, `gen_ai.tool.call.id`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, `error.type=ExecutionError` | `instrumentators/agent_instrumentators/tool.py:26-47,65-86` | +| `execute_code {agent}` | `code_execution` | `gen_ai.operation.name=execute_code`, `gen_ai.agent.name`, `ag2.code_execution.exit_code`, `ag2.code_execution.output`, `error.type=CodeExecutionError` | `instrumentators/agent_instrumentators/code.py:43-68` | +| `await_human_input {agent}` | `human_input` | `gen_ai.operation.name=await_human_input`, `gen_ai.agent.name`, `ag2.human_input.prompt`, `ag2.human_input.response` | `instrumentators/agent_instrumentators/human_input.py:24-32,48-55` | +| `speaker_selection` | `speaker_selection` | `gen_ai.operation.name=speaker_selection`, `ag2.speaker_selection.candidates`, `ag2.speaker_selection.selected` | `instrumentators/pattern.py:163-176,193-206` | +| `a2a-execution` | none | none | `instrumentators/a2a.py:61` | + +## Troubleshooting + +- No traces: `register()` needs `project_name` or `FI_PROJECT_NAME`; the + collector drops a batch whose resource has no `project_name`. +- No LLM spans after a second `setup()` on another provider: upstream patches + `OpenAIWrapper.create` once and keeps the first provider + (`llm_wrapper.py:64-65`). Call `uninstrument()` on the first handle first. +- Calling `trace_provider.add_span_processor(...)` after `register()` drops + Future AGI's default exporter; that is `fi_instrumentation` behaviour. + `setup()` does not use it, and puts this package's processor back first + when you call it (see Privacy). +- `setup()` raises and names three packages: you have Microsoft AutoGen, `ag2` + 1.x, or `pyautogen` instead of `autogen` 0.14.x. +- Content missing: check `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` (and the `_MESSAGES` / `_TEXT` variants) and any `TraceConfig` you pass to `setup()`. +- Exporter errors are logged and never fail the agent. diff --git a/python/frameworks/ag2-classic/examples/basic_chat.py b/python/frameworks/ag2-classic/examples/basic_chat.py new file mode 100644 index 000000000..48a1ad614 --- /dev/null +++ b/python/frameworks/ag2-classic/examples/basic_chat.py @@ -0,0 +1,51 @@ +"""Two-agent AG2 Classic chat with a tool, traced to Future AGI. + +Install: pip install traceAI-ag2-classic "autogen[openai]" +(plain autogen has no OpenAI client; the openai extra adds it). + +Requires FI_API_KEY, FI_SECRET_KEY and OPENAI_API_KEY in the environment. +This example calls a real model; the package tests use a loopback fake instead. +""" + +import os + +from autogen import ConversableAgent +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +from traceai_ag2_classic import setup + + +def get_weather(city: str) -> str: + """Return the weather for a city.""" + return f"sunny in {city}" + + +def main() -> None: + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name=os.getenv("FI_PROJECT_NAME", "ag2-classic-example"), + ) + + llm_config = {"config_list": [{"model": "gpt-4o-mini", "api_key": os.environ["OPENAI_API_KEY"]}]} + assistant = ConversableAgent( + "assistant", + llm_config=llm_config, + human_input_mode="NEVER", + is_termination_msg=lambda m: "TERMINATE" in (m.get("content") or ""), + system_message="Use the weather tool, answer, then say TERMINATE.", + ) + user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=3) + assistant.register_for_llm(description="Get the weather")(get_weather) + user.register_for_execution()(get_weather) + + # Content (messages, tool arguments/results) is exported unless FI_HIDE_INPUTS / + # FI_HIDE_OUTPUTS are set (traceAI TraceConfig convention). + setup(tracer_provider=trace_provider, agents=[assistant, user]) + + user.initiate_chat(assistant, message="What is the weather in Paris?", max_turns=3) + trace_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/ag2-classic/pyproject.toml b/python/frameworks/ag2-classic/pyproject.toml new file mode 100644 index 000000000..3da055998 --- /dev/null +++ b/python/frameworks/ag2-classic/pyproject.toml @@ -0,0 +1,47 @@ +[tool.poetry] +name = "traceAI-ag2-classic" +version = "0.1.0" +description = "Future AGI tracing for AG2 Classic (autogen 0.14.x) through its built-in autogen.opentelemetry" +authors = ["Future AGI "] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://github.com/future-agi/traceAI" +repository = "https://github.com/future-agi/traceAI" +keywords = ["opentelemetry", "ag2", "autogen", "ag2-classic", "tracing", "observability", "agents"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Monitoring", +] +packages = [ + { include = "traceai_ag2_classic" }, +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +# AG2 Classic: the `autogen` distribution only. Never vendored or bundled. +# No dependency on `ag2` 1.x, `pyautogen` or `autogen-agentchat`. +autogen = ">=0.14.0,<0.15" +fi-instrumentation-otel = ">=1.1.0" +opentelemetry-api = ">=1.29.0" +opentelemetry-sdk = ">=1.29.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=8.0.0" +openai = ">=2.44.0,<3.0" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_functions = ["test_*"] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/ag2-classic/tests/_contract_app.py b/python/frameworks/ag2-classic/tests/_contract_app.py new file mode 100644 index 000000000..0134f8094 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/_contract_app.py @@ -0,0 +1,62 @@ +"""Contract app: real AG2 Classic + real OTel SDK + real Future AGI OTLP/HTTP exporter. + +Run as a subprocess by ``test_contract_harness.py``. The parent points +``FI_BASE_URL`` at a loopback harness ``Receiver``; ``register()`` then exports +OTLP/HTTP protobuf to ``{FI_BASE_URL}/tracer/v1/traces``. The model is a +loopback fake. Usage: ``python _contract_app.py off|on``. +""" + +from __future__ import annotations + +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +if HERE not in sys.path: + sys.path.insert(0, HERE) + + +def main(argv: list) -> int: + capture = len(argv) > 1 and argv[1] == "on" + + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + from opentelemetry import trace as trace_api + + from _fake_openai import FakeOpenAI + from _scenarios import TOOL_SECRET_CITY, broken_tool_chat, failing_llm_chat, group_chat, two_agent_tool_chat + from traceai_ag2_classic import setup + + global_before = type(trace_api.get_tracer_provider()).__name__ + provider = register( + project_type=ProjectType.OBSERVE, + project_name="ag2-classic-contract", + verbose=False, + ) + tracing = setup(tracer_provider=provider, capture_content=capture) + + out = {} + with FakeOpenAI(tool_arguments={"city": TOOL_SECRET_CITY}) as fake: + chat = two_agent_tool_chat(tracing, fake) + out["chat_id"] = str(chat.chat_id) + broken_tool_chat(tracing, fake) + group = group_chat(tracing, fake) + out["group_chat_id"] = str(group.chat_id) + # The model rejects this one (HTTP 400): an LLM span that ends in an exception. + out["llm_error"] = failing_llm_chat(tracing, fake) + out["llm_requests"] = len(fake.requests) # successful model calls only + out["llm_failed_requests"] = len(fake.failed_requests) + + flushed = provider.force_flush(timeout_millis=10000) + out["flushed"] = bool(flushed) + out["global_provider_before"] = global_before + out["global_provider_after"] = type(trace_api.get_tracer_provider()).__name__ + out["processors"] = [type(p).__name__ for p in provider._active_span_processor._span_processors] + provider.shutdown() + print("CONTRACT_RESULT " + json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/frameworks/ag2-classic/tests/_fake_openai.py b/python/frameworks/ag2-classic/tests/_fake_openai.py new file mode 100644 index 000000000..13037653c --- /dev/null +++ b/python/frameworks/ag2-classic/tests/_fake_openai.py @@ -0,0 +1,162 @@ +"""A loopback-only, OpenAI-compatible fake chat completions server for tests. + +No network and no vendor API: the server binds to 127.0.0.1 on a free port and +answers ``POST /v1/chat/completions`` with deterministic replies chosen from the +request body: + +* a request whose messages contain ``LLM_FAIL_TRIGGER`` gets HTTP 400 (the + OpenAI client raises ``BadRequestError`` and does not retry); it is recorded + in ``failed_requests``, not ``requests``; +* a group-chat speaker-selection prompt gets ``speaker_reply`` (an agent name); +* a request whose last message is a tool result gets ``final_reply``; +* a request that offers tools gets one tool call to the first offered tool; +* anything else gets ``text_reply``. + +Every successful reply carries ``usage`` so token attributes can be asserted. +""" + +from __future__ import annotations + +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional + +PROMPT_TOKENS = 11 +COMPLETION_TOKENS = 7 +FAKE_MODEL = "gpt-4o-mini" +LLM_FAIL_TRIGGER = "FAKE-LLM-REJECT" + + +class FakeOpenAI: + def __init__( + self, + *, + text_reply: str = "Hello from the fake model. TERMINATE", + final_reply: str = "Tool says sunny. TERMINATE", + speaker_reply: str = "critic", + tool_arguments: Optional[Dict[str, Any]] = None, + ) -> None: + self.text_reply = text_reply + self.final_reply = final_reply + self.speaker_reply = speaker_reply + self.tool_arguments = tool_arguments if tool_arguments is not None else {"city": "Paris"} + self.requests: List[Dict[str, Any]] = [] + self.failed_requests: List[Dict[str, Any]] = [] + self._lock = threading.Lock() + self._counter = 0 + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + if not self.path.rstrip("/").endswith("/chat/completions"): + self.send_error(HTTPStatus.NOT_FOUND) + return + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8") or "{}") + status = HTTPStatus.OK + if owner._rejects(body): + status = HTTPStatus.BAD_REQUEST + payload = { + "error": { + "message": "fake model rejected the request", + "type": "invalid_request_error", + "param": None, + "code": "fake_rejected", + } + } + else: + payload = owner._reply(body) + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = "http://127.0.0.1:{0}/v1".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + # Context manager ------------------------------------------------------- + + def __enter__(self) -> "FakeOpenAI": + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + # Helpers --------------------------------------------------------------- + + def llm_config(self, **extra: Any) -> Dict[str, Any]: + entry = { + "api_type": "openai", + "model": FAKE_MODEL, + "api_key": "sk-placeholder-not-a-real-key", + "base_url": self.base_url, + } + entry.update(extra) + return {"config_list": [entry], "cache_seed": None} + + def _rejects(self, body: Dict[str, Any]) -> bool: + messages = body.get("messages") or [] + if not any(LLM_FAIL_TRIGGER in str(m.get("content") or "") for m in messages if isinstance(m, dict)): + return False + with self._lock: + self.failed_requests.append(body) + return True + + def _reply(self, body: Dict[str, Any]) -> Dict[str, Any]: + with self._lock: + self.requests.append(body) + self._counter += 1 + n = self._counter + messages = body.get("messages") or [] + joined = " ".join(str(m.get("content") or "") for m in messages if isinstance(m, dict)) + last = messages[-1] if messages else {} + + message: Dict[str, Any] + finish = "stop" + if "select the next role" in joined or "Then select the next role" in joined: + message = {"role": "assistant", "content": self.speaker_reply} + elif isinstance(last, dict) and last.get("role") == "tool": + message = {"role": "assistant", "content": self.final_reply} + elif body.get("tools"): + tool = body["tools"][0]["function"]["name"] + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_fake_{0}".format(n), + "type": "function", + "function": {"name": tool, "arguments": json.dumps(self.tool_arguments)}, + } + ], + } + finish = "tool_calls" + else: + message = {"role": "assistant", "content": self.text_reply} + + return { + "id": "chatcmpl-fake-{0}".format(n), + "object": "chat.completion", + "created": 1700000000, + "model": body.get("model") or FAKE_MODEL, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + }, + } diff --git a/python/frameworks/ag2-classic/tests/_scenarios.py b/python/frameworks/ag2-classic/tests/_scenarios.py new file mode 100644 index 000000000..e3e3aa7fd --- /dev/null +++ b/python/frameworks/ag2-classic/tests/_scenarios.py @@ -0,0 +1,114 @@ +"""AG2 Classic scenarios shared by the in-process tests and the contract app. + +Every scenario talks to a :class:`FakeOpenAI` on 127.0.0.1. No vendor API. +""" + +from __future__ import annotations + +from typing import Any, Dict + +SECRET_PROMPT = "PRIVATE-PROMPT-7f3a: what is the weather in Paris?" +TOOL_SECRET_CITY = "PRIVATE-CITY-91c2" + + +def _terminates(message: Dict[str, Any]) -> bool: + return "TERMINATE" in str(message.get("content") or "") + + +def get_weather(city: str) -> str: + """Return the weather for a city.""" + return "sunny in {0}".format(city) + + +def broken_tool(city: str) -> str: + """A tool that always fails.""" + raise RuntimeError("tool exploded for {0}".format(city)) + + +def two_agent_tool_chat(tracing: Any, fake: Any, *, message: str = SECRET_PROMPT) -> Any: + """user -> assistant, the assistant calls ``get_weather`` once, then stops.""" + from autogen import ConversableAgent + + assistant = ConversableAgent( + "assistant", + llm_config=fake.llm_config(), + human_input_mode="NEVER", + is_termination_msg=_terminates, + ) + user = ConversableAgent( + "user", + llm_config=False, + human_input_mode="NEVER", + max_consecutive_auto_reply=3, + is_termination_msg=_terminates, + ) + assistant.register_for_llm(description="Get the weather")(get_weather) + user.register_for_execution()(get_weather) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + return user.initiate_chat(assistant, message=message, max_turns=3, silent=True) + + +def broken_tool_chat(tracing: Any, fake: Any) -> Any: + """The assistant calls a tool that raises; upstream catches it.""" + from autogen import ConversableAgent + + assistant = ConversableAgent( + "assistant_b", + llm_config=fake.llm_config(), + human_input_mode="NEVER", + is_termination_msg=_terminates, + ) + user = ConversableAgent( + "user_b", + llm_config=False, + human_input_mode="NEVER", + max_consecutive_auto_reply=2, + is_termination_msg=_terminates, + ) + assistant.register_for_llm(description="Always fails")(broken_tool) + user.register_for_execution()(broken_tool) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + return user.initiate_chat(assistant, message="call the broken tool", max_turns=2, silent=True) + + +def failing_llm_chat(tracing: Any, fake: Any) -> str: + """The model endpoint rejects the request (HTTP 400); the error reaches the caller. + + The prompt also carries ``SECRET_PROMPT`` so the error path is checked for + content leaks. Returns the exception class name the caller saw. + """ + from autogen import ConversableAgent + + from _fake_openai import LLM_FAIL_TRIGGER + + assistant = ConversableAgent("assistant_f", llm_config=fake.llm_config(), human_input_mode="NEVER") + user = ConversableAgent("user_f", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=1) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + try: + user.initiate_chat( + assistant, message="{0} {1}".format(LLM_FAIL_TRIGGER, SECRET_PROMPT), max_turns=1, silent=True + ) + except Exception as exc: # the caller sees the model error + return type(exc).__name__ + raise AssertionError("the fake model was expected to reject the request") + + +def group_chat(tracing: Any, fake: Any, *, message: str = "Write a haiku about tracing") -> Any: + """writer -> GroupChatManager with LLM ("auto") speaker selection.""" + from autogen import ConversableAgent + from autogen.agentchat.groupchat import GroupChat, GroupChatManager + + writer = ConversableAgent("writer", llm_config=fake.llm_config(), human_input_mode="NEVER") + critic = ConversableAgent("critic", llm_config=fake.llm_config(), human_input_mode="NEVER") + groupchat = GroupChat( + agents=[writer, critic], + messages=[], + max_round=2, + speaker_selection_method="auto", + ) + manager = GroupChatManager(groupchat=groupchat, llm_config=fake.llm_config()) + tracing.instrument_agent(manager) + return writer.initiate_chat(manager, message=message, max_turns=1, silent=True) diff --git a/python/frameworks/ag2-classic/tests/test_contract_harness.py b/python/frameworks/ag2-classic/tests/test_contract_harness.py new file mode 100644 index 000000000..d48bd3e37 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_contract_harness.py @@ -0,0 +1,256 @@ +"""Shared-harness contract test for traceAI-ag2-classic. + +Runs ``_contract_app.py`` in a subprocess with the real ``autogen`` 0.14.x, +the real OpenTelemetry SDK and Future AGI's real OTLP/HTTP exporter, pointed at +a loopback harness ``Receiver`` through ``FI_BASE_URL`` (exporter path +``/tracer/v1/traces``). The model is a loopback fake; no network, no vendor +API, placeholder keys only. + +The Receiver drops resource attributes and headers; those are asserted in +``test_setup.py``. +""" + +from __future__ import annotations + +import base64 +import json +import os +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +HERE = Path(__file__).resolve().parent +PACKAGE_DIR = HERE.parent +PYTHON_DIR = PACKAGE_DIR.parent.parent +sys.path.insert(0, str(PYTHON_DIR / "tests")) + +from harness import Receiver, run # noqa: E402 + +from _scenarios import SECRET_PROMPT, TOOL_SECRET_CITY # noqa: E402 + +CONTENT_KEYS = ( + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + "ag2.human_input.prompt", + "ag2.human_input.response", + "ag2.code_execution.output", + "ag2.chats.summaries", + "input.value", + "output.value", +) + + +def _value(any_value: Dict[str, Any]) -> Any: + if "stringValue" in any_value: + return any_value["stringValue"] + if "intValue" in any_value: + return int(any_value["intValue"]) + if "doubleValue" in any_value: + return float(any_value["doubleValue"]) + if "boolValue" in any_value: + return bool(any_value["boolValue"]) + return any_value + + +def _flatten(span: Dict[str, Any]) -> Dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _run_app(receiver: Receiver, mode: str) -> Dict[str, Any]: + env = dict(os.environ) + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": "placeholder-api-key", + "FI_SECRET_KEY": "placeholder-secret-key", + "OPENAI_API_KEY": "sk-placeholder-not-a-real-key", + "PYTHONPATH": os.pathsep.join( + [str(PACKAGE_DIR), str(PYTHON_DIR), str(HERE), env.get("PYTHONPATH", "")] + ), + "AUTOGEN_USE_DOCKER": "False", + } + ) + env.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) + env.pop("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", None) + result = run([sys.executable, str(HERE / "_contract_app.py"), mode], env=env, stdin=None, timeout=180) + stdout = result.stdout.decode("utf-8", "replace") + stderr = result.stderr.decode("utf-8", "replace") + assert not result.timed_out, "contract app timed out\nSTDERR:\n" + stderr[-4000:] + assert result.returncode == 0, "contract app failed\nSTDERR:\n" + stderr[-4000:] + lines = [line for line in stdout.splitlines() if line.startswith("CONTRACT_RESULT ")] + assert lines, "no CONTRACT_RESULT line\nSTDOUT:\n" + stdout[-2000:] + return json.loads(lines[-1][len("CONTRACT_RESULT ") :]) + + +def _by_name(spans: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: + grouped: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for span in spans: + grouped[span["name"]].append(span) + return grouped + + +def test_ag2_classic_contract_content_off_round_trip(): + with Receiver() as receiver: + app = _run_app(receiver, "off") + spans = receiver.spans() + + assert app["flushed"] is True + # One provider: setup() never touched the global provider and added no exporter. + assert app["global_provider_before"] == app["global_provider_after"] == "ProxyTracerProvider" + assert app["processors"] == ["AG2ClassicSpanProcessor", "BatchSpanProcessor"] + assert app["llm_requests"] >= 4 + + assert spans, "receiver got no spans" + names = _by_name(spans) + for expected in ( + "conversation user", + "invoke_agent assistant", + "invoke_agent user", + "chat gpt-4o-mini", + "execute_tool get_weather", + "execute_tool broken_tool", + "speaker_selection", + "conversation writer", + "conversation checking_agent", + ): + assert expected in names, "missing span {0!r}; got {1}".format(expected, sorted(names)) + + # Span kind: upstream ag2.span.type preserved, gen_ai.span.kind added. + expected_kind = { + "chat gpt-4o-mini": ("llm", "LLM"), + "invoke_agent assistant": ("agent", "AGENT"), + "invoke_agent user": ("agent", "AGENT"), + "execute_tool get_weather": ("tool", "TOOL"), + "conversation user": ("conversation", "CHAIN"), + "speaker_selection": ("speaker_selection", "CHAIN"), + } + for name, (span_type, kind) in expected_kind.items(): + for span in names[name]: + attrs = _flatten(span) + assert attrs["ag2.span.type"] == span_type, name + assert attrs["gen_ai.span.kind"] == kind, name + + # Model, provider, usage and cost on LLM spans (the one rejected call is + # asserted below). + for span in names["chat gpt-4o-mini"]: + if span.get("status", {}).get("code") == "STATUS_CODE_ERROR": + continue + attrs = _flatten(span) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.request.model"] == "gpt-4o-mini" + assert attrs["gen_ai.response.model"] == "gpt-4o-mini" + assert attrs["gen_ai.usage.input_tokens"] == 11 + assert attrs["gen_ai.usage.output_tokens"] == 7 + assert attrs["gen_ai.usage.total_tokens"] == 18 + assert attrs["gen_ai.cost.total"] > 0 + assert "gen_ai.agent.name" in attrs + + # Tool name and call id. + tool = _flatten(names["execute_tool get_weather"][0]) + assert tool["gen_ai.tool.name"] == "get_weather" + assert tool["gen_ai.tool.type"] == "function" + assert tool["gen_ai.tool.call.id"].startswith("call_fake_") + + # Session: outermost conversation only. + conversation = _flatten(names["conversation user"][0]) + assert conversation["gen_ai.conversation.id"] == app["chat_id"] + assert conversation["session.id"] == app["chat_id"] + group_root = _flatten(names["conversation writer"][0]) + assert group_root["session.id"] == app["group_chat_id"] + for nested in names["conversation checking_agent"]: + nested_attrs = _flatten(nested) + assert "gen_ai.conversation.id" in nested_attrs + assert "session.id" not in nested_attrs + + # AC-07: a tool that raises is caught upstream; status is promoted to ERROR. + broken = names["execute_tool broken_tool"][0] + assert _flatten(broken)["error.type"] == "ExecutionError" + assert broken["status"].get("code") == "STATUS_CODE_ERROR" + assert names["execute_tool get_weather"][0].get("status", {}).get("code") != "STATUS_CODE_ERROR" + + # A model call that raises: upstream sets error.type and re-raises + # (llm_wrapper.py:97-101). Status stays ERROR through Future AGI's + # exporting processor, which turns only UNSET into OK. + assert app["llm_error"] == "BadRequestError" + assert app["llm_failed_requests"] == 1 + llm_spans = [s for s in spans if _flatten(s).get("gen_ai.span.kind") == "LLM"] + failed_llm = [s for s in llm_spans if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] + assert len(failed_llm) == 1 + failed_attrs = _flatten(failed_llm[0]) + assert failed_attrs["error.type"] == "BadRequestError" + assert failed_attrs["ag2.span.type"] == "llm" + for key in ("gen_ai.usage.input_tokens", "gen_ai.usage.total_tokens", "gen_ai.cost.total"): + assert key not in failed_attrs, key + ok_llm = [s for s in llm_spans if s not in failed_llm] + for span in ok_llm: + assert span.get("status", {}).get("code") != "STATUS_CODE_ERROR", span["name"] + + # AC-04: every span of the group chat run shares one trace id. + group_trace = names["conversation writer"][0]["traceId"] + for name in ("speaker_selection", "conversation checking_agent", "invoke_agent critic"): + for span in names[name]: + assert span["traceId"] == group_trace, name + assert names["conversation user"][0]["traceId"] != group_trace + assert len(base64.b64decode(group_trace)) == 16 + + # AC-06: content is absent when capture is off. + for span in spans: + attrs = _flatten(span) + for key in CONTENT_KEYS: + assert key not in attrs, "{0} leaked on {1}".format(key, span["name"]) + blob = json.dumps(attrs) + assert SECRET_PROMPT not in blob, span["name"] + assert TOOL_SECRET_CITY not in blob, span["name"] + + # Usage and cost live on LLM spans only. fi-collector promotes token and + # cost keys on any span and Observe sums them over a trace, so an + # aggregate on a conversation/agent span would count every call twice. + for span in spans: + attrs = _flatten(span) + if attrs.get("gen_ai.span.kind") == "LLM": + continue + leaked = [ + key + for key in attrs + if key.startswith("gen_ai.usage.") or key in ("gen_ai.cost.total", "llm.cost.total") + ] + assert not leaked, "usage/cost keys {0} on non-LLM span {1}".format(leaked, span["name"]) + # Each successful model call is one LLM span with 11 + 7 = 18 tokens; the + # failed call carries none. + assert len(ok_llm) == app["llm_requests"] + per_trace: Dict[str, List[int]] = defaultdict(lambda: [0, 0]) + for span in spans: + tokens = _flatten(span).get("gen_ai.usage.total_tokens", 0) + per_trace[span["traceId"]][0] += tokens + for span in ok_llm: + per_trace[span["traceId"]][1] += 1 + for trace_id, (total, calls) in per_trace.items(): + assert total == calls * 18, (trace_id, total, calls) + assert sum(total for total, _calls in per_trace.values()) == app["llm_requests"] * 18 + + +def test_ag2_classic_contract_content_on_round_trip(): + with Receiver() as receiver: + _run_app(receiver, "on") + spans = receiver.spans() + + names = _by_name(spans) + conversation = _flatten(names["conversation user"][0]) + assert SECRET_PROMPT in conversation["gen_ai.input.messages"] + assert SECRET_PROMPT in conversation["input.value"] + assert conversation["input.mime_type"] == "application/json" + + tool = _flatten(names["execute_tool get_weather"][0]) + assert json.loads(tool["input.value"]) == {"city": TOOL_SECRET_CITY} + assert tool["output.value"] == "sunny in {0}".format(TOOL_SECRET_CITY) + + # capture_content=True is forwarded to upstream instrument_llm_wrapper(capture_messages=True). + llm = _flatten(names["chat gpt-4o-mini"][0]) + assert "gen_ai.input.messages" in llm + assert "gen_ai.output.messages" in llm diff --git a/python/frameworks/ag2-classic/tests/test_guard.py b/python/frameworks/ag2-classic/tests/test_guard.py new file mode 100644 index 000000000..774056fd7 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_guard.py @@ -0,0 +1,135 @@ +"""AC-05: version guard with fake module layouts (no other package installed).""" + +from __future__ import annotations + +import types + +import pytest + +from traceai_ag2_classic import AG2ClassicCompatibilityError, check_autogen_classic + + +def _layout( + *, + owners=("autogen",), + versions=None, + version="0.14.1", + has_otel=True, + importable=True, + agentchat=False, + ag2_module=False, +): + versions = dict(versions or {"autogen": "0.14.1"}) + modules = {} + if importable: + modules["autogen"] = types.ModuleType("autogen") + if version is not None: + vm = types.ModuleType("autogen.version") + setattr(vm, "__version__", version) + modules["autogen.version"] = vm + + def import_module(name): + if name in modules: + return modules[name] + raise ImportError("No module named {0!r}".format(name)) + + def find_spec(name): + if name == "autogen.opentelemetry": + return object() if (importable and has_otel) else None + if name == "autogen_agentchat": + return object() if agentchat else None + if name == "ag2": + return object() if ag2_module else None + return None + + return { + "distributions_for": lambda top: list(owners) if top == "autogen" else [], + "distribution_version": versions.get, + "import_module": import_module, + "find_spec": find_spec, + } + + +def _assert_names_three_packages(error): + text = str(error) + for name in ("`autogen` 0.14.x", "`ag2` 1.x", "`autogen-agentchat`"): + assert name in text + assert "pyautogen" in text + + +@pytest.mark.parametrize("version", ["0.14.0", "0.14.1", "0.14"]) +def test_supported_versions_pass(version): + assert check_autogen_classic(**_layout(version=version, versions={"autogen": version})) == version + + +@pytest.mark.parametrize("version", ["0.13.9", "0.15.0", "0.140.0", "1.1.2", "0.2.35"]) +def test_unsupported_versions_raise(version): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(version=version, versions={"autogen": version})) + _assert_names_three_packages(info.value) + assert version in str(info.value) + + +def test_autogen_0_14_0_alias_layout_backed_by_ag2_0_14_0_passes(): + # PyPI autogen==0.14.0 requires ag2==0.14.0, which ships the `autogen` module. + layout = _layout(owners=("ag2",), versions={"autogen": "0.14.0", "ag2": "0.14.0"}, version="0.14.0") + assert check_autogen_classic(**layout) == "0.14.0" + + +@pytest.mark.parametrize( + "versions", + [ + {"ag2": "0.14.0"}, # ag2 installed directly, no `autogen` distribution + {"ag2": "0.9.0", "autogen": "0.9.0"}, # older ag2 alias line + {"ag2": "1.1.2", "autogen": "0.14.1"}, # mismatched pairing + ], +) +def test_autogen_module_owned_by_ag2_outside_the_alias_layout_raises(versions): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=("ag2",), versions=versions)) + assert "installed by `ag2`" in str(info.value) + _assert_names_three_packages(info.value) + + +@pytest.mark.parametrize("owners", [("pyautogen",), ("autogen", "pyautogen"), ("PyAutoGen",)]) +def test_pyautogen_owner_raises(owners): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=owners)) + assert "pyautogen" in info.value.reason + _assert_names_three_packages(info.value) + + +def test_missing_opentelemetry_module_raises(): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(has_otel=False)) + assert "autogen.opentelemetry" in str(info.value) + _assert_names_three_packages(info.value) + + +def test_microsoft_autogen_layout_raises(): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=(), importable=False, agentchat=True)) + assert "autogen_agentchat" in str(info.value) + _assert_names_three_packages(info.value) + + +def test_ag2_1x_only_layout_raises(): + # ag2 1.x ships `ag2`, not `autogen`, so `import autogen` fails. + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=(), importable=False, ag2_module=True)) + assert "`ag2` (AG2 1.x, `import ag2`) is installed" in str(info.value) + _assert_names_three_packages(info.value) + + +def test_missing_version_module_raises(): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(version=None)) + assert "autogen.version" in str(info.value) + + +def test_installed_autogen_passes_guard(): + pytest.importorskip("autogen.opentelemetry") + from autogen.version import __version__ + + assert check_autogen_classic() == __version__ + assert __version__.startswith("0.14.") diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py new file mode 100644 index 000000000..242ba4d29 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -0,0 +1,429 @@ +"""Unit tests for the AG2 Classic attribute mapping (no autogen run needed).""" + +from __future__ import annotations + +import json + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_ag2_classic import ( + AG2_SCOPE, + CONTENT_KEYS, + KIND_BY_SPAN_TYPE, + AG2ClassicSpanProcessor, + kind_for_span_type, + map_ag2_attributes, +) + + +# AC-02: kind map from the upstream SpanType enum --------------------------------- + + +EXPECTED_KINDS = { + "conversation": "CHAIN", + "multi_conversation": "CHAIN", + "agent": "AGENT", + "llm": "LLM", + "tool": "TOOL", + "handoff": "CHAIN", + "speaker_selection": "CHAIN", + "human_input": "CHAIN", + "code_execution": "TOOL", +} + + +def test_kind_map_matches_architecture_table(): + assert KIND_BY_SPAN_TYPE == EXPECTED_KINDS + + +def test_kind_map_covers_every_installed_span_type(): + consts = pytest.importorskip("autogen.opentelemetry.consts") + installed = {member.value for member in consts.SpanType} + assert installed == set(KIND_BY_SPAN_TYPE) + for value in installed: + assert kind_for_span_type(value) == EXPECTED_KINDS[value] + + +def test_scope_name_matches_installed_consts(): + consts = pytest.importorskip("autogen.opentelemetry.consts") + assert consts.INSTRUMENTING_MODULE_NAME == AG2_SCOPE + + +def test_unknown_or_missing_span_type_is_left_alone(): + attrs = {"gen_ai.operation.name": "chat", "gen_ai.input.messages": "[]"} + assert map_ag2_attributes(attrs) == attrs + assert map_ag2_attributes({"ag2.span.type": "nope"}) == {"ag2.span.type": "nope"} + assert kind_for_span_type(None) is None + + +# Usage, cost, request parameters ------------------------------------------------- + + +def test_llm_span_gets_total_tokens_cost_and_parameters(): + attrs = { + "ag2.span.type": "llm", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.cost": 0.0001, + "gen_ai.request.temperature": 0.2, + "gen_ai.request.max_tokens": 64, + } + mapped = map_ag2_attributes(attrs) + assert mapped["gen_ai.span.kind"] == "LLM" + assert mapped["gen_ai.usage.total_tokens"] == 18 + assert mapped["gen_ai.cost.total"] == 0.0001 + assert mapped["gen_ai.usage.cost"] == 0.0001 # upstream key kept + assert json.loads(mapped["gen_ai.request.parameters"]) == {"temperature": 0.2, "max_tokens": 64} + + +def test_zero_upstream_cost_is_not_reported_as_the_cost(): + # AG2 reports cost 0 for any model missing from its price table ("Model + # ... is not found. The cost will be 0."). gen_ai.cost.total=0 would stop + # Future AGI from pricing the call from its own tokens and model table. + mapped = map_ag2_attributes( + { + "ag2.span.type": "llm", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "claude-sonnet-5", + "gen_ai.usage.input_tokens": 477, + "gen_ai.usage.output_tokens": 78, + "gen_ai.usage.cost": 0, + } + ) + assert "gen_ai.cost.total" not in mapped + assert mapped["gen_ai.usage.cost"] == 0 # upstream key kept + assert mapped["gen_ai.usage.total_tokens"] == 555 + + +def test_llm_span_without_usage_records_no_tokens(): + mapped = map_ag2_attributes({"ag2.span.type": "llm", "gen_ai.operation.name": "chat"}) + assert "gen_ai.usage.total_tokens" not in mapped + assert "gen_ai.cost.total" not in mapped + + +def test_conversation_aggregate_usage_is_not_relabelled(): + attrs = { + "ag2.span.type": "conversation", + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 14, + "gen_ai.usage.cost": 0.0002, + } + mapped = map_ag2_attributes(attrs) + assert "gen_ai.usage.total_tokens" not in mapped + assert "gen_ai.cost.total" not in mapped + # The chat-wide cost moves off the gen_ai.usage.* namespace like the tokens. + assert "gen_ai.usage.cost" not in mapped + assert mapped["ag2.usage.cost"] == 0.0002 + + +@pytest.mark.parametrize("span_type", sorted(set(EXPECTED_KINDS) - {"llm"})) +def test_no_gen_ai_usage_key_survives_on_a_non_llm_span(span_type): + attrs = { + "ag2.span.type": span_type, + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 14, + "gen_ai.usage.total_tokens": 36, + "gen_ai.usage.cost": 0.0002, + } + mapped = map_ag2_attributes(attrs) + assert not [key for key in mapped if key.startswith("gen_ai.usage.") or key == "gen_ai.cost.total"] + assert mapped["ag2.usage.total_tokens"] == 36 + + +def test_conversation_aggregate_tokens_do_not_double_count_the_trace(): + """chat.py:81-82 puts the chat-wide sum on the conversation span. + + fi-collector promotes gen_ai.usage.* on any span and Observe sums + total_tokens over a trace, so only LLM spans keep the promoted keys. + """ + attrs = { + "ag2.span.type": "conversation", + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 14, + } + mapped = map_ag2_attributes(attrs) + assert "gen_ai.usage.input_tokens" not in mapped + assert "gen_ai.usage.output_tokens" not in mapped + assert mapped["ag2.usage.input_tokens"] == 22 + assert mapped["ag2.usage.output_tokens"] == 14 + + +# Session ---------------------------------------------------------------------- + + +def test_root_conversation_id_becomes_session_id(): + mapped = map_ag2_attributes({"ag2.span.type": "conversation", "gen_ai.conversation.id": "123"}) + assert mapped["session.id"] == "123" + + +def test_nested_conversation_id_is_not_a_session(): + mapped = map_ag2_attributes( + {"ag2.span.type": "conversation", "gen_ai.conversation.id": "456"}, root_conversation=False + ) + assert "session.id" not in mapped + assert mapped["gen_ai.conversation.id"] == "456" + + +def test_existing_session_id_wins(): + mapped = map_ag2_attributes( + {"ag2.span.type": "conversation", "gen_ai.conversation.id": "1", "session.id": "mine"} + ) + assert mapped["session.id"] == "mine" + + +# AC-06 content default ------------------------------------------------------------ + + +CONTENT_SAMPLE = { + "gen_ai.input.messages": '[{"role": "user", "parts": [{"type": "text", "content": "secret"}]}]', + "gen_ai.output.messages": '[{"role": "assistant", "parts": []}]', + "gen_ai.tool.call.arguments": '{"city": "secret"}', + "gen_ai.tool.call.result": "secret result", + "ag2.human_input.prompt": "secret prompt", + "ag2.human_input.response": "secret answer", + "ag2.code_execution.output": "secret output", + "ag2.chats.summaries": '["secret summary"]', +} + + +@pytest.mark.parametrize("span_type", sorted(EXPECTED_KINDS)) +def test_content_is_dropped_when_hidden(span_type): + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": span_type, "gen_ai.agent.name": "a"}) + mapped = map_ag2_attributes(attrs, capture_content=False) + for key in CONTENT_KEYS: + assert key not in mapped + assert "secret" not in json.dumps(mapped) + assert mapped["gen_ai.agent.name"] == "a" + + +@pytest.mark.parametrize("span_type", sorted(EXPECTED_KINDS)) +def test_content_is_kept_by_default(span_type): + # traceAI TraceConfig convention: content on unless FI_HIDE_* says otherwise. + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": span_type}) + mapped = map_ag2_attributes(attrs) + for key in CONTENT_KEYS: + if key in attrs: + assert mapped[key] == attrs[key] + + +def test_hide_inputs_only_removes_the_input_side(): + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": "tool"}) + mapped = map_ag2_attributes(attrs, hide_inputs=True) + assert "gen_ai.tool.call.arguments" not in mapped + assert "input.value" not in mapped + assert mapped["gen_ai.tool.call.result"] == attrs["gen_ai.tool.call.result"] + assert mapped["output.value"] == attrs["gen_ai.tool.call.result"] + + +def test_hide_outputs_only_removes_the_output_side(): + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": "tool"}) + mapped = map_ag2_attributes(attrs, hide_outputs=True) + assert "gen_ai.tool.call.result" not in mapped + assert "output.value" not in mapped + assert mapped["gen_ai.tool.call.arguments"] == attrs["gen_ai.tool.call.arguments"] + + +def test_capture_on_lifts_messages_to_input_output(): + attrs = { + "ag2.span.type": "agent", + "gen_ai.input.messages": CONTENT_SAMPLE["gen_ai.input.messages"], + "gen_ai.output.messages": CONTENT_SAMPLE["gen_ai.output.messages"], + } + mapped = map_ag2_attributes(attrs, capture_content=True) + assert mapped["input.value"] == attrs["gen_ai.input.messages"] + assert mapped["input.mime_type"] == "application/json" + assert mapped["output.value"] == attrs["gen_ai.output.messages"] + assert mapped["gen_ai.input.messages"] == attrs["gen_ai.input.messages"] + + +def test_capture_on_tool_human_and_code_spans(): + tool = map_ag2_attributes( + {"ag2.span.type": "tool", "gen_ai.tool.call.arguments": '{"a": 1}', "gen_ai.tool.call.result": "ok"}, + capture_content=True, + ) + assert tool["input.value"] == '{"a": 1}' and tool["input.mime_type"] == "application/json" + assert tool["output.value"] == "ok" and tool["output.mime_type"] == "text/plain" + + human = map_ag2_attributes( + {"ag2.span.type": "human_input", "ag2.human_input.prompt": "p", "ag2.human_input.response": "r"}, + capture_content=True, + ) + assert (human["input.value"], human["output.value"]) == ("p", "r") + + code = map_ag2_attributes( + {"ag2.span.type": "code_execution", "ag2.code_execution.output": "42"}, capture_content=True + ) + assert code["output.value"] == "42" + assert code["gen_ai.span.kind"] == "TOOL" + + +# The processor on a real SDK pipeline ------------------------------------------- + + +def _pipeline(capture_content=False): + exporter = InMemorySpanExporter() + provider = TracerProvider() + processor = AG2ClassicSpanProcessor(capture_content=capture_content) + provider.add_span_processor(processor) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def test_processor_maps_only_the_ag2_scope(): + provider, exporter = _pipeline() + ag2 = provider.get_tracer(AG2_SCOPE) + other = provider.get_tracer("someone.else") + with ag2.start_as_current_span("chat m") as span: + span.set_attribute("ag2.span.type", "llm") + span.set_attribute("gen_ai.input.messages", "secret") + with other.start_as_current_span("x") as span: + span.set_attribute("ag2.span.type", "llm") + span.set_attribute("gen_ai.input.messages", "kept") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["chat m"].attributes["gen_ai.span.kind"] == "LLM" + assert "gen_ai.input.messages" not in by_name["chat m"].attributes + assert "gen_ai.span.kind" not in by_name["x"].attributes + assert by_name["x"].attributes["gen_ai.input.messages"] == "kept" + + +def test_processor_promotes_error_type_to_error_status(): + provider, exporter = _pipeline() + tracer = provider.get_tracer(AG2_SCOPE) + with tracer.start_as_current_span("execute_tool t") as span: + span.set_attribute("ag2.span.type", "tool") + span.set_attribute("error.type", "ExecutionError") + with tracer.start_as_current_span("execute_tool ok") as span: + span.set_attribute("ag2.span.type", "tool") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["execute_tool t"].status.status_code is StatusCode.ERROR + assert by_name["execute_tool t"].status.description == "ExecutionError" + assert by_name["execute_tool ok"].status.status_code is StatusCode.UNSET + + +def test_processor_keeps_session_on_outer_conversation_only(): + provider, exporter = _pipeline() + tracer = provider.get_tracer(AG2_SCOPE) + with tracer.start_as_current_span("conversation outer") as outer: + outer.set_attribute("ag2.span.type", "conversation") + with tracer.start_as_current_span("speaker_selection") as sel: + sel.set_attribute("ag2.span.type", "speaker_selection") + with tracer.start_as_current_span("conversation inner") as inner: + inner.set_attribute("ag2.span.type", "conversation") + inner.set_attribute("gen_ai.conversation.id", "inner-id") + outer.set_attribute("gen_ai.conversation.id", "outer-id") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["conversation outer"].attributes["session.id"] == "outer-id" + assert "session.id" not in by_name["conversation inner"].attributes + assert by_name["speaker_selection"].attributes["gen_ai.span.kind"] == "CHAIN" + + +def test_non_ag2_span_between_conversations_does_not_split_the_session(): + """Agent-as-tool: the user's own span wraps an inner initiate_chat.""" + provider, exporter = _pipeline() + processor = provider._active_span_processor._span_processors[0] + ag2 = provider.get_tracer(AG2_SCOPE) + user = provider.get_tracer("user.app") + with ag2.start_as_current_span("conversation outer") as outer: + outer.set_attribute("ag2.span.type", "conversation") + with user.start_as_current_span("user work"): + with user.start_as_current_span("user inner work"): + with ag2.start_as_current_span("conversation inner") as inner: + inner.set_attribute("ag2.span.type", "conversation") + inner.set_attribute("gen_ai.conversation.id", "inner-id") + outer.set_attribute("gen_ai.conversation.id", "outer-id") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["conversation outer"].attributes["session.id"] == "outer-id" + assert "session.id" not in by_name["conversation inner"].attributes + assert "session.id" not in by_name["user work"].attributes + # Every tracked span, AG2 or not, is released when it ends. + assert processor._live == {} + + +def test_live_span_tracking_is_bounded(monkeypatch): + from traceai_ag2_classic import _processor + + monkeypatch.setattr(_processor, "_MAX_LIVE_SPANS", 2) + provider, exporter = _pipeline() + processor = provider._active_span_processor._span_processors[0] + user = provider.get_tracer("user.app") + with user.start_as_current_span("a"), user.start_as_current_span("b"), user.start_as_current_span("c"): + assert len(processor._live) == 2 + assert processor._live == {} + assert len(exporter.get_finished_spans()) == 3 + + +# Context attributes (using_attributes / using_session / using_user) ------------ + + +def test_context_attributes_are_copied_onto_ag2_spans(): + from fi_instrumentation import using_attributes + + provider, exporter = _pipeline() + ag2 = provider.get_tracer(AG2_SCOPE) + other = provider.get_tracer("someone.else") + with using_attributes(session_id="my-session", user_id="u-1", metadata={"k": "v"}, tags=["t1"]): + with ag2.start_as_current_span("conversation c") as conversation: + conversation.set_attribute("ag2.span.type", "conversation") + with ag2.start_as_current_span("chat m") as llm: + llm.set_attribute("ag2.span.type", "llm") + conversation.set_attribute("gen_ai.conversation.id", "chat-1") + with other.start_as_current_span("x"): + pass + by_name = {s.name: s.attributes for s in exporter.get_finished_spans()} + for name in ("conversation c", "chat m"): + assert by_name[name]["session.id"] == "my-session", name + assert by_name[name]["user.id"] == "u-1", name + assert json.loads(by_name[name]["metadata"]) == {"k": "v"}, name + assert tuple(by_name[name]["tag.tags"]) == ("t1",), name + # The user's session wins over the upstream chat id; the chat id is kept. + assert by_name["conversation c"]["gen_ai.conversation.id"] == "chat-1" + # Other AG2 spans carry the session id there (fi using_* sets it in context). + assert by_name["chat m"]["gen_ai.conversation.id"] == "my-session" + # Spans from other instrumentations are left to their own tracer. + assert "session.id" not in by_name["x"] and "user.id" not in by_name["x"] + + +def test_context_attributes_do_not_overwrite_upstream_values(): + from fi_instrumentation import using_session + + provider, exporter = _pipeline() + ag2 = provider.get_tracer(AG2_SCOPE) + with using_session("my-session"): + with ag2.start_as_current_span( + "conversation c", attributes={"ag2.span.type": "conversation", "gen_ai.conversation.id": "chat-1"} + ): + pass + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["session.id"] == "my-session" + assert attrs["gen_ai.conversation.id"] == "chat-1" + + +def test_processor_preserves_bounded_attribute_limits(): + from opentelemetry.attributes import BoundedAttributes + + provider, exporter = _pipeline() + with provider.get_tracer(AG2_SCOPE).start_as_current_span("chat") as span: + span.set_attribute("ag2.span.type", "llm") + finished = exporter.get_finished_spans()[0] + assert isinstance(finished._attributes, BoundedAttributes) + with pytest.raises(TypeError): + finished._attributes["x"] = 1 # immutable after end, as the SDK leaves it + + +def test_processor_never_raises_into_the_sdk(): + provider, exporter = _pipeline() + processor = provider._active_span_processor._span_processors[0] + live = processor._live + processor._live = None # force an internal failure + try: + with provider.get_tracer(AG2_SCOPE).start_as_current_span("chat") as span: + span.set_attribute("ag2.span.type", "llm") + finally: + processor._live = live + assert [s.name for s in exporter.get_finished_spans()] == ["chat"] diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py new file mode 100644 index 000000000..aa47805c2 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -0,0 +1,777 @@ +"""In-process tests for setup(): one provider, resource/headers, and real AG2 Classic runs. + +Real ``autogen`` 0.14.x agents talk to a loopback fake OpenAI server; spans go +to an in-memory exporter. Each test undoes the global ``OpenAIWrapper.create`` +patch through ``AG2ClassicTracing.uninstrument()``. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import subprocess +import sys +from pathlib import Path + +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +pytest.importorskip("autogen.opentelemetry") + +from _fake_openai import COMPLETION_TOKENS, FAKE_MODEL, PROMPT_TOKENS, FakeOpenAI # noqa: E402 +from _scenarios import ( # noqa: E402 + SECRET_PROMPT, + TOOL_SECRET_CITY, + broken_tool_chat, + failing_llm_chat, + group_chat, + two_agent_tool_chat, +) + +from traceai_ag2_classic import CONTENT_KEYS, AG2ClassicSpanProcessor, setup # noqa: E402 + +PACKAGE_DIR = Path(__file__).resolve().parent.parent +PYTHON_DIR = PACKAGE_DIR.parent.parent + + +@pytest.fixture +def pipeline(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + handles = [] + + def make(**kwargs): + # Most tests exercise the hidden-content path; the shown-by-default + # behaviour has its own tests below (they pass capture_content=None). + kwargs.setdefault("capture_content", False) + handle = setup(tracer_provider=provider, **kwargs) + handles.append(handle) + return handle + + yield provider, exporter, make + for handle in handles: + handle.uninstrument() + + +@pytest.fixture +def fake(): + with FakeOpenAI(tool_arguments={"city": TOOL_SECRET_CITY}) as server: + yield server + + +def _by_name(exporter): + grouped = {} + for span in exporter.get_finished_spans(): + grouped.setdefault(span.name, []).append(span) + return grouped + + +def _exporter_of(processor): + exporter = getattr(processor, "span_exporter", None) + if exporter is None: + exporter = getattr(getattr(processor, "_batch_processor", None), "_exporter", None) + return exporter + + +# One provider, resource attributes and headers ----------------------------------- + + +class _RecordingCollector: + """Loopback HTTP server that records path, headers and body of each POST. + + The shared harness Receiver drops headers and resource attributes, so this + unit test records them itself. + """ + + def __init__(self): + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + self.requests = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + owner.requests.append((self.path, dict(self.headers.items()), body)) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format, *args): # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +def test_register_then_setup_keeps_one_provider_resource_and_headers(monkeypatch): + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + + from traceai_ag2_classic import AG2_SCOPE + + collector = _RecordingCollector() + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + monkeypatch.setenv("FI_BASE_URL", collector.origin) + saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + global_before = trace_api.get_tracer_provider() + provider = register(project_type=ProjectType.OBSERVE, project_name="ag2-classic-unit", verbose=False) + tracing = setup(tracer_provider=provider) + try: + processors = provider._active_span_processor._span_processors + assert isinstance(processors[0], AG2ClassicSpanProcessor) + exporting = [p for p in processors if _exporter_of(p) is not None] + assert len(exporting) == 1, "setup() must not add a second exporter" + assert type(_exporter_of(exporting[0])).__name__ == "HTTPSpanExporter" + # setup() never set or replaced the global provider. + assert trace_api.get_tracer_provider() is global_before + assert tracing.autogen_version.startswith("0.14.") + + with provider.get_tracer(AG2_SCOPE).start_as_current_span("chat m") as span: + span.set_attribute("ag2.span.type", "llm") + assert provider.force_flush(timeout_millis=10000) + + assert len(collector.requests) == 1 + path, headers, body = collector.requests[0] + assert path == "/tracer/v1/traces" + lowered = {k.lower(): v for k, v in headers.items()} + assert lowered["x-api-key"] == "placeholder-api-key" + assert lowered["x-secret-key"] == "placeholder-secret-key" + assert "authorization" not in lowered + assert lowered["content-type"] == "application/x-protobuf" + + request = ExportTraceServiceRequest() + request.ParseFromString(body) + resource = {kv.key: kv.value.string_value for kv in request.resource_spans[0].resource.attributes} + assert resource["project_name"] == "ag2-classic-unit" + assert resource["project_type"] == "observe" + assert "openinference.project.name" not in resource + exported = request.resource_spans[0].scope_spans[0].spans[0] + attrs = {kv.key: kv.value.string_value for kv in exported.attributes} + assert attrs["gen_ai.span.kind"] == "LLM" + finally: + tracing.uninstrument() + provider.shutdown() + collector.close() + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(monkeypatch, fake): + """fi's TracerProvider.add_span_processor shuts down and clears every + processor, ours included (fi_instrumentation/otel.py:336-339). setup() + re-prepends and re-enables ours so the new processor still sees filtered spans. + """ + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") + saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + provider = register( + project_type=ProjectType.OBSERVE, + project_name="ag2-classic-add", + verbose=False, + set_global_tracer_provider=False, + ) + tracing = setup(tracer_provider=provider, capture_content=False) + memory = InMemorySpanExporter() + try: + added = SimpleSpanProcessor(memory) + provider.add_span_processor(added) + processors = provider._active_span_processor._span_processors + assert processors == (tracing.processor, added) + assert tracing.processor._disabled is False + + two_agent_tool_chat(tracing, fake) + spans = memory.get_finished_spans() + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) + conversation = next(s for s in spans if s.name == "conversation user").attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" + + # A second add keeps exactly one copy of ours, still first. + second = SimpleSpanProcessor(InMemorySpanExporter()) + provider.add_span_processor(second) + assert provider._active_span_processor._span_processors == (tracing.processor, added, second) + finally: + tracing.uninstrument() + provider.shutdown() + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +def test_add_span_processor_on_a_plain_sdk_provider_keeps_one_processor_first(pipeline): + provider, _exporter, make = pipeline + first = make() + make() # a second setup() must not stack a second guard or processor + extra = SimpleSpanProcessor(InMemorySpanExporter()) + provider.add_span_processor(extra) + processors = provider._active_span_processor._span_processors + assert processors[0] is first.processor + assert processors[-1] is extra + assert sum(isinstance(p, AG2ClassicSpanProcessor) for p in processors) == 1 + + +def test_setup_again_reopens_a_processor_that_was_shut_down(pipeline, fake): + """A shutdown that bypassed the add_span_processor guard (for example an + unbound ``TracerProvider.add_span_processor(provider, p)``) leaves the + processor disabled; on_end then skips filtering. A second setup() reuses + that processor and must turn filtering back on, not export content. + """ + _provider, exporter, make = pipeline + first = make() + first.processor.shutdown() + second = make() + assert second.processor is first.processor + + exporter.clear() + two_agent_tool_chat(second, fake) + spans = exporter.get_finished_spans() + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) + assert second.processor._disabled is False + + +def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline): + from autogen.oai.client import OpenAIWrapper + + provider, _exporter, make = pipeline + original = OpenAIWrapper.create + first = make() + assert getattr(OpenAIWrapper.create, "__otel_wrapped__", False) is True + second = make() + ours = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2ClassicSpanProcessor)] + assert len(ours) == 1 and first.processor is second.processor + assert second.owns_llm_wrapper is False # upstream wraps once (llm_wrapper.py:64-65) + first.uninstrument() + assert OpenAIWrapper.create is original + # Upstream has no per-agent undo, so agents instrumented earlier keep + # emitting spans on this provider. The processor stays installed and live + # so those spans are still filtered; the second handle keeps working too. + ours = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2ClassicSpanProcessor)] + assert ours == [first.processor] + assert provider._active_span_processor._span_processors[0] is first.processor + assert first.processor._disabled is False + + +def test_uninstrument_leaves_a_later_third_party_wrapper_in_place(pipeline, caplog): + """Another library wraps OpenAIWrapper.create after setup(). Restoring + our captured original would silently drop that wrapper. + """ + from autogen.oai import client as oai_client_module + from autogen.oai.client import OpenAIWrapper + + _provider, _exporter, make = pipeline + original = OpenAIWrapper.create + tracing = make() + traced = OpenAIWrapper.create + assert tracing.owns_llm_wrapper is True and traced is not original + + def third_party_create(self, **config): + return traced(self, **config) + + OpenAIWrapper.create = third_party_create + try: + with caplog.at_level(logging.WARNING, logger="traceai_ag2_classic"): + tracing.uninstrument() + assert OpenAIWrapper.create is third_party_create + assert oai_client_module.OpenAIWrapper.create is third_party_create + assert any( + r.levelno == logging.WARNING and "OpenAIWrapper.create" in r.getMessage() for r in caplog.records + ) + + # Once the other wrapper is gone, the same handle can still restore. + OpenAIWrapper.create = traced + tracing.uninstrument() + assert OpenAIWrapper.create is original + assert tracing.owns_llm_wrapper is False + finally: + OpenAIWrapper.create = original + oai_client_module.OpenAIWrapper.create = original + + +PROMOTED_USAGE_KEYS = ( + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.cost.total", + "llm.cost.total", +) + + +def _assert_no_content_and_no_promoted_usage_off_llm(spans): + assert spans, "no spans were exported" + for span in spans: + attrs = dict(span.attributes) + for key in CONTENT_KEYS: + assert key not in attrs, (key, span.name) + blob = json.dumps(attrs, default=str) + assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob, span.name + if attrs.get("gen_ai.span.kind") != "LLM": + leaked = [k for k in attrs if k in PROMOTED_USAGE_KEYS or k.startswith("gen_ai.usage.")] + assert not leaked, (span.name, leaked) + + +def test_previously_instrumented_agents_stay_filtered_after_uninstrument(pipeline, fake): + from autogen import ConversableAgent + + _provider, exporter, make = pipeline + tracing = make() + assistant = ConversableAgent( + "assistant", + llm_config=fake.llm_config(), + human_input_mode="NEVER", + is_termination_msg=lambda m: "TERMINATE" in str(m.get("content") or ""), + ) + user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=1) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + tracing.uninstrument() + exporter.clear() + + user.initiate_chat(assistant, message=SECRET_PROMPT, max_turns=1, silent=True) + spans = exporter.get_finished_spans() + names = {s.name for s in spans} + # Upstream patched these agents for good; their spans still arrive. + assert {"conversation user", "invoke_agent assistant"} <= names + _assert_no_content_and_no_promoted_usage_off_llm(spans) + conversation = next(s for s in spans if s.name == "conversation user").attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" + assert conversation["ag2.usage.input_tokens"] == PROMPT_TOKENS + + +def test_second_handle_keeps_filtering_after_first_handle_uninstruments(pipeline, fake): + _provider, exporter, make = pipeline + first = make() + second = make() + first.uninstrument() + exporter.clear() + two_agent_tool_chat(second, fake) + spans = exporter.get_finished_spans() + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) + + +def test_setup_rejects_a_non_sdk_provider(): + # NoOpTracerProvider has no span processors: setup() refuses instead of creating a provider. + with pytest.raises(TypeError, match="SDK TracerProvider"): + setup(tracer_provider=trace_api.NoOpTracerProvider()) + + +def test_setup_warns_when_span_processors_run_concurrently(caplog): + """ConcurrentMultiSpanProcessor runs every on_end in parallel, so an + exporter can serialize a span before this processor removed its content. + """ + from opentelemetry.sdk.trace import ConcurrentMultiSpanProcessor + + concurrent = TracerProvider(active_span_processor=ConcurrentMultiSpanProcessor()) + plain = TracerProvider() + try: + with caplog.at_level(logging.WARNING, logger="traceai_ag2_classic"): + setup(tracer_provider=concurrent, instrument_llm=False) + warnings = [ + r for r in caplog.records if r.levelno == logging.WARNING and r.name.startswith("traceai_ag2_classic") + ] + assert len(warnings) == 1, [r.getMessage() for r in warnings] + assert "ConcurrentMultiSpanProcessor" in warnings[0].getMessage() + + caplog.clear() + with caplog.at_level(logging.WARNING, logger="traceai_ag2_classic"): + setup(tracer_provider=plain, instrument_llm=False) + assert not [r for r in caplog.records if "ConcurrentMultiSpanProcessor" in r.getMessage()] + finally: + concurrent.shutdown() + plain.shutdown() + + +def test_setup_rolls_back_the_llm_patch_when_an_agent_fails(fake): + from autogen import ConversableAgent + from autogen.oai import client as oai_client_module + from autogen.oai.client import OpenAIWrapper + + provider = TracerProvider() + original = OpenAIWrapper.create + good = ConversableAgent("good", llm_config=fake.llm_config(), human_input_mode="NEVER") + try: + # Upstream instrument_agent raises AttributeError on a non-agent. + with pytest.raises(AttributeError): + setup(tracer_provider=provider, agents=[good, object()]) + assert OpenAIWrapper.create is original + assert oai_client_module.OpenAIWrapper.create is original + # A later setup() can still own the global LLM patch. + retry = setup(tracer_provider=provider) + assert retry.owns_llm_wrapper is True + retry.uninstrument() + assert OpenAIWrapper.create is original + finally: + OpenAIWrapper.create = original + oai_client_module.OpenAIWrapper.create = original + + +# AC-01 / AC-03 / AC-06: two-agent chat with a tool, content off -------------------- + + +def test_two_agent_chat_kinds_usage_session_and_no_content(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make() + result = two_agent_tool_chat(tracing, fake) + spans = _by_name(exporter) + + assert {"conversation user", "invoke_agent assistant", "invoke_agent user", "execute_tool get_weather"} <= set( + spans + ) + llm_spans = spans["chat {0}".format(FAKE_MODEL)] + assert len(llm_spans) == 2 + for span in llm_spans: + attrs = span.attributes + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.request.model"] == FAKE_MODEL + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.usage.input_tokens"] == PROMPT_TOKENS + assert attrs["gen_ai.usage.output_tokens"] == COMPLETION_TOKENS + assert attrs["gen_ai.usage.total_tokens"] == PROMPT_TOKENS + COMPLETION_TOKENS + assert attrs["gen_ai.cost.total"] > 0 + + assert spans["invoke_agent assistant"][0].attributes["gen_ai.span.kind"] == "AGENT" + tool = spans["execute_tool get_weather"][0] + assert tool.attributes["gen_ai.span.kind"] == "TOOL" + assert tool.attributes["gen_ai.tool.name"] == "get_weather" + assert tool.status.status_code is StatusCode.UNSET + + conversation = spans["conversation user"][0].attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" + assert conversation["session.id"] == str(result.chat_id) + # Upstream aggregate on the conversation span (chat.py:73-82) is kept under + # ag2.usage.* so the trace total counts each LLM call once. + assert "gen_ai.usage.input_tokens" not in conversation + assert conversation["ag2.usage.input_tokens"] == 2 * PROMPT_TOKENS + promoted = sum( + s.attributes.get("gen_ai.usage.input_tokens", 0) + for group in spans.values() + for s in group + ) + assert promoted == 2 * PROMPT_TOKENS + + # LLM spans are children of the agent span, all in one trace. + trace_ids = {s.context.trace_id for group in spans.values() for s in group} + assert len(trace_ids) == 1 + + for group in spans.values(): + for span in group: + for key in CONTENT_KEYS: + assert key not in span.attributes, (key, span.name) + blob = json.dumps(dict(span.attributes), default=str) + assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob + + +def test_capture_content_on_keeps_and_lifts_content(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make(capture_content=True) + two_agent_tool_chat(tracing, fake) + spans = _by_name(exporter) + conversation = spans["conversation user"][0].attributes + assert SECRET_PROMPT in conversation["input.value"] + tool = spans["execute_tool get_weather"][0].attributes + assert json.loads(tool["gen_ai.tool.call.arguments"]) == {"city": TOOL_SECRET_CITY} + assert tool["output.value"] == "sunny in {0}".format(TOOL_SECRET_CITY) + llm = spans["chat {0}".format(FAKE_MODEL)][0].attributes + assert SECRET_PROMPT in llm["gen_ai.input.messages"] + assert SECRET_PROMPT in llm["input.value"] + + +def test_default_follows_traceconfig_and_keeps_content(pipeline, fake, monkeypatch): + """traceAI convention: no FI_HIDE_* means content is exported (no second switch).""" + for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_INPUT_MESSAGES", "FI_HIDE_OUTPUT_MESSAGES"): + monkeypatch.delenv(name, raising=False) + _provider, exporter, make = pipeline + tracing = make(capture_content=None) + assert tracing.capture_content is True + two_agent_tool_chat(tracing, fake) + spans = _by_name(exporter) + assert SECRET_PROMPT in spans["conversation user"][0].attributes["input.value"] + tool = spans["execute_tool get_weather"][0].attributes + assert json.loads(tool["gen_ai.tool.call.arguments"]) == {"city": TOOL_SECRET_CITY} + llm = spans["chat {0}".format(FAKE_MODEL)][0].attributes + assert SECRET_PROMPT in llm["gen_ai.input.messages"] + + +def test_fi_hide_env_vars_hide_all_content(pipeline, fake, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + _provider, exporter, make = pipeline + tracing = make(capture_content=None) + assert tracing.capture_content is False + two_agent_tool_chat(tracing, fake) + for span in exporter.get_finished_spans(): + for key in CONTENT_KEYS: + assert key not in span.attributes, (key, span.name) + blob = json.dumps(dict(span.attributes), default=str) + assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob + + +def test_traceconfig_hide_inputs_keeps_outputs(pipeline, fake): + from fi_instrumentation import TraceConfig + + _provider, exporter, make = pipeline + tracing = make(capture_content=None, config=TraceConfig(hide_inputs=True)) + two_agent_tool_chat(tracing, fake) + tool = _by_name(exporter)["execute_tool get_weather"][0].attributes + assert "gen_ai.tool.call.arguments" not in tool + assert tool["gen_ai.tool.call.result"] == "sunny in {0}".format(TOOL_SECRET_CITY) or TOOL_SECRET_CITY in tool["gen_ai.tool.call.result"] + for span in exporter.get_finished_spans(): + assert "gen_ai.input.messages" not in span.attributes + # The chat-wide output history still carries the prompt; only the + # input side is hidden. + assert "gen_ai.tool.call.arguments" not in span.attributes + assert "input.value" not in span.attributes + + +# AC-07: tool exception -> ERROR --------------------------------------------------- + + +def test_tool_exception_sets_error_status(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make() + broken_tool_chat(tracing, fake) + tool = _by_name(exporter)["execute_tool broken_tool"][0] + assert tool.attributes["error.type"] == "ExecutionError" + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "ExecutionError" + + +def test_llm_exception_keeps_error_status_and_error_type(pipeline, fake): + """The model call raises: upstream sets error.type and re-raises (llm_wrapper.py:97-101).""" + _provider, exporter, make = pipeline + tracing = make() + assert failing_llm_chat(tracing, fake) == "BadRequestError" + assert len(fake.failed_requests) == 1 + finished = exporter.get_finished_spans() + llm = [s for s in finished if s.attributes.get("ag2.span.type") == "llm"] + assert len(llm) == 1 + attrs = llm[0].attributes + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["error.type"] == "BadRequestError" + assert llm[0].status.status_code is StatusCode.ERROR + # A failed call has no usage: nothing to count, nothing invented. + for key in ("gen_ai.usage.total_tokens", "gen_ai.cost.total"): + assert key not in attrs + _assert_no_content_and_no_promoted_usage_off_llm(finished) + + +# AC-04: group chat, one trace id ---------------------------------------------------- + + +def test_group_chat_is_one_trace_with_one_session(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make() + result = group_chat(tracing, fake) + finished = exporter.get_finished_spans() + names = {s.name for s in finished} + assert {"conversation writer", "speaker_selection", "invoke_agent critic", "conversation checking_agent"} <= names + assert len({s.context.trace_id for s in finished}) == 1 + + sessions = {s.attributes.get("session.id") for s in finished} - {None} + assert sessions == {str(result.chat_id)} + selection = next(s for s in finished if s.name == "speaker_selection") + assert selection.attributes["gen_ai.span.kind"] == "CHAIN" + assert selection.attributes["ag2.speaker_selection.selected"] == "critic" + + +def test_user_span_around_an_inner_chat_keeps_one_session(pipeline, fake): + """Agent-as-tool: a tool runs an inner chat inside the user's own span.""" + from autogen import ConversableAgent + + provider, exporter, make = pipeline + tracing = make() + user_tracer = provider.get_tracer("user.app") + terminates = lambda m: "TERMINATE" in str(m.get("content") or "") # noqa: E731 + expert = ConversableAgent("expert", llm_config=fake.llm_config(), human_input_mode="NEVER") + asker = ConversableAgent("asker", llm_config=False, human_input_mode="NEVER") + tracing.instrument_agent(expert) + tracing.instrument_agent(asker) + + def ask_expert(city: str) -> str: + """Ask the expert about a city.""" + with user_tracer.start_as_current_span("ask_expert work"): + inner = asker.initiate_chat(expert, message="about " + city, max_turns=1, silent=True) + return str(inner.summary) + + assistant = ConversableAgent( + "assistant", llm_config=fake.llm_config(), human_input_mode="NEVER", is_termination_msg=terminates + ) + user = ConversableAgent( + "user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=3, is_termination_msg=terminates + ) + assistant.register_for_llm(description="Ask the expert")(ask_expert) + user.register_for_execution()(ask_expert) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + result = user.initiate_chat(assistant, message="hi", max_turns=3, silent=True) + + finished = exporter.get_finished_spans() + names = {s.name for s in finished} + assert {"conversation user", "execute_tool ask_expert", "ask_expert work", "conversation asker"} <= names + assert len({s.context.trace_id for s in finished}) == 1 + assert {s.attributes.get("session.id") for s in finished} - {None} == {str(result.chat_id)} + inner = next(s for s in finished if s.name == "conversation asker") + assert inner.attributes["gen_ai.conversation.id"] != str(result.chat_id) + assert "session.id" not in inner.attributes + + +# Context attributes and initiate_chats sessions ----------------------------------- + + +def test_using_attributes_sets_session_and_user_on_ag2_spans(pipeline, fake): + from fi_instrumentation import using_attributes + + _provider, exporter, make = pipeline + tracing = make() + with using_attributes(session_id="my-session", user_id="u-1"): + result = two_agent_tool_chat(tracing, fake) + finished = exporter.get_finished_spans() + assert {s.name for s in finished} >= {"conversation user", "chat {0}".format(FAKE_MODEL), "execute_tool get_weather"} + for span in finished: + assert span.attributes["session.id"] == "my-session", span.name + assert span.attributes["user.id"] == "u-1", span.name + conversation = next(s for s in finished if s.name == "conversation user").attributes + assert conversation["gen_ai.conversation.id"] == str(result.chat_id) + # Only the initiate_chat conversation span keeps the chat id there; every + # other AG2 span carries the session id that using_attributes put in context. + for span in finished: + if span.name != "conversation user": + assert span.attributes["gen_ai.conversation.id"] == "my-session", span.name + + +def _initiate_chats(tracing, fake): + from autogen import ConversableAgent + + first = ConversableAgent("first", llm_config=fake.llm_config(), human_input_mode="NEVER") + second = ConversableAgent("second", llm_config=fake.llm_config(), human_input_mode="NEVER") + sender = ConversableAgent("sender", llm_config=False, human_input_mode="NEVER") + for agent in (first, second, sender): + tracing.instrument_agent(agent) + return sender.initiate_chats( + [ + {"recipient": first, "message": "one", "max_turns": 1, "silent": True}, + {"recipient": second, "message": "two", "max_turns": 1, "silent": True}, + ] + ) + + +def test_initiate_chats_has_one_session_per_chat_by_default(pipeline, fake): + _provider, exporter, make = pipeline + results = _initiate_chats(make(), fake) + finished = exporter.get_finished_spans() + assert len({s.context.trace_id for s in finished}) == 1 + sessions = {s.attributes.get("session.id") for s in finished} - {None} + assert sessions == {str(r.chat_id) for r in results} and len(sessions) == 2 + multi = next(s for s in finished if s.attributes.get("ag2.span.type") == "multi_conversation") + assert "session.id" not in multi.attributes + + +def test_using_session_gives_initiate_chats_one_session(pipeline, fake): + from fi_instrumentation import using_session + + _provider, exporter, make = pipeline + tracing = make() + with using_session("batch-1"): + _initiate_chats(tracing, fake) + finished = exporter.get_finished_spans() + assert any(s.attributes.get("ag2.span.type") == "multi_conversation" for s in finished) + assert {s.attributes.get("session.id") for s in finished} == {"batch-1"} + + +def test_patterns_are_passed_to_upstream_instrument_pattern(pipeline, fake): + from autogen import ConversableAgent + from autogen.agentchat import initiate_group_chat + from autogen.agentchat.group.patterns import RoundRobinPattern + + _provider, exporter, make = pipeline + alpha = ConversableAgent("alpha", llm_config=fake.llm_config(), human_input_mode="NEVER") + beta = ConversableAgent("beta", llm_config=fake.llm_config(), human_input_mode="NEVER") + pattern = RoundRobinPattern(initial_agent=alpha, agents=[alpha, beta]) + make(patterns=[pattern]) + assert getattr(pattern.prepare_group_chat, "__otel_wrapped__", False) is True + result, _ctx, _last = initiate_group_chat(pattern=pattern, messages="hello pattern", max_rounds=3) + finished = exporter.get_finished_spans() + names = {s.name for s in finished} + assert {"invoke_agent alpha", "invoke_agent beta", "chat {0}".format(FAKE_MODEL)} <= names + assert len({s.context.trace_id for s in finished}) == 1 + assert {s.attributes.get("session.id") for s in finished} - {None} == {str(result.chat_id)} + + +# Export failure never fails the agent --------------------------------------------- + + +def test_export_failure_does_not_fail_the_agent(monkeypatch, fake): + import socket + + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + with socket.socket() as sock: # a loopback port with nothing listening + sock.bind(("127.0.0.1", 0)) + closed_port = sock.getsockname()[1] + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:{0}".format(closed_port)) + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + provider = register(project_type=ProjectType.OBSERVE, project_name="ag2-classic-down", verbose=False, timeout=1) + tracing = setup(tracer_provider=provider) + try: + result = two_agent_tool_chat(tracing, fake) + assert result.chat_id + assert "TERMINATE" in result.chat_history[-1]["content"] + provider.force_flush(timeout_millis=3000) # export fails; must not raise + finally: + tracing.uninstrument() + provider.shutdown() + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +# AC-08 (partial): importing the package never imports ag2 ------------------------------ + + +def test_import_does_not_import_ag2_or_autogen_until_setup(): + code = ( + "import sys, importlib.metadata as md\n" + "import traceai_ag2_classic\n" + "assert 'autogen' not in sys.modules, 'package import pulled autogen'\n" + "traceai_ag2_classic.check_autogen_classic()\n" + "assert 'ag2' not in sys.modules, 'an ag2 module was imported'\n" + "try:\n" + " print('AG2_DIST', md.version('ag2'))\n" + "except md.PackageNotFoundError:\n" + " print('AG2_DIST none')\n" + "print('AUTOGEN_DIST', md.version('autogen'))\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(PACKAGE_DIR), str(PYTHON_DIR), env.get("PYTHONPATH", "")]) + proc = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=120) + assert proc.returncode == 0, proc.stderr[-3000:] + lines = dict(line.split(" ", 1) for line in proc.stdout.splitlines() if line.startswith(("AG2_DIST", "AUTOGEN_DIST"))) + if lines["AUTOGEN_DIST"] == "0.14.0": + # PyPI autogen==0.14.0 is an alias that requires ag2==0.14.0 (see _guard.py). + assert lines["AG2_DIST"] == "0.14.0" + else: + assert lines["AG2_DIST"] == "none" diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py b/python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py new file mode 100644 index 000000000..720110810 --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py @@ -0,0 +1,34 @@ +"""Future AGI tracing for AG2 Classic (the ``autogen`` distribution, 0.14.x). + +Native-first: AG2 Classic ships ``autogen.opentelemetry``. ``setup()`` calls +its ``instrument_llm_wrapper`` / ``instrument_agent`` / ``instrument_pattern`` +on the Future AGI tracer provider you pass, and adds a span processor that maps +the upstream keys to Future AGI conventions. Importing this package does not +import ``autogen``; ``setup()`` does, after the version guard. +""" + +from ._guard import AG2ClassicCompatibilityError, check_autogen_classic +from ._processor import ( + AG2_SCOPE, + CONTENT_KEYS, + KIND_BY_SPAN_TYPE, + AG2ClassicSpanProcessor, + kind_for_span_type, + map_ag2_attributes, +) +from ._setup import AG2ClassicTracing, setup +from .version import __version__ + +__all__ = [ + "AG2_SCOPE", + "CONTENT_KEYS", + "KIND_BY_SPAN_TYPE", + "AG2ClassicCompatibilityError", + "AG2ClassicSpanProcessor", + "AG2ClassicTracing", + "check_autogen_classic", + "kind_for_span_type", + "map_ag2_attributes", + "setup", + "__version__", +] diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py new file mode 100644 index 000000000..e6eae24d9 --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py @@ -0,0 +1,151 @@ +"""Version guard for AG2 Classic (``import autogen``, 0.14.x). + +Three different projects are easy to confuse: + +* ``autogen`` 0.14.x (AG2 Classic, ``ag2ai/ag2classic``): this package. +* ``ag2`` 1.x (AG2, ``import ag2``): use ``traceAI-ag2``. +* ``autogen-agentchat`` (Microsoft AutoGen): use ``traceAI-autogen``. + +``pyautogen`` is an older alias and is out of scope for this release. + +Installed-layout facts this guard is written against (read, not assumed): + +* ``autogen==0.14.1`` is a standalone distribution that ships the ``autogen`` + module itself (Source: ``ag2ai/ag2classic``). +* ``autogen==0.14.0`` is a metadata-only alias wheel (6 entries, all under + ``autogen-0.14.0.dist-info/``, no modules) whose only runtime requirement is + ``ag2==0.14.0``; the ``autogen`` module is shipped by the ``ag2`` 0.14.0 + distribution. Its ``autogen/opentelemetry`` tree is byte-identical to the + 0.14.1 one. That exact pairing is accepted. +* ``ag2`` 1.x ships only an ``ag2`` top-level module, never ``autogen``. + +The guard runs before anything is instrumented. It never imports ``ag2`` and +never imports Microsoft AutoGen. +""" + +from __future__ import annotations + +import importlib +import importlib.util +from typing import Callable, Iterable, List, Mapping, Optional + +SUPPORTED_SERIES = "0.14" + +_GUIDANCE = ( + "traceAI-ag2-classic supports only AG2 Classic: the `autogen` distribution 0.14.x " + "(import `autogen`, ships `autogen.opentelemetry`). " + "Three packages are easy to confuse: " + "`autogen` 0.14.x -> traceAI-ag2-classic; " + "`ag2` 1.x -> traceAI-ag2; " + "`autogen-agentchat` (Microsoft AutoGen) -> traceAI-autogen. " + "`pyautogen` is not supported by this package." +) + + +class AG2ClassicCompatibilityError(RuntimeError): + """Raised when the importable ``autogen`` is not AG2 Classic 0.14.x.""" + + def __init__(self, reason: str) -> None: + super().__init__("{0} {1}".format(reason, _GUIDANCE)) + self.reason = reason + + +def _normalize(name: str) -> str: + return name.strip().lower().replace("_", "-").replace(".", "-") + + +def _default_distributions_for(top_level: str) -> List[str]: + from importlib import metadata + + try: + mapping: Mapping[str, Iterable[str]] = metadata.packages_distributions() + except Exception: # pragma: no cover - defensive, metadata is stdlib on 3.10+ + return [] + return list(mapping.get(top_level, [])) + + +def _default_distribution_version(name: str) -> Optional[str]: + from importlib import metadata + + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + +def _default_find_spec(name: str) -> Optional[object]: + try: + return importlib.util.find_spec(name) + except (ImportError, ValueError): + return None + + +def _is_supported(version: Optional[str]) -> bool: + if not version: + return False + return version == SUPPORTED_SERIES or version.startswith(SUPPORTED_SERIES + ".") + + +def check_autogen_classic( + *, + distributions_for: Callable[[str], Iterable[str]] = _default_distributions_for, + distribution_version: Callable[[str], Optional[str]] = _default_distribution_version, + import_module: Callable[[str], object] = importlib.import_module, + find_spec: Callable[[str], Optional[object]] = _default_find_spec, +) -> str: + """Return the installed AG2 Classic version, or raise. + + Raises :class:`AG2ClassicCompatibilityError` when: + + * the top-level ``autogen`` module is installed by ``pyautogen``; + * it is installed by ``ag2``, except the ``autogen==0.14.x`` alias layout + (``autogen`` 0.14.x distribution plus ``ag2`` 0.14.x); + * ``autogen`` cannot be imported (for example only ``ag2`` 1.x or + Microsoft ``autogen-agentchat`` is installed); + * ``autogen.version.__version__`` or ``autogen.opentelemetry`` is missing; + * ``autogen.version.__version__`` does not start with ``0.14``. + + The keyword arguments exist so tests can supply a fake module layout + without installing any of the other packages. + """ + owners = sorted({_normalize(d) for d in distributions_for("autogen")}) + if "pyautogen" in owners: + raise AG2ClassicCompatibilityError("The top-level `autogen` module is installed by `pyautogen`.") + if "ag2" in owners: + ag2_version = distribution_version("ag2") + alias_version = distribution_version("autogen") + if not (_is_supported(ag2_version) and _is_supported(alias_version)): + raise AG2ClassicCompatibilityError( + "The top-level `autogen` module is installed by `ag2` {0} (`autogen` distribution: {1}).".format( + ag2_version or "unknown", alias_version or "not installed" + ) + ) + + try: + import_module("autogen") + except ImportError as error: + hints = [] + if find_spec("ag2") is not None: + hints.append("`ag2` (AG2 1.x, `import ag2`) is installed") + if find_spec("autogen_agentchat") is not None: + hints.append("Microsoft AutoGen (`autogen_agentchat`) is installed") + hint = (" " + "; ".join(hints) + ".") if hints else "" + raise AG2ClassicCompatibilityError("`autogen` is not importable ({0}).{1}".format(error, hint)) from error + + try: + version_module = import_module("autogen.version") + version = str(getattr(version_module, "__version__")) + except (ImportError, AttributeError) as error: + raise AG2ClassicCompatibilityError( + "`autogen.version.__version__` was not found, so this is not the AG2 Classic layout." + ) from error + + if find_spec("autogen.opentelemetry") is None: + raise AG2ClassicCompatibilityError("`autogen` {0} has no `autogen.opentelemetry` module.".format(version)) + + if not _is_supported(version): + raise AG2ClassicCompatibilityError( + "`autogen` {0} is installed; only {1}.x is supported.".format(version, SUPPORTED_SERIES) + ) + + return version diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py new file mode 100644 index 000000000..5aa6c15df --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -0,0 +1,443 @@ +"""Span processor that normalizes AG2 Classic spans to Future AGI conventions. + +``autogen`` 0.14.x ships its own OpenTelemetry instrumentation +(``autogen.opentelemetry``). Its spans already use most GenAI semantic +convention keys. This processor adds only what Future AGI needs and enforces +the content default. Every key below was read from the installed +``autogen/opentelemetry/instrumentators/`` at 0.14.1 (see README "Attribute +inventory" for file:line evidence). + +What it does, per span on the ``opentelemetry.instrumentation.ag2`` scope: + +* ``gen_ai.span.kind`` from upstream ``ag2.span.type`` (the ``SpanType`` enum). +* ``session.id`` from ``gen_ai.conversation.id`` on the outermost + ``conversation`` span only. Nested chats (for example group-chat speaker + selection, or an inner chat a tool starts inside the user's own span) get + their own ``chat_id`` upstream; those are not copied, so one run maps to one + session. The ancestor walk follows every live span, AG2 or not. +* Future AGI context attributes (``using_session``, ``using_user``, + ``using_attributes``: ``session.id``, ``user.id``, ``metadata``, ...) copied + onto AG2 spans when they start; a context ``session.id`` wins over the + conversation id. +* ``gen_ai.cost.total`` from upstream ``gen_ai.usage.cost`` on LLM spans when + it is above 0, and ``gen_ai.usage.total_tokens`` = input + output on LLM + spans. Upstream reports 0 for any model missing from its price table, and a + ``gen_ai.cost.total`` of 0 would stop Future AGI from pricing the call itself. +* ``gen_ai.request.parameters`` JSON from upstream ``gen_ai.request.*`` + sampling keys. +* Status ``ERROR`` when upstream set ``error.type`` but left the status unset + (tool failures are caught inside ``ConversableAgent.execute_function``, so + upstream never sets the status itself). +* Content follows the traceAI ``TraceConfig`` convention: captured by default, + hidden with ``FI_HIDE_INPUTS`` / ``FI_HIDE_OUTPUTS`` (and the message/text + variants). Upstream agent, conversation, tool, human-input and + code-execution spans always record message bodies, tool arguments/results, + prompts and code output; there is no upstream flag for those, so when a side + is hidden this processor removes its keys before export. Kept content is + lifted into ``input.value`` / ``output.value``. + +Spans from any other instrumentation scope pass through untouched. The +processor never raises into the SDK. +""" + +from __future__ import annotations + +import json +import logging +import threading +from typing import Any, Dict, Mapping, Optional, Tuple + +from opentelemetry import context as context_api +from opentelemetry.context import Context +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor +from opentelemetry.trace import Status, StatusCode + +from fi_instrumentation.fi_types import FiMimeTypeValues, FiSpanKindValues, SpanAttributes +from fi_instrumentation import get_attributes_from_context + +logger = logging.getLogger(__name__) + +# Read from autogen/opentelemetry/consts.py at 0.14.1 (INSTRUMENTING_MODULE_NAME). +AG2_SCOPE = "opentelemetry.instrumentation.ag2" +# Set on every upstream span by the instrumentators (e.g. llm_wrapper.py:74). +SPAN_TYPE_KEY = "ag2.span.type" + +# SpanType values (consts.py:13-22) -> Future AGI span kind. +KIND_BY_SPAN_TYPE: Dict[str, str] = { + "llm": FiSpanKindValues.LLM.value, + "agent": FiSpanKindValues.AGENT.value, + "tool": FiSpanKindValues.TOOL.value, + "code_execution": FiSpanKindValues.TOOL.value, + "conversation": FiSpanKindValues.CHAIN.value, + "multi_conversation": FiSpanKindValues.CHAIN.value, + "speaker_selection": FiSpanKindValues.CHAIN.value, + "human_input": FiSpanKindValues.CHAIN.value, + # Declared upstream with a TODO and never emitted at 0.14.1. Kept as CHAIN. + "handoff": FiSpanKindValues.CHAIN.value, +} + +# Upstream keys (verbatim strings from the instrumentators). +INPUT_MESSAGES = "gen_ai.input.messages" +OUTPUT_MESSAGES = "gen_ai.output.messages" +TOOL_ARGUMENTS = "gen_ai.tool.call.arguments" +TOOL_RESULT = "gen_ai.tool.call.result" +HUMAN_PROMPT = "ag2.human_input.prompt" +HUMAN_RESPONSE = "ag2.human_input.response" +CODE_OUTPUT = "ag2.code_execution.output" +CHAT_SUMMARIES = "ag2.chats.summaries" +CONVERSATION_ID = "gen_ai.conversation.id" +UPSTREAM_COST = "gen_ai.usage.cost" +ERROR_TYPE = "error.type" +INPUT_TOKENS = "gen_ai.usage.input_tokens" +OUTPUT_TOKENS = "gen_ai.usage.output_tokens" +USAGE_PREFIX = "gen_ai.usage." + +# Keys that carry user (input side) or model/tool (output side) content. +# Removed when the matching side is hidden. +INPUT_CONTENT_KEYS = ( + INPUT_MESSAGES, + TOOL_ARGUMENTS, + HUMAN_PROMPT, + SpanAttributes.INPUT_VALUE, + SpanAttributes.INPUT_MIME_TYPE, +) +OUTPUT_CONTENT_KEYS = ( + OUTPUT_MESSAGES, + TOOL_RESULT, + HUMAN_RESPONSE, + CODE_OUTPUT, + CHAT_SUMMARIES, + SpanAttributes.OUTPUT_VALUE, + SpanAttributes.OUTPUT_MIME_TYPE, +) +CONTENT_KEYS = INPUT_CONTENT_KEYS + OUTPUT_CONTENT_KEYS + + +def resolve_hidden( + capture_content: Optional[bool], hide_inputs: bool = False, hide_outputs: bool = False +) -> Tuple[bool, bool]: + """Return ``(hide_inputs, hide_outputs)``. + + An explicit ``capture_content`` wins: True shows both sides, False hides + both. ``None`` uses ``hide_inputs`` / ``hide_outputs`` (traceAI default: + both shown). + """ + if capture_content is None: + return bool(hide_inputs), bool(hide_outputs) + return (not capture_content), (not capture_content) + +# utils.py set_llm_request_params (lines 213-219). +REQUEST_PARAMETER_KEYS = ( + "gen_ai.request.temperature", + "gen_ai.request.max_tokens", + "gen_ai.request.top_p", + "gen_ai.request.frequency_penalty", + "gen_ai.request.presence_penalty", +) + +# Only a live ``conversation`` ancestor makes a conversation "nested". Chats +# started by ``initiate_chats`` (``multi_conversation``) stay roots: each one is +# its own upstream chat with its own ``chat_id``. +_CONVERSATION_TYPES = ("conversation",) + +# Upper bound on live spans tracked (AG2 or not) for the nested-conversation +# check, so a span that is started and never ended cannot grow memory without +# limit. Spans past the bound are not tracked; an ancestor walk stops there. +_MAX_LIVE_SPANS = 10000 + + +def kind_for_span_type(span_type: Any) -> Optional[str]: + """Return the Future AGI span kind for an upstream ``ag2.span.type`` value.""" + if not isinstance(span_type, str): + return None + return KIND_BY_SPAN_TYPE.get(span_type) + + +def _json_mime(value: str) -> str: + stripped = value.strip() + if (stripped.startswith("{") and stripped.endswith("}")) or ( + stripped.startswith("[") and stripped.endswith("]") + ): + return FiMimeTypeValues.JSON.value + return FiMimeTypeValues.TEXT.value + + +def _set_io(mapped: Dict[str, Any], value_key: str, mime_key: str, value: Any) -> None: + if value is None or value_key in mapped: + return + text = value if isinstance(value, str) else json.dumps(value, default=str) + mapped[value_key] = text + mapped[mime_key] = _json_mime(text) + + +def _surface_content(mapped: Dict[str, Any], span_type: str) -> None: + if span_type == "tool": + _set_io(mapped, SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE, mapped.get(TOOL_ARGUMENTS)) + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(TOOL_RESULT)) + elif span_type == "human_input": + _set_io(mapped, SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE, mapped.get(HUMAN_PROMPT)) + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(HUMAN_RESPONSE)) + elif span_type == "code_execution": + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(CODE_OUTPUT)) + elif span_type == "multi_conversation": + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(CHAT_SUMMARIES)) + else: + _set_io(mapped, SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE, mapped.get(INPUT_MESSAGES)) + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(OUTPUT_MESSAGES)) + + +def _bundle_request_parameters(mapped: Dict[str, Any]) -> None: + if SpanAttributes.GEN_AI_REQUEST_PARAMETERS in mapped: + return + params = { + key.split("gen_ai.request.", 1)[1]: mapped[key] for key in REQUEST_PARAMETER_KEYS if key in mapped + } + if params: + mapped[SpanAttributes.GEN_AI_REQUEST_PARAMETERS] = json.dumps(params) + + +def map_ag2_attributes( + attributes: Mapping[str, Any], + *, + capture_content: Optional[bool] = None, + hide_inputs: bool = False, + hide_outputs: bool = False, + root_conversation: bool = True, +) -> Dict[str, Any]: + """Return a new attribute dict with Future AGI keys added. + + ``root_conversation`` is False for a ``conversation`` span nested inside + another live AG2 conversation; its ``gen_ai.conversation.id`` is then not + copied to ``session.id``. + + Attributes without an upstream ``ag2.span.type`` are returned unchanged. + """ + mapped = dict(attributes or {}) + span_type = mapped.get(SPAN_TYPE_KEY) + kind = kind_for_span_type(span_type) + if kind is None: + return mapped + + mapped[SpanAttributes.GEN_AI_SPAN_KIND] = kind + + conversation_id = mapped.get(CONVERSATION_ID) + if ( + span_type == "conversation" + and root_conversation + and conversation_id + and SpanAttributes.SESSION_ID not in mapped + ): + mapped[SpanAttributes.SESSION_ID] = str(conversation_id) + + if kind == FiSpanKindValues.LLM.value: + cost = mapped.get(UPSTREAM_COST) + # 0 means "not in AG2's price table" (oai/client.py logs "The cost will + # be 0"); leave the cost to Future AGI's own pricing then. + if isinstance(cost, (int, float)) and cost > 0 and SpanAttributes.GEN_AI_COST_TOTAL not in mapped: + mapped[SpanAttributes.GEN_AI_COST_TOTAL] = cost + inp = mapped.get(INPUT_TOKENS) + out = mapped.get(OUTPUT_TOKENS) + if ( + isinstance(inp, int) + and isinstance(out, int) + and SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS not in mapped + ): + mapped[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = inp + out + _bundle_request_parameters(mapped) + else: + # fi-collector promotes token keys into the token columns on any span + # (adapter.go inputTokenKeys/outputTokenKeys/totalTokenKeys) and Observe + # sums total_tokens over a trace. Only LLM spans carry per-call usage; + # the conversation span's chat-wide sum (chat.py:73-82) would double it. + # Every gen_ai.usage.* key, including upstream's aggregate + # gen_ai.usage.cost, moves to ag2.usage.* on non-LLM spans. + for key in [k for k in mapped if k.startswith(USAGE_PREFIX)]: + mapped.setdefault("ag2.usage." + key[len(USAGE_PREFIX) :], mapped.pop(key)) + + hide_in, hide_out = resolve_hidden(capture_content, hide_inputs, hide_outputs) + if hide_in: + for key in INPUT_CONTENT_KEYS: + mapped.pop(key, None) + if hide_out: + for key in OUTPUT_CONTENT_KEYS: + mapped.pop(key, None) + if not (hide_in and hide_out): + _surface_content(mapped, str(span_type)) + + return mapped + + +def _replace_attributes(span: ReadableSpan, mapped: Dict[str, Any]) -> None: + """Write ``mapped`` back onto the ended span so later processors see it. + + ``ReadableSpan.attributes`` is a read-only view over ``span._attributes``. + The SDK passes one ReadableSpan object to every processor in order, so + replacing ``_attributes`` here is visible to the exporting processor that + runs after this one. ``BoundedAttributes`` limits are preserved. + """ + old = getattr(span, "_attributes", None) + try: + from opentelemetry.attributes import BoundedAttributes + + if isinstance(old, BoundedAttributes): + new = BoundedAttributes( + maxlen=old.maxlen, + attributes=mapped, + immutable=True, + max_value_len=old.max_value_len, + ) + new.dropped = old.dropped + span._attributes = new # type: ignore[attr-defined] + return + except Exception: # pragma: no cover - fall back to a plain mapping + pass + span._attributes = mapped # type: ignore[attr-defined] + + +class AG2ClassicSpanProcessor(SpanProcessor): + """Normalize ``autogen.opentelemetry`` spans before they are exported. + + Install it ahead of the exporting processor (``setup()`` does this), so the + exporter sees the mapped attributes. + """ + + def __init__( + self, + capture_content: Optional[bool] = None, + *, + hide_inputs: bool = False, + hide_outputs: bool = False, + ) -> None: + self.hide_inputs, self.hide_outputs = resolve_hidden(capture_content, hide_inputs, hide_outputs) + # span_id -> (parent span_id, the span if it is an AG2 span else None), + # for every live span on the provider, AG2 or not. A user span between + # two AG2 conversations must not hide the outer one. + self._live: Dict[int, Tuple[Optional[int], Optional[Span]]] = {} + self._lock = threading.Lock() + self._disabled = False + + # SpanProcessor interface ---------------------------------------------- + + def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None: + if self._disabled: + return + try: + ag2 = _is_ag2(span) + parent = span.parent + entry = (parent.span_id if parent is not None else None, span if ag2 else None) + with self._lock: + if len(self._live) < _MAX_LIVE_SPANS: + self._live[span.context.span_id] = entry + if ag2: + _apply_context_attributes(span, parent_context) + except Exception: # pragma: no cover - never break the SDK + return + + def on_end(self, span: ReadableSpan) -> None: + if self._disabled: + return + try: + with self._lock: + self._live.pop(span.context.span_id, None) + if not _is_ag2(span): + return + attributes = dict(span.attributes or {}) + root = True + if attributes.get(SPAN_TYPE_KEY) == "conversation": + root = not self._has_live_conversation_ancestor(span) + mapped = map_ag2_attributes( + attributes, + hide_inputs=self.hide_inputs, + hide_outputs=self.hide_outputs, + root_conversation=root, + ) + if mapped != attributes: + _replace_attributes(span, mapped) + + error_type = attributes.get(ERROR_TYPE) + if error_type and span.status.status_code is StatusCode.UNSET: + span._status = Status(StatusCode.ERROR, str(error_type)) # type: ignore[attr-defined] + except Exception: # pragma: no cover - never break the SDK + logger.debug("traceai-ag2-classic: span mapping failed", exc_info=True) + + @property + def capture_content(self) -> bool: + """True when any content side is still exported.""" + return not (self.hide_inputs and self.hide_outputs) + + def shutdown(self) -> None: + self._disabled = True + try: + with self._lock: + self._live.clear() + except Exception: # pragma: no cover - never break SDK shutdown + pass + + def _reopen(self) -> None: + """Undo :meth:`shutdown` after Future AGI's ``add_span_processor`` ran. + + That method shuts down every processor on the provider, not only its + own default exporter; ``setup()`` re-installs this one and calls this. + """ + self._disabled = False + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True + + # Internals -------------------------------------------------------------- + + def _has_live_conversation_ancestor(self, span: ReadableSpan) -> bool: + parent = span.parent + parent_id = parent.span_id if parent is not None else None + seen = 0 + with self._lock: + while parent_id is not None and seen < 256: + entry = self._live.get(parent_id) + if entry is None: + return False + parent_id, ag2_span = entry + if ag2_span is not None: + attrs = ag2_span.attributes or {} + if attrs.get(SPAN_TYPE_KEY) in _CONVERSATION_TYPES: + return True + seen += 1 + return False + + +def _is_ag2(span: Any) -> bool: + scope = getattr(span, "instrumentation_scope", None) + return getattr(scope, "name", None) == AG2_SCOPE + + +def _context_attributes(parent_context: Optional[Context]) -> Dict[str, Any]: + """Return Future AGI context attributes (``using_attributes`` & co.). + + Upstream spans come from a plain OpenTelemetry tracer, which never reads + these; only Future AGI's own tracer applies them + (``fi_instrumentation/instrumentation/_tracers.py:165``). + """ + if parent_context is None: + return dict(get_attributes_from_context()) + token = context_api.attach(parent_context) + try: + return dict(get_attributes_from_context()) + finally: + context_api.detach(token) + + +def _apply_context_attributes(span: Span, parent_context: Optional[Context]) -> None: + """Copy context attributes onto a starting AG2 span. + + Keys upstream already set are left alone. A ``session.id`` from + ``using_session`` / ``using_attributes`` is therefore on the span before + it ends, and wins over the ``gen_ai.conversation.id`` copy in + :func:`map_ag2_attributes`. Upstream sets the conversation span's + ``gen_ai.conversation.id`` later, so it keeps the real chat id. + """ + values = _context_attributes(parent_context) + if not values: + return + existing = span.attributes or {} + missing = {key: value for key, value in values.items() if key not in existing} + if missing: + span.set_attributes(missing) diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py new file mode 100644 index 000000000..fd6925d05 --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -0,0 +1,368 @@ +"""One-call enabler for AG2 Classic's built-in ``autogen.opentelemetry``. + +``setup()`` does four things, in order: + +1. Checks that the importable ``autogen`` is AG2 Classic 0.14.x (``_guard``). +2. Prepends :class:`AG2ClassicSpanProcessor` to the provider you pass, ahead + of the exporting processor ``fi_instrumentation.register()`` installed. +3. Calls upstream ``instrument_llm_wrapper`` (global, patches + ``OpenAIWrapper.create``) with ``capture_messages`` set when any content side is shown. +4. Calls upstream ``instrument_agent`` / ``instrument_pattern`` for the agents + and patterns you pass. + +It never creates a ``TracerProvider``, never sets the global provider, and +never constructs an exporter (in particular not the gRPC exporter the +``autogen[tracing]`` extra installs). One provider, the one you pass. +""" + +from __future__ import annotations + +import logging +from typing import Any, Iterable, Optional + +from opentelemetry import trace as trace_api + +from ._guard import check_autogen_classic +from fi_instrumentation import TraceConfig + +from ._processor import AG2ClassicSpanProcessor, resolve_hidden + +logger = logging.getLogger(__name__) + + +def _active_multi_processor(provider: Any) -> Any: + active = getattr(provider, "_active_span_processor", None) + if active is None or not hasattr(active, "_span_processors"): + raise TypeError( + "traceai_ag2_classic.setup() needs an OpenTelemetry SDK TracerProvider, for example " + "the one returned by fi_instrumentation.register(project_type=ProjectType.OBSERVE, " + "project_name=...). It does not create or register a provider itself. Got {0}.".format( + type(provider).__name__ + ) + ) + return active + + +def _warn_if_concurrent(active: Any) -> None: + """Warn when the provider runs its span processors in parallel. + + The SDK's ``ConcurrentMultiSpanProcessor`` calls every processor's + ``on_end`` at once on a thread pool, so an exporting processor can + serialize a span before this package has removed its content. Filtering + is only ordered on the default ``SynchronousMultiSpanProcessor``. + """ + from opentelemetry.sdk.trace import ConcurrentMultiSpanProcessor + + if isinstance(active, ConcurrentMultiSpanProcessor): + logger.warning( + "traceai-ag2-classic: this tracer provider uses ConcurrentMultiSpanProcessor, which runs " + "span processors in parallel, so an exporter may send a span before message content is " + "removed and attributes are mapped. Use a provider with the default " + "SynchronousMultiSpanProcessor (for example fi_instrumentation.register())." + ) + + +def _install_processor( + provider: Any, hide_inputs: bool, hide_outputs: bool +) -> AG2ClassicSpanProcessor: + """Prepend our processor so it runs before the exporting processor. + + ``provider.add_span_processor`` is deliberately not used: Future AGI's + ``TracerProvider`` drops its default exporting processor on the first + ``add_span_processor`` call after ``register()``, which would lose spans. + Prepending also guarantees the exporter sees the mapped attributes, and + that the ``error.type`` -> ERROR promotion runs before Future AGI's + exporting processor turns every UNSET status into OK + (``fi_instrumentation/otel.py`` ``BatchSpanProcessor.on_end``). + + The provider instance's ``add_span_processor`` is wrapped once (see + :func:`_guard_add_span_processor`) so a later call by the user keeps this + processor installed, first and enabled. + """ + active = _active_multi_processor(provider) + _warn_if_concurrent(active) + processor = getattr(getattr(provider, "add_span_processor", None), _GUARD_ATTR, None) + if not isinstance(processor, AG2ClassicSpanProcessor): + processor = next( + (p for p in tuple(active._span_processors) if isinstance(p, AG2ClassicSpanProcessor)), + None, + ) + if processor is not None: + if (processor.hide_inputs, processor.hide_outputs) != (hide_inputs, hide_outputs): + logger.warning( + "traceai-ag2-classic is already installed on this provider with " + "hide_inputs=%s hide_outputs=%s; keeping that setting.", + processor.hide_inputs, + processor.hide_outputs, + ) + if processor not in tuple(active._span_processors): + _prepend(active, processor) + if processor._disabled: + # Shut down by a path that bypassed the add_span_processor guard + # (an unbound TracerProvider.add_span_processor(provider, p), or + # the multi-processor's shutdown()). A disabled processor skips + # filtering in on_end, so reusing it as is would export content. + processor._reopen() + else: + processor = AG2ClassicSpanProcessor(hide_inputs=hide_inputs, hide_outputs=hide_outputs) + _prepend(active, processor) + _guard_add_span_processor(provider, processor) + return processor + + +# Attribute on our add_span_processor wrapper; holds the provider's processor. +_GUARD_ATTR = "_traceai_ag2_classic_processor" + + +def _prepend(active: Any, processor: AG2ClassicSpanProcessor) -> None: + with getattr(active, "_lock", _NullLock()): + rest = tuple(p for p in active._span_processors if p is not processor) + active._span_processors = (processor,) + rest + + +def _guard_add_span_processor(provider: Any, processor: AG2ClassicSpanProcessor) -> None: + """Keep ``processor`` first and enabled across ``provider.add_span_processor``. + + Future AGI's ``TracerProvider.add_span_processor`` shuts down and clears + every processor while ``register()``'s default exporter is still in place + (``fi_instrumentation/otel.py:336-339``), ours included. Without this, + content would be exported and conversation spans would keep the promoted + token keys after a user adds their own processor. Only this provider + instance is wrapped, once. + """ + current = getattr(provider, "add_span_processor", None) + if current is None or getattr(current, _GUARD_ATTR, None) is not None: + return + + def add_span_processor(*args: Any, **kwargs: Any) -> Any: + was_enabled = not processor._disabled + result = current(*args, **kwargs) + try: + if was_enabled and processor._disabled: + processor._reopen() + _prepend(_active_multi_processor(provider), processor) + except Exception: # pragma: no cover - never break the caller's add + logger.warning("traceai-ag2-classic could not re-install its span processor.", exc_info=True) + return result + + setattr(add_span_processor, _GUARD_ATTR, processor) + add_span_processor.__doc__ = getattr(current, "__doc__", None) + try: + provider.add_span_processor = add_span_processor + except Exception: # pragma: no cover - provider without instance attributes + logger.warning( + "traceai-ag2-classic could not guard %s.add_span_processor; calling it after setup() " + "may remove this package's span processor.", + type(provider).__name__, + ) + + +class _NullLock: + def __enter__(self) -> None: + return None + + def __exit__(self, *_exc: Any) -> None: + return None + + +class AG2ClassicTracing: + """Handle returned by :func:`setup`. + + Use it to instrument agents or patterns created after ``setup()``. + """ + + def __init__( + self, + *, + tracer_provider: Any, + processor: AG2ClassicSpanProcessor, + autogen_version: str, + capture_content: bool, + ) -> None: + self.tracer_provider = tracer_provider + self.processor = processor + self.autogen_version = autogen_version + self.capture_content = capture_content + self._original_create: Any = None + self._traced_create: Any = None + self.owns_llm_wrapper = False + + # Upstream calls --------------------------------------------------------- + + def _instrument_llm_wrapper(self) -> None: + from autogen.oai.client import OpenAIWrapper + from autogen.opentelemetry import instrument_llm_wrapper + + current = OpenAIWrapper.create + if hasattr(current, "__otel_wrapped__"): + # Upstream returns early when already wrapped (llm_wrapper.py:64-65), + # so LLM spans keep going to whichever provider wrapped it first. + logger.warning( + "autogen OpenAIWrapper.create is already instrumented; LLM spans keep using the " + "tracer provider and capture setting from that earlier call." + ) + return + self._original_create = current + instrument_llm_wrapper( + tracer_provider=self.tracer_provider, capture_messages=self.capture_content + ) + self._traced_create = OpenAIWrapper.create + self.owns_llm_wrapper = True + + def instrument_agent(self, agent: Any) -> Any: + """Call upstream ``instrument_agent`` on ``agent``. + + For a ``GroupChatManager`` it also instruments every agent in its group + chat and the group chat itself (speaker-selection spans), mirroring what + upstream ``instrument_pattern`` does for patterns (pattern.py:93-104). + """ + from autogen.agentchat.groupchat import GroupChat, GroupChatManager + from autogen.opentelemetry import instrument_agent + from autogen.opentelemetry.instrumentators.pattern import instrument_groupchat + + provider = self.tracer_provider + instrument_agent(agent, tracer_provider=provider) + if isinstance(agent, GroupChatManager): + groupchat = agent.groupchat + for member in list(getattr(groupchat, "agents", []) or []): + instrument_agent(member, tracer_provider=provider) + instrument_groupchat(groupchat, tracer_provider=provider) + # GroupChatManager.__init__ registers a shallow copy of the group chat + # in _reply_func_list; instrument that copy too (pattern.py:98-104). + for entry in getattr(agent, "_reply_func_list", []) or []: + config = entry.get("config") if isinstance(entry, dict) else None + if isinstance(config, GroupChat) and config is not groupchat: + instrument_groupchat(config, tracer_provider=provider) + return agent + + def instrument_pattern(self, pattern: Any) -> Any: + """Call upstream ``instrument_pattern`` on ``pattern``.""" + from autogen.opentelemetry import instrument_pattern + + return instrument_pattern(pattern, tracer_provider=self.tracer_provider) + + def instrument_a2a_server(self, server: Any) -> Any: + """Call upstream ``instrument_a2a_server``. Optional; only if you ask. + + Upstream exports it only when its imports succeed (``autogen[a2a]``). + """ + try: + from autogen.opentelemetry import instrument_a2a_server # type: ignore[attr-defined] + except ImportError as error: + raise ImportError( + "autogen.opentelemetry.instrument_a2a_server is not available; install the " + "`autogen[a2a]` extra to trace an A2A server." + ) from error + return instrument_a2a_server(server, tracer_provider=self.tracer_provider) + + # Teardown --------------------------------------------------------------- + + def uninstrument(self) -> None: + """Restore ``OpenAIWrapper.create`` if this handle patched it. + + The span processor stays on the provider. Upstream patches agents per + instance and offers no undo, so agents instrumented earlier keep + emitting spans on this provider; the processor keeps dropping their + content and moving their aggregate usage off non-LLM spans. Every + handle on a provider shares that one processor, so removing it here + would also unfilter the other handles. It is shut down with the + provider (``provider.shutdown()``). + + ``OpenAIWrapper.create`` is restored only while it is still the + traced function upstream installed for this handle. If something else + replaced it after ``setup()`` (another library's wrapper around ours), + it is left in place and a warning is logged; call ``uninstrument()`` + again once that wrapper is gone. + """ + if self.owns_llm_wrapper and self._original_create is not None: + from autogen.oai import client as oai_client_module + from autogen.oai.client import OpenAIWrapper + + if OpenAIWrapper.create is not self._traced_create: + logger.warning( + "traceai-ag2-classic: OpenAIWrapper.create was replaced after setup(); leaving that " + "replacement in place. Call uninstrument() again after it is removed." + ) + return + OpenAIWrapper.create = self._original_create + oai_client_module.OpenAIWrapper.create = self._original_create + self.owns_llm_wrapper = False + self._original_create = None + self._traced_create = None + + +def _hidden_from_config( + config: Optional[TraceConfig], capture_content: Optional[bool] +) -> "tuple[bool, bool]": + if capture_content is not None: + return resolve_hidden(capture_content) + cfg = config if config is not None else TraceConfig() + return ( + bool(cfg.hide_inputs or cfg.hide_input_messages or cfg.hide_input_text), + bool(cfg.hide_outputs or cfg.hide_output_messages or cfg.hide_output_text), + ) + + +def setup( + tracer_provider: Optional[Any] = None, + *, + agents: Iterable[Any] = (), + patterns: Iterable[Any] = (), + config: Optional[TraceConfig] = None, + capture_content: Optional[bool] = None, + instrument_llm: bool = True, +) -> AG2ClassicTracing: + """Enable AG2 Classic's native tracing on a Future AGI tracer provider. + + Args: + tracer_provider: The provider returned by ``fi_instrumentation.register``. + Defaults to the global provider, which must be an SDK provider. + agents: Agents to pass to upstream ``instrument_agent``. + patterns: Group patterns to pass to upstream ``instrument_pattern``. + config: A ``fi_instrumentation.TraceConfig``. Defaults to + ``TraceConfig()``, which reads ``FI_HIDE_INPUTS`` / + ``FI_HIDE_OUTPUTS`` / ``FI_HIDE_INPUT_MESSAGES`` / + ``FI_HIDE_OUTPUT_MESSAGES`` / ``FI_HIDE_INPUT_TEXT`` / + ``FI_HIDE_OUTPUT_TEXT``. Content (message bodies, tool + arguments/results, human input, code output) is captured unless + hidden, like every traceAI instrumentor. + capture_content: Optional override of ``config``. ``False`` hides all + content, ``True`` keeps all of it. + instrument_llm: Call upstream ``instrument_llm_wrapper`` (LLM spans). + + Returns: + An :class:`AG2ClassicTracing` handle for agents created later. + + Raises: + AG2ClassicCompatibilityError: ``autogen`` is not AG2 Classic 0.14.x. + TypeError: The provider is not an OpenTelemetry SDK provider. + + If instrumenting an agent or pattern raises, ``setup()`` restores + ``OpenAIWrapper.create`` before re-raising, so no global patch is left + without a handle to undo it. + """ + version = check_autogen_classic() + provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + hide_inputs, hide_outputs = _hidden_from_config(config, capture_content) + processor = _install_processor(provider, hide_inputs, hide_outputs) + handle = AG2ClassicTracing( + tracer_provider=provider, + processor=processor, + autogen_version=version, + capture_content=processor.capture_content, + ) + if instrument_llm: + handle._instrument_llm_wrapper() + try: + for agent in agents: + handle.instrument_agent(agent) + for pattern in patterns: + handle.instrument_pattern(pattern) + except BaseException: + # The caller never receives the handle, so undo the global + # OpenAIWrapper.create patch here. The processor stays (see + # AG2ClassicTracing.uninstrument): agents instrumented before the + # failure keep emitting spans and still need filtering. + handle.uninstrument() + raise + return handle diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/version.py b/python/frameworks/ag2-classic/traceai_ag2_classic/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out