From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/22] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/22] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/22] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 8bfc3de7d6b2f9792b1c8581864d1523b8ff2529 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:43:21 -0700 Subject: [PATCH 04/22] feat(ag2-classic): add traceAI-ag2-classic native enabler for autogen 0.14.x New package python/frameworks/ag2-classic (traceAI-ag2-classic). setup() enables AG2 Classic's built-in autogen.opentelemetry (instrument_llm_wrapper, instrument_agent, instrument_pattern) on the Future AGI tracer provider from register(); it never creates a provider, sets the global provider or adds an exporter. AG2ClassicSpanProcessor is prepended ahead of the exporting processor and, from keys read in autogen/opentelemetry/instrumentators at 0.14.1: - maps ag2.span.type to gen_ai.span.kind (handoff -> CHAIN, never emitted) - copies the outermost gen_ai.conversation.id to session.id - adds gen_ai.cost.total (from gen_ai.usage.cost) and gen_ai.usage.total_tokens on LLM spans - sets ERROR status from upstream error.type (tool failures are caught inside execute_function, so upstream never sets status) - drops message/tool/human-input/code-output content unless capture_content=True (upstream has no flag outside the LLM wrapper) Version guard rejects non-0.14 autogen, Microsoft AutoGen, ag2 1.x and pyautogen layouts, and accepts the autogen==0.14.0 alias backed by ag2==0.14.0. Tests: mapping and guard unit tests, in-process real autogen runs against a loopback fake OpenAI server, and a shared-harness contract test over the real OTLP/HTTP exporter. Refs: TH-8237 --- python/frameworks/ag2-classic/CHANGELOG.md | 14 + python/frameworks/ag2-classic/README.md | 163 ++++++++ .../ag2-classic/examples/basic_chat.py | 47 +++ python/frameworks/ag2-classic/pyproject.toml | 47 +++ .../ag2-classic/tests/_contract_app.py | 59 +++ .../ag2-classic/tests/_fake_openai.py | 137 +++++++ .../ag2-classic/tests/_scenarios.py | 91 +++++ .../tests/test_contract_harness.py | 210 ++++++++++ .../ag2-classic/tests/test_guard.py | 135 +++++++ .../ag2-classic/tests/test_processor.py | 267 +++++++++++++ .../ag2-classic/tests/test_setup.py | 365 ++++++++++++++++++ .../traceai_ag2_classic/__init__.py | 34 ++ .../ag2-classic/traceai_ag2_classic/_guard.py | 150 +++++++ .../traceai_ag2_classic/_processor.py | 328 ++++++++++++++++ .../ag2-classic/traceai_ag2_classic/_setup.py | 234 +++++++++++ .../traceai_ag2_classic/version.py | 1 + 16 files changed, 2282 insertions(+) create mode 100644 python/frameworks/ag2-classic/CHANGELOG.md create mode 100644 python/frameworks/ag2-classic/README.md create mode 100644 python/frameworks/ag2-classic/examples/basic_chat.py create mode 100644 python/frameworks/ag2-classic/pyproject.toml create mode 100644 python/frameworks/ag2-classic/tests/_contract_app.py create mode 100644 python/frameworks/ag2-classic/tests/_fake_openai.py create mode 100644 python/frameworks/ag2-classic/tests/_scenarios.py create mode 100644 python/frameworks/ag2-classic/tests/test_contract_harness.py create mode 100644 python/frameworks/ag2-classic/tests/test_guard.py create mode 100644 python/frameworks/ag2-classic/tests/test_processor.py create mode 100644 python/frameworks/ag2-classic/tests/test_setup.py create mode 100644 python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py create mode 100644 python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py create mode 100644 python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py create mode 100644 python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py create mode 100644 python/frameworks/ag2-classic/traceai_ag2_classic/version.py diff --git a/python/frameworks/ag2-classic/CHANGELOG.md b/python/frameworks/ag2-classic/CHANGELOG.md new file mode 100644 index 000000000..40fde0c86 --- /dev/null +++ b/python/frameworks/ag2-classic/CHANGELOG.md @@ -0,0 +1,14 @@ +## [0.1.0] - 2026-10-04 +### Features +- New package for AG2 Classic (`autogen` 0.14.x). `setup()` enables upstream + `autogen.opentelemetry` (`instrument_llm_wrapper`, `instrument_agent`, + `instrument_pattern`) on the Future AGI tracer provider, without creating a + second provider or exporter. +- `AG2ClassicSpanProcessor` maps `ag2.span.type` to `gen_ai.span.kind`, copies + the outermost `gen_ai.conversation.id` to `session.id`, adds + `gen_ai.cost.total` and `gen_ai.usage.total_tokens` on LLM spans, and sets + ERROR status from upstream `error.type`. +- Content off by default: upstream message, tool, human-input and code-output + keys are removed unless `capture_content=True`. +- Version guard: rejects non-0.14 `autogen`, Microsoft AutoGen, `ag2` 1.x and + `pyautogen` layouts with an error that names the matching package. diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md new file mode 100644 index 000000000..7ebe8f024 --- /dev/null +++ b/python/frameworks/ag2-classic/README.md @@ -0,0 +1,163 @@ +# traceAI-ag2-classic + +Send AG2 Classic traces to Future AGI. AG2 Classic is the `autogen` distribution +at 0.14.x, imported as `autogen`, from `ag2ai/ag2classic`. It ships its own +OpenTelemetry instrumentation, `autogen.opentelemetry`. This package turns that +on against a Future AGI tracer provider and maps its span attributes to Future +AGI conventions. Alpha. Validated against fixtures and a loopback fake model; no +live vendor run is claimed. + +| You installed | Import | Package | +|---|---|---| +| `autogen` 0.14.x | `autogen` | `traceAI-ag2-classic` (this package) | +| `ag2` 1.x | `ag2` | `traceAI-ag2` | +| `autogen-agentchat` | `autogen_agentchat` (Microsoft AutoGen) | `traceAI-autogen` | +| `pyautogen` | older alias | not supported | + +Supported: `autogen` 0.14.0 and 0.14.1. Upstream `ag2ai/ag2classic` at +`86fa5ccb3f9424cee87edb4a9ea58f191ea958ba`, Apache-2.0, copyright AG2ai. +`autogen` is a normal dependency (`>=0.14.0,<0.15`); it is never vendored. + +Note on 0.14.0: PyPI `autogen==0.14.0` is a small alias whose only requirement +is `ag2==0.14.0`, and the `autogen` module is shipped by that `ag2` 0.14.0 +distribution. Its `autogen/opentelemetry` tree is identical to 0.14.1's, so the +version guard accepts exactly that pairing. `autogen==0.14.1` ships the module +itself and installs no `ag2`. `ag2` 1.x ships only an `ag2` module. + +## Install + +```bash +pip install "autogen>=0.14.0,<0.15" +pip install traceAI-ag2-classic +``` + +`autogen[tracing]` also works. That extra adds `opentelemetry-sdk` and the OTLP +gRPC exporter; this package never constructs that exporter. + +```bash +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export FI_PROJECT_NAME="my-chatbot" +``` + +## Set up + +```python +from autogen import ConversableAgent +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_ag2_classic import setup + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="my-chatbot", +) + +assistant = ConversableAgent("assistant", llm_config={"config_list": [{"model": "gpt-4o-mini"}]}) +user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER") + +tracing = setup(tracer_provider=trace_provider, agents=[assistant, user]) +user.initiate_chat(assistant, message="Hello", max_turns=2) + +trace_provider.force_flush() # short scripts: flush before exit +``` + +Upstream instruments agents per instance, so pass every agent you want traced, +or call `tracing.instrument_agent(agent)` for agents created later. Pass a +`GroupChatManager` and its group chat members and speaker selection are +instrumented too. Pass group patterns with `patterns=[...]` or +`tracing.instrument_pattern(pattern)`. `tracing.instrument_a2a_server(server)` +is available only when upstream exports it (the `autogen[a2a]` extra); it is +never called unless you ask. + +`setup()`: + +1. Checks that `autogen` is AG2 Classic 0.14.x. It raises + `AG2ClassicCompatibilityError`, naming the packages above, when `autogen` + is missing (for example only `ag2` 1.x or Microsoft AutoGen is installed), + when the version is not 0.14.x, when `autogen.opentelemetry` is missing, or + when the `autogen` module comes from `pyautogen` or from `ag2` outside the + 0.14.0 alias pairing. +2. Puts `AG2ClassicSpanProcessor` first on the provider you pass, ahead of the + Future AGI exporter. +3. Calls upstream `instrument_llm_wrapper(capture_messages=capture_content)`, + then `instrument_agent` and `instrument_pattern` for what you pass. + +It never creates a provider, never sets the global provider, and never adds an +exporter. One provider, the one `register()` returned. HTTP to the collector is +the default; pass `transport=Transport.GRPC` (from `fi_instrumentation.otel`) to `register()` for gRPC. Do not +also install the `[tracing]` extra's gRPC exporter as a global provider, or +spans are duplicated. + +`tracing.uninstrument()` removes the processor and restores +`OpenAIWrapper.create`. Agents stay instrumented; upstream has no per-agent undo. + +## Conformance + +Every row was read from `autogen/opentelemetry/instrumentators/` at 0.14.1. +LLM, agent, tool, conversation, group-chat speaker selection and pattern spans +are exercised by real `autogen` runs against a loopback fake model in +`tests/`; code execution, human input, `initiate_chats` and remote-agent spans +are covered by mapping unit tests only. + +| Field | What you get | When it is omitted | +|---|---|---| +| Span kind | `gen_ai.span.kind` from upstream `ag2.span.type`: `llm` → `LLM`; `agent` → `AGENT`; `tool`, `code_execution` → `TOOL`; `conversation`, `multi_conversation`, `speaker_selection`, `human_input` → `CHAIN`; `handoff` → `CHAIN` (declared upstream as TODO, never emitted at 0.14.x). `ag2.span.type` and `gen_ai.operation.name` are kept. | The `a2a-execution` span from `instrument_a2a_server` has no `ag2.span.type` and gets no kind. | +| Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | +| Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans also carry upstream's aggregate input/output tokens for the whole chat; they are passed through unchanged and get no `total_tokens`, so do not sum them with the LLM spans. | +| Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). | Conversation spans keep only upstream `gen_ai.usage.cost` (aggregate). | +| Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. | Nested chats (for example the internal chat behind group-chat speaker selection) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. | +| User | Not emitted upstream. | Always. | +| Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | +| Errors | Exceptions keep OTel's ERROR status. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | +| Retrieval | Not emitted upstream. | Always. | + +## Privacy + +Content is off unless you opt in with `setup(..., capture_content=True)`. + +Upstream has a content flag only on `instrument_llm_wrapper` +(`capture_messages`, default False). Its agent, conversation, tool, +human-input and code-execution spans always record message bodies, tool +arguments and results, the human prompt and answer, code output and chat +summaries. With content off this package removes these keys before export: +`gen_ai.input.messages`, `gen_ai.output.messages`, `gen_ai.tool.call.arguments`, +`gen_ai.tool.call.result`, `ag2.human_input.prompt`, `ag2.human_input.response`, +`ag2.code_execution.output`, `ag2.chats.summaries`, and any `input.value` / +`output.value`. That is stricter than upstream. With content on they are kept +and lifted into `input.value` / `output.value`. + +## Attribute inventory (autogen 0.14.1) + +Paths are relative to `autogen/opentelemetry/`. Scope name +`opentelemetry.instrumentation.ag2`, schema +`https://opentelemetry.io/schemas/1.11.0` (`consts.py:25-26`). + +| Span (name) | `ag2.span.type` | Keys emitted | Source | +|---|---|---|---| +| `chat {model}` | `llm` | `gen_ai.operation.name=chat`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.agent.name`, `gen_ai.request.{temperature,max_tokens,top_p,frequency_penalty,presence_penalty}`, `gen_ai.response.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.response.finish_reasons`, `gen_ai.usage.cost`, `error.type`; messages only if `capture_messages` | `instrumentators/llm_wrapper.py:74-135,153`, `utils.py:213-222` | +| `invoke_agent {agent}` | `agent` | `gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.input.messages`, `gen_ai.output.messages` (always) | `instrumentators/agent_instrumentators/reply.py:36-58,75-95` | +| `invoke_agent {agent}` (remote) | `agent` | `gen_ai.agent.remote`, `server.address` | `instrumentators/agent_instrumentators/remote.py:43-48` | +| `conversation {agent}` | `conversation` | `gen_ai.operation.name=conversation`, `gen_ai.agent.name`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.conversation.max_turns`, `gen_ai.input.messages`, `gen_ai.conversation.id`, `gen_ai.conversation.turns`, `gen_ai.output.messages`, `gen_ai.usage.cost`, `gen_ai.response.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | `instrumentators/agent_instrumentators/chat.py:33-82,100-147` | +| `conversation {agent}` (resume, run_chat) | `conversation` | `gen_ai.conversation.resumed`; run_chat: messages | `instrumentators/agent_instrumentators/chat.py:167-170,191-237` | +| `agent.initiate_chats` / `initiate_chats` | `multi_conversation` | `gen_ai.operation.name=initiate_chats`, `ag2.chats.{count,mode,recipients,ids,summaries,prerequisites}` | `instrumentators/agent_instrumentators/chat.py:253-318` | +| `execute_tool {name}` | `tool` | `gen_ai.operation.name=execute_tool`, `gen_ai.tool.name`, `gen_ai.tool.type`, `gen_ai.tool.call.id`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, `error.type=ExecutionError` | `instrumentators/agent_instrumentators/tool.py:26-47,65-86` | +| `execute_code {agent}` | `code_execution` | `gen_ai.operation.name=execute_code`, `gen_ai.agent.name`, `ag2.code_execution.exit_code`, `ag2.code_execution.output`, `error.type=CodeExecutionError` | `instrumentators/agent_instrumentators/code.py:43-68` | +| `await_human_input {agent}` | `human_input` | `gen_ai.operation.name=await_human_input`, `gen_ai.agent.name`, `ag2.human_input.prompt`, `ag2.human_input.response` | `instrumentators/agent_instrumentators/human_input.py:24-32,48-55` | +| `speaker_selection` | `speaker_selection` | `gen_ai.operation.name=speaker_selection`, `ag2.speaker_selection.candidates`, `ag2.speaker_selection.selected` | `instrumentators/pattern.py:163-176,193-206` | +| `a2a-execution` | none | none | `instrumentators/a2a.py:61` | + +## Troubleshooting + +- No traces: `register()` needs `project_name` or `FI_PROJECT_NAME`; the + collector drops a batch whose resource has no `project_name`. +- No LLM spans after a second `setup()` on another provider: upstream patches + `OpenAIWrapper.create` once and keeps the first provider + (`llm_wrapper.py:64-65`). Call `uninstrument()` on the first handle first. +- Calling `trace_provider.add_span_processor(...)` after `register()` drops + Future AGI's default exporter (and this processor); that is + `fi_instrumentation` behaviour. `setup()` does not use it. +- `setup()` raises and names three packages: you have Microsoft AutoGen, `ag2` + 1.x, or `pyautogen` instead of `autogen` 0.14.x. +- Content missing: content is off by default here; pass `capture_content=True`. +- Exporter errors are logged and never fail the agent. diff --git a/python/frameworks/ag2-classic/examples/basic_chat.py b/python/frameworks/ag2-classic/examples/basic_chat.py new file mode 100644 index 000000000..c7092b862 --- /dev/null +++ b/python/frameworks/ag2-classic/examples/basic_chat.py @@ -0,0 +1,47 @@ +"""Two-agent AG2 Classic chat with a tool, traced to Future AGI. + +Requires FI_API_KEY, FI_SECRET_KEY and OPENAI_API_KEY in the environment. +This example calls a real model; the package tests use a loopback fake instead. +""" + +import os + +from autogen import ConversableAgent +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +from traceai_ag2_classic import setup + + +def get_weather(city: str) -> str: + """Return the weather for a city.""" + return f"sunny in {city}" + + +def main() -> None: + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name=os.getenv("FI_PROJECT_NAME", "ag2-classic-example"), + ) + + llm_config = {"config_list": [{"model": "gpt-4o-mini", "api_key": os.environ["OPENAI_API_KEY"]}]} + assistant = ConversableAgent( + "assistant", + llm_config=llm_config, + human_input_mode="NEVER", + is_termination_msg=lambda m: "TERMINATE" in (m.get("content") or ""), + system_message="Use the weather tool, answer, then say TERMINATE.", + ) + user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=3) + assistant.register_for_llm(description="Get the weather")(get_weather) + user.register_for_execution()(get_weather) + + # Content (messages, tool arguments/results) is off unless capture_content=True. + setup(tracer_provider=trace_provider, agents=[assistant, user]) + + user.initiate_chat(assistant, message="What is the weather in Paris?", max_turns=3) + trace_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/ag2-classic/pyproject.toml b/python/frameworks/ag2-classic/pyproject.toml new file mode 100644 index 000000000..29bc4df7e --- /dev/null +++ b/python/frameworks/ag2-classic/pyproject.toml @@ -0,0 +1,47 @@ +[tool.poetry] +name = "traceAI-ag2-classic" +version = "0.1.0" +description = "Future AGI tracing for AG2 Classic (autogen 0.14.x) through its built-in autogen.opentelemetry" +authors = ["Future AGI "] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://github.com/future-agi/traceAI" +repository = "https://github.com/future-agi/traceAI" +keywords = ["opentelemetry", "ag2", "autogen", "ag2-classic", "tracing", "observability", "agents"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Monitoring", +] +packages = [ + { include = "traceai_ag2_classic" }, +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +# AG2 Classic: the `autogen` distribution only. Never vendored or bundled. +# No dependency on `ag2` 1.x, `pyautogen` or `autogen-agentchat`. +autogen = ">=0.14.0,<0.15" +fi-instrumentation-otel = ">=1.0.0" +opentelemetry-api = ">=1.29.0" +opentelemetry-sdk = ">=1.29.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=8.0.0" +openai = ">=2.44.0,<3.0" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_functions = ["test_*"] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/ag2-classic/tests/_contract_app.py b/python/frameworks/ag2-classic/tests/_contract_app.py new file mode 100644 index 000000000..a8f301328 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/_contract_app.py @@ -0,0 +1,59 @@ +"""Contract app: real AG2 Classic + real OTel SDK + real Future AGI OTLP/HTTP exporter. + +Run as a subprocess by ``test_contract_harness.py``. The parent points +``FI_BASE_URL`` at a loopback harness ``Receiver``; ``register()`` then exports +OTLP/HTTP protobuf to ``{FI_BASE_URL}/tracer/v1/traces``. The model is a +loopback fake. Usage: ``python _contract_app.py off|on``. +""" + +from __future__ import annotations + +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +if HERE not in sys.path: + sys.path.insert(0, HERE) + + +def main(argv: list) -> int: + capture = len(argv) > 1 and argv[1] == "on" + + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + from opentelemetry import trace as trace_api + + from _fake_openai import FakeOpenAI + from _scenarios import TOOL_SECRET_CITY, broken_tool_chat, group_chat, two_agent_tool_chat + from traceai_ag2_classic import setup + + global_before = type(trace_api.get_tracer_provider()).__name__ + provider = register( + project_type=ProjectType.OBSERVE, + project_name="ag2-classic-contract", + verbose=False, + ) + tracing = setup(tracer_provider=provider, capture_content=capture) + + out = {} + with FakeOpenAI(tool_arguments={"city": TOOL_SECRET_CITY}) as fake: + chat = two_agent_tool_chat(tracing, fake) + out["chat_id"] = str(chat.chat_id) + broken_tool_chat(tracing, fake) + group = group_chat(tracing, fake) + out["group_chat_id"] = str(group.chat_id) + out["llm_requests"] = len(fake.requests) + + flushed = provider.force_flush(timeout_millis=10000) + out["flushed"] = bool(flushed) + out["global_provider_before"] = global_before + out["global_provider_after"] = type(trace_api.get_tracer_provider()).__name__ + out["processors"] = [type(p).__name__ for p in provider._active_span_processor._span_processors] + provider.shutdown() + print("CONTRACT_RESULT " + json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/frameworks/ag2-classic/tests/_fake_openai.py b/python/frameworks/ag2-classic/tests/_fake_openai.py new file mode 100644 index 000000000..8195b852a --- /dev/null +++ b/python/frameworks/ag2-classic/tests/_fake_openai.py @@ -0,0 +1,137 @@ +"""A loopback-only, OpenAI-compatible fake chat completions server for tests. + +No network and no vendor API: the server binds to 127.0.0.1 on a free port and +answers ``POST /v1/chat/completions`` with deterministic replies chosen from the +request body: + +* a group-chat speaker-selection prompt gets ``speaker_reply`` (an agent name); +* a request whose last message is a tool result gets ``final_reply``; +* a request that offers tools gets one tool call to the first offered tool; +* anything else gets ``text_reply``. + +Every reply carries ``usage`` so token attributes can be asserted. +""" + +from __future__ import annotations + +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional + +PROMPT_TOKENS = 11 +COMPLETION_TOKENS = 7 +FAKE_MODEL = "gpt-4o-mini" + + +class FakeOpenAI: + def __init__( + self, + *, + text_reply: str = "Hello from the fake model. TERMINATE", + final_reply: str = "Tool says sunny. TERMINATE", + speaker_reply: str = "critic", + tool_arguments: Optional[Dict[str, Any]] = None, + ) -> None: + self.text_reply = text_reply + self.final_reply = final_reply + self.speaker_reply = speaker_reply + self.tool_arguments = tool_arguments if tool_arguments is not None else {"city": "Paris"} + self.requests: List[Dict[str, Any]] = [] + self._lock = threading.Lock() + self._counter = 0 + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + if not self.path.rstrip("/").endswith("/chat/completions"): + self.send_error(HTTPStatus.NOT_FOUND) + return + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8") or "{}") + payload = owner._reply(body) + data = json.dumps(payload).encode("utf-8") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = "http://127.0.0.1:{0}/v1".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + # Context manager ------------------------------------------------------- + + def __enter__(self) -> "FakeOpenAI": + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + # Helpers --------------------------------------------------------------- + + def llm_config(self, **extra: Any) -> Dict[str, Any]: + entry = { + "api_type": "openai", + "model": FAKE_MODEL, + "api_key": "sk-placeholder-not-a-real-key", + "base_url": self.base_url, + } + entry.update(extra) + return {"config_list": [entry], "cache_seed": None} + + def _reply(self, body: Dict[str, Any]) -> Dict[str, Any]: + with self._lock: + self.requests.append(body) + self._counter += 1 + n = self._counter + messages = body.get("messages") or [] + joined = " ".join(str(m.get("content") or "") for m in messages if isinstance(m, dict)) + last = messages[-1] if messages else {} + + message: Dict[str, Any] + finish = "stop" + if "select the next role" in joined or "Then select the next role" in joined: + message = {"role": "assistant", "content": self.speaker_reply} + elif isinstance(last, dict) and last.get("role") == "tool": + message = {"role": "assistant", "content": self.final_reply} + elif body.get("tools"): + tool = body["tools"][0]["function"]["name"] + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_fake_{0}".format(n), + "type": "function", + "function": {"name": tool, "arguments": json.dumps(self.tool_arguments)}, + } + ], + } + finish = "tool_calls" + else: + message = {"role": "assistant", "content": self.text_reply} + + return { + "id": "chatcmpl-fake-{0}".format(n), + "object": "chat.completion", + "created": 1700000000, + "model": body.get("model") or FAKE_MODEL, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + }, + } diff --git a/python/frameworks/ag2-classic/tests/_scenarios.py b/python/frameworks/ag2-classic/tests/_scenarios.py new file mode 100644 index 000000000..b589a435a --- /dev/null +++ b/python/frameworks/ag2-classic/tests/_scenarios.py @@ -0,0 +1,91 @@ +"""AG2 Classic scenarios shared by the in-process tests and the contract app. + +Every scenario talks to a :class:`FakeOpenAI` on 127.0.0.1. No vendor API. +""" + +from __future__ import annotations + +from typing import Any, Dict + +SECRET_PROMPT = "PRIVATE-PROMPT-7f3a: what is the weather in Paris?" +TOOL_SECRET_CITY = "PRIVATE-CITY-91c2" + + +def _terminates(message: Dict[str, Any]) -> bool: + return "TERMINATE" in str(message.get("content") or "") + + +def get_weather(city: str) -> str: + """Return the weather for a city.""" + return "sunny in {0}".format(city) + + +def broken_tool(city: str) -> str: + """A tool that always fails.""" + raise RuntimeError("tool exploded for {0}".format(city)) + + +def two_agent_tool_chat(tracing: Any, fake: Any, *, message: str = SECRET_PROMPT) -> Any: + """user -> assistant, the assistant calls ``get_weather`` once, then stops.""" + from autogen import ConversableAgent + + assistant = ConversableAgent( + "assistant", + llm_config=fake.llm_config(), + human_input_mode="NEVER", + is_termination_msg=_terminates, + ) + user = ConversableAgent( + "user", + llm_config=False, + human_input_mode="NEVER", + max_consecutive_auto_reply=3, + is_termination_msg=_terminates, + ) + assistant.register_for_llm(description="Get the weather")(get_weather) + user.register_for_execution()(get_weather) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + return user.initiate_chat(assistant, message=message, max_turns=3, silent=True) + + +def broken_tool_chat(tracing: Any, fake: Any) -> Any: + """The assistant calls a tool that raises; upstream catches it.""" + from autogen import ConversableAgent + + assistant = ConversableAgent( + "assistant_b", + llm_config=fake.llm_config(), + human_input_mode="NEVER", + is_termination_msg=_terminates, + ) + user = ConversableAgent( + "user_b", + llm_config=False, + human_input_mode="NEVER", + max_consecutive_auto_reply=2, + is_termination_msg=_terminates, + ) + assistant.register_for_llm(description="Always fails")(broken_tool) + user.register_for_execution()(broken_tool) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + return user.initiate_chat(assistant, message="call the broken tool", max_turns=2, silent=True) + + +def group_chat(tracing: Any, fake: Any, *, message: str = "Write a haiku about tracing") -> Any: + """writer -> GroupChatManager with LLM ("auto") speaker selection.""" + from autogen import ConversableAgent + from autogen.agentchat.groupchat import GroupChat, GroupChatManager + + writer = ConversableAgent("writer", llm_config=fake.llm_config(), human_input_mode="NEVER") + critic = ConversableAgent("critic", llm_config=fake.llm_config(), human_input_mode="NEVER") + groupchat = GroupChat( + agents=[writer, critic], + messages=[], + max_round=2, + speaker_selection_method="auto", + ) + manager = GroupChatManager(groupchat=groupchat, llm_config=fake.llm_config()) + tracing.instrument_agent(manager) + return writer.initiate_chat(manager, message=message, max_turns=1, silent=True) diff --git a/python/frameworks/ag2-classic/tests/test_contract_harness.py b/python/frameworks/ag2-classic/tests/test_contract_harness.py new file mode 100644 index 000000000..32bb5f38c --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_contract_harness.py @@ -0,0 +1,210 @@ +"""Shared-harness contract test for traceAI-ag2-classic. + +Runs ``_contract_app.py`` in a subprocess with the real ``autogen`` 0.14.x, +the real OpenTelemetry SDK and Future AGI's real OTLP/HTTP exporter, pointed at +a loopback harness ``Receiver`` through ``FI_BASE_URL`` (exporter path +``/tracer/v1/traces``). The model is a loopback fake; no network, no vendor +API, placeholder keys only. + +The Receiver drops resource attributes and headers; those are asserted in +``test_setup.py``. +""" + +from __future__ import annotations + +import base64 +import json +import os +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +HERE = Path(__file__).resolve().parent +PACKAGE_DIR = HERE.parent +PYTHON_DIR = PACKAGE_DIR.parent.parent +sys.path.insert(0, str(PYTHON_DIR / "tests")) + +from harness import Receiver, run # noqa: E402 + +from _scenarios import SECRET_PROMPT, TOOL_SECRET_CITY # noqa: E402 + +CONTENT_KEYS = ( + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + "ag2.human_input.prompt", + "ag2.human_input.response", + "ag2.code_execution.output", + "ag2.chats.summaries", + "input.value", + "output.value", +) + + +def _value(any_value: Dict[str, Any]) -> Any: + if "stringValue" in any_value: + return any_value["stringValue"] + if "intValue" in any_value: + return int(any_value["intValue"]) + if "doubleValue" in any_value: + return float(any_value["doubleValue"]) + if "boolValue" in any_value: + return bool(any_value["boolValue"]) + return any_value + + +def _flatten(span: Dict[str, Any]) -> Dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _run_app(receiver: Receiver, mode: str) -> Dict[str, Any]: + env = dict(os.environ) + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": "placeholder-api-key", + "FI_SECRET_KEY": "placeholder-secret-key", + "OPENAI_API_KEY": "sk-placeholder-not-a-real-key", + "PYTHONPATH": os.pathsep.join( + [str(PACKAGE_DIR), str(PYTHON_DIR), str(HERE), env.get("PYTHONPATH", "")] + ), + "AUTOGEN_USE_DOCKER": "False", + } + ) + env.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) + env.pop("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", None) + result = run([sys.executable, str(HERE / "_contract_app.py"), mode], env=env, stdin=None, timeout=180) + stdout = result.stdout.decode("utf-8", "replace") + stderr = result.stderr.decode("utf-8", "replace") + assert not result.timed_out, "contract app timed out\nSTDERR:\n" + stderr[-4000:] + assert result.returncode == 0, "contract app failed\nSTDERR:\n" + stderr[-4000:] + lines = [line for line in stdout.splitlines() if line.startswith("CONTRACT_RESULT ")] + assert lines, "no CONTRACT_RESULT line\nSTDOUT:\n" + stdout[-2000:] + return json.loads(lines[-1][len("CONTRACT_RESULT ") :]) + + +def _by_name(spans: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: + grouped: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for span in spans: + grouped[span["name"]].append(span) + return grouped + + +def test_ag2_classic_contract_content_off_round_trip(): + with Receiver() as receiver: + app = _run_app(receiver, "off") + spans = receiver.spans() + + assert app["flushed"] is True + # One provider: setup() never touched the global provider and added no exporter. + assert app["global_provider_before"] == app["global_provider_after"] == "ProxyTracerProvider" + assert app["processors"] == ["AG2ClassicSpanProcessor", "BatchSpanProcessor"] + assert app["llm_requests"] >= 4 + + assert spans, "receiver got no spans" + names = _by_name(spans) + for expected in ( + "conversation user", + "invoke_agent assistant", + "invoke_agent user", + "chat gpt-4o-mini", + "execute_tool get_weather", + "execute_tool broken_tool", + "speaker_selection", + "conversation writer", + "conversation checking_agent", + ): + assert expected in names, "missing span {0!r}; got {1}".format(expected, sorted(names)) + + # Span kind: upstream ag2.span.type preserved, gen_ai.span.kind added. + expected_kind = { + "chat gpt-4o-mini": ("llm", "LLM"), + "invoke_agent assistant": ("agent", "AGENT"), + "invoke_agent user": ("agent", "AGENT"), + "execute_tool get_weather": ("tool", "TOOL"), + "conversation user": ("conversation", "CHAIN"), + "speaker_selection": ("speaker_selection", "CHAIN"), + } + for name, (span_type, kind) in expected_kind.items(): + for span in names[name]: + attrs = _flatten(span) + assert attrs["ag2.span.type"] == span_type, name + assert attrs["gen_ai.span.kind"] == kind, name + + # Model, provider, usage and cost on LLM spans. + for span in names["chat gpt-4o-mini"]: + attrs = _flatten(span) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.request.model"] == "gpt-4o-mini" + assert attrs["gen_ai.response.model"] == "gpt-4o-mini" + assert attrs["gen_ai.usage.input_tokens"] == 11 + assert attrs["gen_ai.usage.output_tokens"] == 7 + assert attrs["gen_ai.usage.total_tokens"] == 18 + assert attrs["gen_ai.cost.total"] > 0 + assert "gen_ai.agent.name" in attrs + + # Tool name and call id. + tool = _flatten(names["execute_tool get_weather"][0]) + assert tool["gen_ai.tool.name"] == "get_weather" + assert tool["gen_ai.tool.type"] == "function" + assert tool["gen_ai.tool.call.id"].startswith("call_fake_") + + # Session: outermost conversation only. + conversation = _flatten(names["conversation user"][0]) + assert conversation["gen_ai.conversation.id"] == app["chat_id"] + assert conversation["session.id"] == app["chat_id"] + group_root = _flatten(names["conversation writer"][0]) + assert group_root["session.id"] == app["group_chat_id"] + for nested in names["conversation checking_agent"]: + nested_attrs = _flatten(nested) + assert "gen_ai.conversation.id" in nested_attrs + assert "session.id" not in nested_attrs + + # AC-07: a tool that raises is caught upstream; status is promoted to ERROR. + broken = names["execute_tool broken_tool"][0] + assert _flatten(broken)["error.type"] == "ExecutionError" + assert broken["status"].get("code") == "STATUS_CODE_ERROR" + assert names["execute_tool get_weather"][0].get("status", {}).get("code") != "STATUS_CODE_ERROR" + + # AC-04: every span of the group chat run shares one trace id. + group_trace = names["conversation writer"][0]["traceId"] + for name in ("speaker_selection", "conversation checking_agent", "invoke_agent critic"): + for span in names[name]: + assert span["traceId"] == group_trace, name + assert names["conversation user"][0]["traceId"] != group_trace + assert len(base64.b64decode(group_trace)) == 16 + + # AC-06: content is absent when capture is off. + for span in spans: + attrs = _flatten(span) + for key in CONTENT_KEYS: + assert key not in attrs, "{0} leaked on {1}".format(key, span["name"]) + blob = json.dumps(attrs) + assert SECRET_PROMPT not in blob, span["name"] + assert TOOL_SECRET_CITY not in blob, span["name"] + + +def test_ag2_classic_contract_content_on_round_trip(): + with Receiver() as receiver: + _run_app(receiver, "on") + spans = receiver.spans() + + names = _by_name(spans) + conversation = _flatten(names["conversation user"][0]) + assert SECRET_PROMPT in conversation["gen_ai.input.messages"] + assert SECRET_PROMPT in conversation["input.value"] + assert conversation["input.mime_type"] == "application/json" + + tool = _flatten(names["execute_tool get_weather"][0]) + assert json.loads(tool["input.value"]) == {"city": TOOL_SECRET_CITY} + assert tool["output.value"] == "sunny in {0}".format(TOOL_SECRET_CITY) + + # capture_content=True is forwarded to upstream instrument_llm_wrapper(capture_messages=True). + llm = _flatten(names["chat gpt-4o-mini"][0]) + assert "gen_ai.input.messages" in llm + assert "gen_ai.output.messages" in llm diff --git a/python/frameworks/ag2-classic/tests/test_guard.py b/python/frameworks/ag2-classic/tests/test_guard.py new file mode 100644 index 000000000..774056fd7 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_guard.py @@ -0,0 +1,135 @@ +"""AC-05: version guard with fake module layouts (no other package installed).""" + +from __future__ import annotations + +import types + +import pytest + +from traceai_ag2_classic import AG2ClassicCompatibilityError, check_autogen_classic + + +def _layout( + *, + owners=("autogen",), + versions=None, + version="0.14.1", + has_otel=True, + importable=True, + agentchat=False, + ag2_module=False, +): + versions = dict(versions or {"autogen": "0.14.1"}) + modules = {} + if importable: + modules["autogen"] = types.ModuleType("autogen") + if version is not None: + vm = types.ModuleType("autogen.version") + setattr(vm, "__version__", version) + modules["autogen.version"] = vm + + def import_module(name): + if name in modules: + return modules[name] + raise ImportError("No module named {0!r}".format(name)) + + def find_spec(name): + if name == "autogen.opentelemetry": + return object() if (importable and has_otel) else None + if name == "autogen_agentchat": + return object() if agentchat else None + if name == "ag2": + return object() if ag2_module else None + return None + + return { + "distributions_for": lambda top: list(owners) if top == "autogen" else [], + "distribution_version": versions.get, + "import_module": import_module, + "find_spec": find_spec, + } + + +def _assert_names_three_packages(error): + text = str(error) + for name in ("`autogen` 0.14.x", "`ag2` 1.x", "`autogen-agentchat`"): + assert name in text + assert "pyautogen" in text + + +@pytest.mark.parametrize("version", ["0.14.0", "0.14.1", "0.14"]) +def test_supported_versions_pass(version): + assert check_autogen_classic(**_layout(version=version, versions={"autogen": version})) == version + + +@pytest.mark.parametrize("version", ["0.13.9", "0.15.0", "0.140.0", "1.1.2", "0.2.35"]) +def test_unsupported_versions_raise(version): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(version=version, versions={"autogen": version})) + _assert_names_three_packages(info.value) + assert version in str(info.value) + + +def test_autogen_0_14_0_alias_layout_backed_by_ag2_0_14_0_passes(): + # PyPI autogen==0.14.0 requires ag2==0.14.0, which ships the `autogen` module. + layout = _layout(owners=("ag2",), versions={"autogen": "0.14.0", "ag2": "0.14.0"}, version="0.14.0") + assert check_autogen_classic(**layout) == "0.14.0" + + +@pytest.mark.parametrize( + "versions", + [ + {"ag2": "0.14.0"}, # ag2 installed directly, no `autogen` distribution + {"ag2": "0.9.0", "autogen": "0.9.0"}, # older ag2 alias line + {"ag2": "1.1.2", "autogen": "0.14.1"}, # mismatched pairing + ], +) +def test_autogen_module_owned_by_ag2_outside_the_alias_layout_raises(versions): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=("ag2",), versions=versions)) + assert "installed by `ag2`" in str(info.value) + _assert_names_three_packages(info.value) + + +@pytest.mark.parametrize("owners", [("pyautogen",), ("autogen", "pyautogen"), ("PyAutoGen",)]) +def test_pyautogen_owner_raises(owners): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=owners)) + assert "pyautogen" in info.value.reason + _assert_names_three_packages(info.value) + + +def test_missing_opentelemetry_module_raises(): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(has_otel=False)) + assert "autogen.opentelemetry" in str(info.value) + _assert_names_three_packages(info.value) + + +def test_microsoft_autogen_layout_raises(): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=(), importable=False, agentchat=True)) + assert "autogen_agentchat" in str(info.value) + _assert_names_three_packages(info.value) + + +def test_ag2_1x_only_layout_raises(): + # ag2 1.x ships `ag2`, not `autogen`, so `import autogen` fails. + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(owners=(), importable=False, ag2_module=True)) + assert "`ag2` (AG2 1.x, `import ag2`) is installed" in str(info.value) + _assert_names_three_packages(info.value) + + +def test_missing_version_module_raises(): + with pytest.raises(AG2ClassicCompatibilityError) as info: + check_autogen_classic(**_layout(version=None)) + assert "autogen.version" in str(info.value) + + +def test_installed_autogen_passes_guard(): + pytest.importorskip("autogen.opentelemetry") + from autogen.version import __version__ + + assert check_autogen_classic() == __version__ + assert __version__.startswith("0.14.") diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py new file mode 100644 index 000000000..3ea5661b5 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -0,0 +1,267 @@ +"""Unit tests for the AG2 Classic attribute mapping (no autogen run needed).""" + +from __future__ import annotations + +import json + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_ag2_classic import ( + AG2_SCOPE, + CONTENT_KEYS, + KIND_BY_SPAN_TYPE, + AG2ClassicSpanProcessor, + kind_for_span_type, + map_ag2_attributes, +) + + +# AC-02: kind map from the upstream SpanType enum --------------------------------- + + +EXPECTED_KINDS = { + "conversation": "CHAIN", + "multi_conversation": "CHAIN", + "agent": "AGENT", + "llm": "LLM", + "tool": "TOOL", + "handoff": "CHAIN", + "speaker_selection": "CHAIN", + "human_input": "CHAIN", + "code_execution": "TOOL", +} + + +def test_kind_map_matches_architecture_table(): + assert KIND_BY_SPAN_TYPE == EXPECTED_KINDS + + +def test_kind_map_covers_every_installed_span_type(): + consts = pytest.importorskip("autogen.opentelemetry.consts") + installed = {member.value for member in consts.SpanType} + assert installed == set(KIND_BY_SPAN_TYPE) + for value in installed: + assert kind_for_span_type(value) == EXPECTED_KINDS[value] + + +def test_scope_name_matches_installed_consts(): + consts = pytest.importorskip("autogen.opentelemetry.consts") + assert consts.INSTRUMENTING_MODULE_NAME == AG2_SCOPE + + +def test_unknown_or_missing_span_type_is_left_alone(): + attrs = {"gen_ai.operation.name": "chat", "gen_ai.input.messages": "[]"} + assert map_ag2_attributes(attrs) == attrs + assert map_ag2_attributes({"ag2.span.type": "nope"}) == {"ag2.span.type": "nope"} + assert kind_for_span_type(None) is None + + +# Usage, cost, request parameters ------------------------------------------------- + + +def test_llm_span_gets_total_tokens_cost_and_parameters(): + attrs = { + "ag2.span.type": "llm", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.cost": 0.0001, + "gen_ai.request.temperature": 0.2, + "gen_ai.request.max_tokens": 64, + } + mapped = map_ag2_attributes(attrs) + assert mapped["gen_ai.span.kind"] == "LLM" + assert mapped["gen_ai.usage.total_tokens"] == 18 + assert mapped["gen_ai.cost.total"] == 0.0001 + assert mapped["gen_ai.usage.cost"] == 0.0001 # upstream key kept + assert json.loads(mapped["gen_ai.request.parameters"]) == {"temperature": 0.2, "max_tokens": 64} + + +def test_llm_span_without_usage_records_no_tokens(): + mapped = map_ag2_attributes({"ag2.span.type": "llm", "gen_ai.operation.name": "chat"}) + assert "gen_ai.usage.total_tokens" not in mapped + assert "gen_ai.cost.total" not in mapped + + +def test_conversation_aggregate_usage_is_not_relabelled(): + attrs = { + "ag2.span.type": "conversation", + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 14, + "gen_ai.usage.cost": 0.0002, + } + mapped = map_ag2_attributes(attrs) + assert "gen_ai.usage.total_tokens" not in mapped + assert "gen_ai.cost.total" not in mapped + + +# Session ---------------------------------------------------------------------- + + +def test_root_conversation_id_becomes_session_id(): + mapped = map_ag2_attributes({"ag2.span.type": "conversation", "gen_ai.conversation.id": "123"}) + assert mapped["session.id"] == "123" + + +def test_nested_conversation_id_is_not_a_session(): + mapped = map_ag2_attributes( + {"ag2.span.type": "conversation", "gen_ai.conversation.id": "456"}, root_conversation=False + ) + assert "session.id" not in mapped + assert mapped["gen_ai.conversation.id"] == "456" + + +def test_existing_session_id_wins(): + mapped = map_ag2_attributes( + {"ag2.span.type": "conversation", "gen_ai.conversation.id": "1", "session.id": "mine"} + ) + assert mapped["session.id"] == "mine" + + +# AC-06 content default ------------------------------------------------------------ + + +CONTENT_SAMPLE = { + "gen_ai.input.messages": '[{"role": "user", "parts": [{"type": "text", "content": "secret"}]}]', + "gen_ai.output.messages": '[{"role": "assistant", "parts": []}]', + "gen_ai.tool.call.arguments": '{"city": "secret"}', + "gen_ai.tool.call.result": "secret result", + "ag2.human_input.prompt": "secret prompt", + "ag2.human_input.response": "secret answer", + "ag2.code_execution.output": "secret output", + "ag2.chats.summaries": '["secret summary"]', +} + + +@pytest.mark.parametrize("span_type", sorted(EXPECTED_KINDS)) +def test_content_is_dropped_by_default(span_type): + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": span_type, "gen_ai.agent.name": "a"}) + mapped = map_ag2_attributes(attrs) + for key in CONTENT_KEYS: + assert key not in mapped + assert "secret" not in json.dumps(mapped) + assert mapped["gen_ai.agent.name"] == "a" + + +def test_capture_on_lifts_messages_to_input_output(): + attrs = { + "ag2.span.type": "agent", + "gen_ai.input.messages": CONTENT_SAMPLE["gen_ai.input.messages"], + "gen_ai.output.messages": CONTENT_SAMPLE["gen_ai.output.messages"], + } + mapped = map_ag2_attributes(attrs, capture_content=True) + assert mapped["input.value"] == attrs["gen_ai.input.messages"] + assert mapped["input.mime_type"] == "application/json" + assert mapped["output.value"] == attrs["gen_ai.output.messages"] + assert mapped["gen_ai.input.messages"] == attrs["gen_ai.input.messages"] + + +def test_capture_on_tool_human_and_code_spans(): + tool = map_ag2_attributes( + {"ag2.span.type": "tool", "gen_ai.tool.call.arguments": '{"a": 1}', "gen_ai.tool.call.result": "ok"}, + capture_content=True, + ) + assert tool["input.value"] == '{"a": 1}' and tool["input.mime_type"] == "application/json" + assert tool["output.value"] == "ok" and tool["output.mime_type"] == "text/plain" + + human = map_ag2_attributes( + {"ag2.span.type": "human_input", "ag2.human_input.prompt": "p", "ag2.human_input.response": "r"}, + capture_content=True, + ) + assert (human["input.value"], human["output.value"]) == ("p", "r") + + code = map_ag2_attributes( + {"ag2.span.type": "code_execution", "ag2.code_execution.output": "42"}, capture_content=True + ) + assert code["output.value"] == "42" + assert code["gen_ai.span.kind"] == "TOOL" + + +# The processor on a real SDK pipeline ------------------------------------------- + + +def _pipeline(capture_content=False): + exporter = InMemorySpanExporter() + provider = TracerProvider() + processor = AG2ClassicSpanProcessor(capture_content=capture_content) + provider.add_span_processor(processor) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def test_processor_maps_only_the_ag2_scope(): + provider, exporter = _pipeline() + ag2 = provider.get_tracer(AG2_SCOPE) + other = provider.get_tracer("someone.else") + with ag2.start_as_current_span("chat m") as span: + span.set_attribute("ag2.span.type", "llm") + span.set_attribute("gen_ai.input.messages", "secret") + with other.start_as_current_span("x") as span: + span.set_attribute("ag2.span.type", "llm") + span.set_attribute("gen_ai.input.messages", "kept") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["chat m"].attributes["gen_ai.span.kind"] == "LLM" + assert "gen_ai.input.messages" not in by_name["chat m"].attributes + assert "gen_ai.span.kind" not in by_name["x"].attributes + assert by_name["x"].attributes["gen_ai.input.messages"] == "kept" + + +def test_processor_promotes_error_type_to_error_status(): + provider, exporter = _pipeline() + tracer = provider.get_tracer(AG2_SCOPE) + with tracer.start_as_current_span("execute_tool t") as span: + span.set_attribute("ag2.span.type", "tool") + span.set_attribute("error.type", "ExecutionError") + with tracer.start_as_current_span("execute_tool ok") as span: + span.set_attribute("ag2.span.type", "tool") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["execute_tool t"].status.status_code is StatusCode.ERROR + assert by_name["execute_tool t"].status.description == "ExecutionError" + assert by_name["execute_tool ok"].status.status_code is StatusCode.UNSET + + +def test_processor_keeps_session_on_outer_conversation_only(): + provider, exporter = _pipeline() + tracer = provider.get_tracer(AG2_SCOPE) + with tracer.start_as_current_span("conversation outer") as outer: + outer.set_attribute("ag2.span.type", "conversation") + with tracer.start_as_current_span("speaker_selection") as sel: + sel.set_attribute("ag2.span.type", "speaker_selection") + with tracer.start_as_current_span("conversation inner") as inner: + inner.set_attribute("ag2.span.type", "conversation") + inner.set_attribute("gen_ai.conversation.id", "inner-id") + outer.set_attribute("gen_ai.conversation.id", "outer-id") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["conversation outer"].attributes["session.id"] == "outer-id" + assert "session.id" not in by_name["conversation inner"].attributes + assert by_name["speaker_selection"].attributes["gen_ai.span.kind"] == "CHAIN" + + +def test_processor_preserves_bounded_attribute_limits(): + from opentelemetry.attributes import BoundedAttributes + + provider, exporter = _pipeline() + with provider.get_tracer(AG2_SCOPE).start_as_current_span("chat") as span: + span.set_attribute("ag2.span.type", "llm") + finished = exporter.get_finished_spans()[0] + assert isinstance(finished._attributes, BoundedAttributes) + with pytest.raises(TypeError): + finished._attributes["x"] = 1 # immutable after end, as the SDK leaves it + + +def test_processor_never_raises_into_the_sdk(): + provider, exporter = _pipeline() + processor = provider._active_span_processor._span_processors[0] + live = processor._live + processor._live = None # force an internal failure + try: + with provider.get_tracer(AG2_SCOPE).start_as_current_span("chat") as span: + span.set_attribute("ag2.span.type", "llm") + finally: + processor._live = live + assert [s.name for s in exporter.get_finished_spans()] == ["chat"] diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py new file mode 100644 index 000000000..e8cf99683 --- /dev/null +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -0,0 +1,365 @@ +"""In-process tests for setup(): one provider, resource/headers, and real AG2 Classic runs. + +Real ``autogen`` 0.14.x agents talk to a loopback fake OpenAI server; spans go +to an in-memory exporter. Each test undoes the global ``OpenAIWrapper.create`` +patch through ``AG2ClassicTracing.uninstrument()``. +""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +from pathlib import Path + +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +pytest.importorskip("autogen.opentelemetry") + +from _fake_openai import COMPLETION_TOKENS, FAKE_MODEL, PROMPT_TOKENS, FakeOpenAI # noqa: E402 +from _scenarios import ( # noqa: E402 + SECRET_PROMPT, + TOOL_SECRET_CITY, + broken_tool_chat, + group_chat, + two_agent_tool_chat, +) + +from traceai_ag2_classic import CONTENT_KEYS, AG2ClassicSpanProcessor, setup # noqa: E402 + +PACKAGE_DIR = Path(__file__).resolve().parent.parent +PYTHON_DIR = PACKAGE_DIR.parent.parent + + +@pytest.fixture +def pipeline(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + handles = [] + + def make(**kwargs): + handle = setup(tracer_provider=provider, **kwargs) + handles.append(handle) + return handle + + yield provider, exporter, make + for handle in handles: + handle.uninstrument() + + +@pytest.fixture +def fake(): + with FakeOpenAI(tool_arguments={"city": TOOL_SECRET_CITY}) as server: + yield server + + +def _by_name(exporter): + grouped = {} + for span in exporter.get_finished_spans(): + grouped.setdefault(span.name, []).append(span) + return grouped + + +def _exporter_of(processor): + exporter = getattr(processor, "span_exporter", None) + if exporter is None: + exporter = getattr(getattr(processor, "_batch_processor", None), "_exporter", None) + return exporter + + +# One provider, resource attributes and headers ----------------------------------- + + +class _RecordingCollector: + """Loopback HTTP server that records path, headers and body of each POST. + + The shared harness Receiver drops headers and resource attributes, so this + unit test records them itself. + """ + + def __init__(self): + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + self.requests = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + owner.requests.append((self.path, dict(self.headers.items()), body)) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format, *args): # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +def test_register_then_setup_keeps_one_provider_resource_and_headers(monkeypatch): + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + + from traceai_ag2_classic import AG2_SCOPE + + collector = _RecordingCollector() + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + monkeypatch.setenv("FI_BASE_URL", collector.origin) + saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + global_before = trace_api.get_tracer_provider() + provider = register(project_type=ProjectType.OBSERVE, project_name="ag2-classic-unit", verbose=False) + tracing = setup(tracer_provider=provider) + try: + processors = provider._active_span_processor._span_processors + assert isinstance(processors[0], AG2ClassicSpanProcessor) + exporting = [p for p in processors if _exporter_of(p) is not None] + assert len(exporting) == 1, "setup() must not add a second exporter" + assert type(_exporter_of(exporting[0])).__name__ == "HTTPSpanExporter" + # setup() never set or replaced the global provider. + assert trace_api.get_tracer_provider() is global_before + assert tracing.autogen_version.startswith("0.14.") + + with provider.get_tracer(AG2_SCOPE).start_as_current_span("chat m") as span: + span.set_attribute("ag2.span.type", "llm") + assert provider.force_flush(timeout_millis=10000) + + assert len(collector.requests) == 1 + path, headers, body = collector.requests[0] + assert path == "/tracer/v1/traces" + lowered = {k.lower(): v for k, v in headers.items()} + assert lowered["x-api-key"] == "placeholder-api-key" + assert lowered["x-secret-key"] == "placeholder-secret-key" + assert "authorization" not in lowered + assert lowered["content-type"] == "application/x-protobuf" + + request = ExportTraceServiceRequest() + request.ParseFromString(body) + resource = {kv.key: kv.value.string_value for kv in request.resource_spans[0].resource.attributes} + assert resource["project_name"] == "ag2-classic-unit" + assert resource["project_type"] == "observe" + assert "openinference.project.name" not in resource + exported = request.resource_spans[0].scope_spans[0].spans[0] + attrs = {kv.key: kv.value.string_value for kv in exported.attributes} + assert attrs["gen_ai.span.kind"] == "LLM" + finally: + tracing.uninstrument() + provider.shutdown() + collector.close() + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline): + from autogen.oai.client import OpenAIWrapper + + provider, _exporter, make = pipeline + original = OpenAIWrapper.create + first = make() + assert getattr(OpenAIWrapper.create, "__otel_wrapped__", False) is True + second = make() + ours = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2ClassicSpanProcessor)] + assert len(ours) == 1 and first.processor is second.processor + assert second.owns_llm_wrapper is False # upstream wraps once (llm_wrapper.py:64-65) + first.uninstrument() + assert OpenAIWrapper.create is original + assert not any( + isinstance(p, AG2ClassicSpanProcessor) for p in provider._active_span_processor._span_processors + ) + + +def test_setup_rejects_a_non_sdk_provider(): + # NoOpTracerProvider has no span processors: setup() refuses instead of creating a provider. + with pytest.raises(TypeError, match="SDK TracerProvider"): + setup(tracer_provider=trace_api.NoOpTracerProvider()) + + +# AC-01 / AC-03 / AC-06: two-agent chat with a tool, content off -------------------- + + +def test_two_agent_chat_kinds_usage_session_and_no_content(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make() + result = two_agent_tool_chat(tracing, fake) + spans = _by_name(exporter) + + assert {"conversation user", "invoke_agent assistant", "invoke_agent user", "execute_tool get_weather"} <= set( + spans + ) + llm_spans = spans["chat {0}".format(FAKE_MODEL)] + assert len(llm_spans) == 2 + for span in llm_spans: + attrs = span.attributes + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.request.model"] == FAKE_MODEL + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.usage.input_tokens"] == PROMPT_TOKENS + assert attrs["gen_ai.usage.output_tokens"] == COMPLETION_TOKENS + assert attrs["gen_ai.usage.total_tokens"] == PROMPT_TOKENS + COMPLETION_TOKENS + assert attrs["gen_ai.cost.total"] > 0 + + assert spans["invoke_agent assistant"][0].attributes["gen_ai.span.kind"] == "AGENT" + tool = spans["execute_tool get_weather"][0] + assert tool.attributes["gen_ai.span.kind"] == "TOOL" + assert tool.attributes["gen_ai.tool.name"] == "get_weather" + assert tool.status.status_code is StatusCode.UNSET + + conversation = spans["conversation user"][0].attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" + assert conversation["session.id"] == str(result.chat_id) + # Upstream aggregate on the conversation span (chat.py:73-82) is passed through. + assert conversation["gen_ai.usage.input_tokens"] == 2 * PROMPT_TOKENS + + # LLM spans are children of the agent span, all in one trace. + trace_ids = {s.context.trace_id for group in spans.values() for s in group} + assert len(trace_ids) == 1 + + for group in spans.values(): + for span in group: + for key in CONTENT_KEYS: + assert key not in span.attributes, (key, span.name) + blob = json.dumps(dict(span.attributes), default=str) + assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob + + +def test_capture_content_on_keeps_and_lifts_content(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make(capture_content=True) + two_agent_tool_chat(tracing, fake) + spans = _by_name(exporter) + conversation = spans["conversation user"][0].attributes + assert SECRET_PROMPT in conversation["input.value"] + tool = spans["execute_tool get_weather"][0].attributes + assert json.loads(tool["gen_ai.tool.call.arguments"]) == {"city": TOOL_SECRET_CITY} + assert tool["output.value"] == "sunny in {0}".format(TOOL_SECRET_CITY) + llm = spans["chat {0}".format(FAKE_MODEL)][0].attributes + assert SECRET_PROMPT in llm["gen_ai.input.messages"] + assert SECRET_PROMPT in llm["input.value"] + + +# AC-07: tool exception -> ERROR --------------------------------------------------- + + +def test_tool_exception_sets_error_status(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make() + broken_tool_chat(tracing, fake) + tool = _by_name(exporter)["execute_tool broken_tool"][0] + assert tool.attributes["error.type"] == "ExecutionError" + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "ExecutionError" + + +# AC-04: group chat, one trace id ---------------------------------------------------- + + +def test_group_chat_is_one_trace_with_one_session(pipeline, fake): + _provider, exporter, make = pipeline + tracing = make() + result = group_chat(tracing, fake) + finished = exporter.get_finished_spans() + names = {s.name for s in finished} + assert {"conversation writer", "speaker_selection", "invoke_agent critic", "conversation checking_agent"} <= names + assert len({s.context.trace_id for s in finished}) == 1 + + sessions = {s.attributes.get("session.id") for s in finished} - {None} + assert sessions == {str(result.chat_id)} + selection = next(s for s in finished if s.name == "speaker_selection") + assert selection.attributes["gen_ai.span.kind"] == "CHAIN" + assert selection.attributes["ag2.speaker_selection.selected"] == "critic" + + +def test_patterns_are_passed_to_upstream_instrument_pattern(pipeline, fake): + from autogen import ConversableAgent + from autogen.agentchat import initiate_group_chat + from autogen.agentchat.group.patterns import RoundRobinPattern + + _provider, exporter, make = pipeline + alpha = ConversableAgent("alpha", llm_config=fake.llm_config(), human_input_mode="NEVER") + beta = ConversableAgent("beta", llm_config=fake.llm_config(), human_input_mode="NEVER") + pattern = RoundRobinPattern(initial_agent=alpha, agents=[alpha, beta]) + make(patterns=[pattern]) + assert getattr(pattern.prepare_group_chat, "__otel_wrapped__", False) is True + result, _ctx, _last = initiate_group_chat(pattern=pattern, messages="hello pattern", max_rounds=3) + finished = exporter.get_finished_spans() + names = {s.name for s in finished} + assert {"invoke_agent alpha", "invoke_agent beta", "chat {0}".format(FAKE_MODEL)} <= names + assert len({s.context.trace_id for s in finished}) == 1 + assert {s.attributes.get("session.id") for s in finished} - {None} == {str(result.chat_id)} + + +# Export failure never fails the agent --------------------------------------------- + + +def test_export_failure_does_not_fail_the_agent(monkeypatch, fake): + import socket + + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + with socket.socket() as sock: # a loopback port with nothing listening + sock.bind(("127.0.0.1", 0)) + closed_port = sock.getsockname()[1] + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:{0}".format(closed_port)) + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + provider = register(project_type=ProjectType.OBSERVE, project_name="ag2-classic-down", verbose=False, timeout=1) + tracing = setup(tracer_provider=provider) + try: + result = two_agent_tool_chat(tracing, fake) + assert result.chat_id + assert "TERMINATE" in result.chat_history[-1]["content"] + provider.force_flush(timeout_millis=3000) # export fails; must not raise + finally: + tracing.uninstrument() + provider.shutdown() + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +# AC-08 (partial): importing the package never imports ag2 ------------------------------ + + +def test_import_does_not_import_ag2_or_autogen_until_setup(): + code = ( + "import sys, importlib.metadata as md\n" + "import traceai_ag2_classic\n" + "assert 'autogen' not in sys.modules, 'package import pulled autogen'\n" + "traceai_ag2_classic.check_autogen_classic()\n" + "assert 'ag2' not in sys.modules, 'an ag2 module was imported'\n" + "try:\n" + " print('AG2_DIST', md.version('ag2'))\n" + "except md.PackageNotFoundError:\n" + " print('AG2_DIST none')\n" + "print('AUTOGEN_DIST', md.version('autogen'))\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(PACKAGE_DIR), str(PYTHON_DIR), env.get("PYTHONPATH", "")]) + proc = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=120) + assert proc.returncode == 0, proc.stderr[-3000:] + lines = dict(line.split(" ", 1) for line in proc.stdout.splitlines() if line.startswith(("AG2_DIST", "AUTOGEN_DIST"))) + if lines["AUTOGEN_DIST"] == "0.14.0": + # PyPI autogen==0.14.0 is an alias that requires ag2==0.14.0 (see _guard.py). + assert lines["AG2_DIST"] == "0.14.0" + else: + assert lines["AG2_DIST"] == "none" diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py b/python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py new file mode 100644 index 000000000..720110810 --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/__init__.py @@ -0,0 +1,34 @@ +"""Future AGI tracing for AG2 Classic (the ``autogen`` distribution, 0.14.x). + +Native-first: AG2 Classic ships ``autogen.opentelemetry``. ``setup()`` calls +its ``instrument_llm_wrapper`` / ``instrument_agent`` / ``instrument_pattern`` +on the Future AGI tracer provider you pass, and adds a span processor that maps +the upstream keys to Future AGI conventions. Importing this package does not +import ``autogen``; ``setup()`` does, after the version guard. +""" + +from ._guard import AG2ClassicCompatibilityError, check_autogen_classic +from ._processor import ( + AG2_SCOPE, + CONTENT_KEYS, + KIND_BY_SPAN_TYPE, + AG2ClassicSpanProcessor, + kind_for_span_type, + map_ag2_attributes, +) +from ._setup import AG2ClassicTracing, setup +from .version import __version__ + +__all__ = [ + "AG2_SCOPE", + "CONTENT_KEYS", + "KIND_BY_SPAN_TYPE", + "AG2ClassicCompatibilityError", + "AG2ClassicSpanProcessor", + "AG2ClassicTracing", + "check_autogen_classic", + "kind_for_span_type", + "map_ag2_attributes", + "setup", + "__version__", +] diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py new file mode 100644 index 000000000..e6b6e91bd --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py @@ -0,0 +1,150 @@ +"""Version guard for AG2 Classic (``import autogen``, 0.14.x). + +Three different projects are easy to confuse: + +* ``autogen`` 0.14.x (AG2 Classic, ``ag2ai/ag2classic``): this package. +* ``ag2`` 1.x (AG2, ``import ag2``): use ``traceAI-ag2``. +* ``autogen-agentchat`` (Microsoft AutoGen): use ``traceAI-autogen``. + +``pyautogen`` is an older alias and is out of scope for this release. + +Installed-layout facts this guard is written against (read, not assumed): + +* ``autogen==0.14.1`` is a standalone distribution that ships the ``autogen`` + module itself (Source: ``ag2ai/ag2classic``). +* ``autogen==0.14.0`` is an 8-file alias whose only runtime requirement is + ``ag2==0.14.0``; the ``autogen`` module is shipped by the ``ag2`` 0.14.0 + distribution. Its ``autogen/opentelemetry`` tree is byte-identical to the + 0.14.1 one. That exact pairing is accepted. +* ``ag2`` 1.x ships only an ``ag2`` top-level module, never ``autogen``. + +The guard runs before anything is instrumented. It never imports ``ag2`` and +never imports Microsoft AutoGen. +""" + +from __future__ import annotations + +import importlib +import importlib.util +from typing import Callable, Iterable, List, Mapping, Optional + +SUPPORTED_SERIES = "0.14" + +_GUIDANCE = ( + "traceAI-ag2-classic supports only AG2 Classic: the `autogen` distribution 0.14.x " + "(import `autogen`, ships `autogen.opentelemetry`). " + "Three packages are easy to confuse: " + "`autogen` 0.14.x -> traceAI-ag2-classic; " + "`ag2` 1.x -> traceAI-ag2; " + "`autogen-agentchat` (Microsoft AutoGen) -> traceAI-autogen. " + "`pyautogen` is not supported by this package." +) + + +class AG2ClassicCompatibilityError(RuntimeError): + """Raised when the importable ``autogen`` is not AG2 Classic 0.14.x.""" + + def __init__(self, reason: str) -> None: + super().__init__("{0} {1}".format(reason, _GUIDANCE)) + self.reason = reason + + +def _normalize(name: str) -> str: + return name.strip().lower().replace("_", "-").replace(".", "-") + + +def _default_distributions_for(top_level: str) -> List[str]: + from importlib import metadata + + try: + mapping: Mapping[str, Iterable[str]] = metadata.packages_distributions() + except Exception: # pragma: no cover - defensive, metadata is stdlib on 3.10+ + return [] + return list(mapping.get(top_level, [])) + + +def _default_distribution_version(name: str) -> Optional[str]: + from importlib import metadata + + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + +def _default_find_spec(name: str) -> Optional[object]: + try: + return importlib.util.find_spec(name) + except (ImportError, ValueError): + return None + + +def _is_supported(version: Optional[str]) -> bool: + if not version: + return False + return version == SUPPORTED_SERIES or version.startswith(SUPPORTED_SERIES + ".") + + +def check_autogen_classic( + *, + distributions_for: Callable[[str], Iterable[str]] = _default_distributions_for, + distribution_version: Callable[[str], Optional[str]] = _default_distribution_version, + import_module: Callable[[str], object] = importlib.import_module, + find_spec: Callable[[str], Optional[object]] = _default_find_spec, +) -> str: + """Return the installed AG2 Classic version, or raise. + + Raises :class:`AG2ClassicCompatibilityError` when: + + * the top-level ``autogen`` module is installed by ``pyautogen``; + * it is installed by ``ag2``, except the ``autogen==0.14.x`` alias layout + (``autogen`` 0.14.x distribution plus ``ag2`` 0.14.x); + * ``autogen`` cannot be imported (for example only ``ag2`` 1.x or + Microsoft ``autogen-agentchat`` is installed); + * ``autogen.version.__version__`` or ``autogen.opentelemetry`` is missing; + * ``autogen.version.__version__`` does not start with ``0.14``. + + The keyword arguments exist so tests can supply a fake module layout + without installing any of the other packages. + """ + owners = sorted({_normalize(d) for d in distributions_for("autogen")}) + if "pyautogen" in owners: + raise AG2ClassicCompatibilityError("The top-level `autogen` module is installed by `pyautogen`.") + if "ag2" in owners: + ag2_version = distribution_version("ag2") + alias_version = distribution_version("autogen") + if not (_is_supported(ag2_version) and _is_supported(alias_version)): + raise AG2ClassicCompatibilityError( + "The top-level `autogen` module is installed by `ag2` {0} (`autogen` distribution: {1}).".format( + ag2_version or "unknown", alias_version or "not installed" + ) + ) + + try: + import_module("autogen") + except ImportError as error: + hints = [] + if find_spec("ag2") is not None: + hints.append("`ag2` (AG2 1.x, `import ag2`) is installed") + if find_spec("autogen_agentchat") is not None: + hints.append("Microsoft AutoGen (`autogen_agentchat`) is installed") + hint = (" " + "; ".join(hints) + ".") if hints else "" + raise AG2ClassicCompatibilityError("`autogen` is not importable ({0}).{1}".format(error, hint)) from error + + try: + version_module = import_module("autogen.version") + version = str(getattr(version_module, "__version__")) + except (ImportError, AttributeError) as error: + raise AG2ClassicCompatibilityError( + "`autogen.version.__version__` was not found, so this is not the AG2 Classic layout." + ) from error + + if find_spec("autogen.opentelemetry") is None: + raise AG2ClassicCompatibilityError("`autogen` {0} has no `autogen.opentelemetry` module.".format(version)) + + if not _is_supported(version): + raise AG2ClassicCompatibilityError( + "`autogen` {0} is installed; only {1}.x is supported.".format(version, SUPPORTED_SERIES) + ) + + return version diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py new file mode 100644 index 000000000..b7e76a5fe --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -0,0 +1,328 @@ +"""Span processor that normalizes AG2 Classic spans to Future AGI conventions. + +``autogen`` 0.14.x ships its own OpenTelemetry instrumentation +(``autogen.opentelemetry``). Its spans already use most GenAI semantic +convention keys. This processor adds only what Future AGI needs and enforces +the content default. Every key below was read from the installed +``autogen/opentelemetry/instrumentators/`` at 0.14.1 (see README "Attribute +inventory" for file:line evidence). + +What it does, per span on the ``opentelemetry.instrumentation.ag2`` scope: + +* ``gen_ai.span.kind`` from upstream ``ag2.span.type`` (the ``SpanType`` enum). +* ``session.id`` from ``gen_ai.conversation.id`` on the outermost + ``conversation`` span only. Nested chats (for example group-chat speaker + selection) get their own ``chat_id`` upstream; those are not copied, so one + run maps to one session. +* ``gen_ai.cost.total`` from upstream ``gen_ai.usage.cost`` on LLM spans, and + ``gen_ai.usage.total_tokens`` = input + output on LLM spans. +* ``gen_ai.request.parameters`` JSON from upstream ``gen_ai.request.*`` + sampling keys. +* Status ``ERROR`` when upstream set ``error.type`` but left the status unset + (tool failures are caught inside ``ConversableAgent.execute_function``, so + upstream never sets the status itself). +* Content: off by default. Upstream agent, conversation, tool, human-input and + code-execution spans always record message bodies, tool arguments/results, + prompts and code output; there is no upstream flag for those. With + ``capture_content=False`` (default) those keys are removed. With + ``capture_content=True`` they are kept and lifted into ``input.value`` / + ``output.value``. + +Spans from any other instrumentation scope pass through untouched. The +processor never raises into the SDK. +""" + +from __future__ import annotations + +import json +import logging +import threading +from typing import Any, Dict, Mapping, Optional + +from opentelemetry.context import Context +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor +from opentelemetry.trace import Status, StatusCode + +from fi_instrumentation.fi_types import FiMimeTypeValues, FiSpanKindValues, SpanAttributes + +logger = logging.getLogger(__name__) + +# Read from autogen/opentelemetry/consts.py at 0.14.1 (INSTRUMENTING_MODULE_NAME). +AG2_SCOPE = "opentelemetry.instrumentation.ag2" +# Set on every upstream span by the instrumentators (e.g. llm_wrapper.py:74). +SPAN_TYPE_KEY = "ag2.span.type" + +# SpanType values (consts.py:13-22) -> Future AGI span kind. +KIND_BY_SPAN_TYPE: Dict[str, str] = { + "llm": FiSpanKindValues.LLM.value, + "agent": FiSpanKindValues.AGENT.value, + "tool": FiSpanKindValues.TOOL.value, + "code_execution": FiSpanKindValues.TOOL.value, + "conversation": FiSpanKindValues.CHAIN.value, + "multi_conversation": FiSpanKindValues.CHAIN.value, + "speaker_selection": FiSpanKindValues.CHAIN.value, + "human_input": FiSpanKindValues.CHAIN.value, + # Declared upstream with a TODO and never emitted at 0.14.1. Kept as CHAIN. + "handoff": FiSpanKindValues.CHAIN.value, +} + +# Upstream keys (verbatim strings from the instrumentators). +INPUT_MESSAGES = "gen_ai.input.messages" +OUTPUT_MESSAGES = "gen_ai.output.messages" +TOOL_ARGUMENTS = "gen_ai.tool.call.arguments" +TOOL_RESULT = "gen_ai.tool.call.result" +HUMAN_PROMPT = "ag2.human_input.prompt" +HUMAN_RESPONSE = "ag2.human_input.response" +CODE_OUTPUT = "ag2.code_execution.output" +CHAT_SUMMARIES = "ag2.chats.summaries" +CONVERSATION_ID = "gen_ai.conversation.id" +UPSTREAM_COST = "gen_ai.usage.cost" +ERROR_TYPE = "error.type" +INPUT_TOKENS = "gen_ai.usage.input_tokens" +OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + +# Every key that carries user or model content. Removed unless capture is on. +CONTENT_KEYS = ( + INPUT_MESSAGES, + OUTPUT_MESSAGES, + TOOL_ARGUMENTS, + TOOL_RESULT, + HUMAN_PROMPT, + HUMAN_RESPONSE, + CODE_OUTPUT, + CHAT_SUMMARIES, + SpanAttributes.INPUT_VALUE, + SpanAttributes.INPUT_MIME_TYPE, + SpanAttributes.OUTPUT_VALUE, + SpanAttributes.OUTPUT_MIME_TYPE, +) + +# utils.py set_llm_request_params (lines 213-219). +REQUEST_PARAMETER_KEYS = ( + "gen_ai.request.temperature", + "gen_ai.request.max_tokens", + "gen_ai.request.top_p", + "gen_ai.request.frequency_penalty", + "gen_ai.request.presence_penalty", +) + +# Only a live ``conversation`` ancestor makes a conversation "nested". Chats +# started by ``initiate_chats`` (``multi_conversation``) stay roots: each one is +# its own upstream chat with its own ``chat_id``. +_CONVERSATION_TYPES = ("conversation",) + +# Upper bound on live AG2 spans tracked for the nested-conversation check, so a +# span that is started and never ended cannot grow memory without limit. +_MAX_LIVE_SPANS = 10000 + + +def kind_for_span_type(span_type: Any) -> Optional[str]: + """Return the Future AGI span kind for an upstream ``ag2.span.type`` value.""" + if not isinstance(span_type, str): + return None + return KIND_BY_SPAN_TYPE.get(span_type) + + +def _json_mime(value: str) -> str: + stripped = value.strip() + if (stripped.startswith("{") and stripped.endswith("}")) or ( + stripped.startswith("[") and stripped.endswith("]") + ): + return FiMimeTypeValues.JSON.value + return FiMimeTypeValues.TEXT.value + + +def _set_io(mapped: Dict[str, Any], value_key: str, mime_key: str, value: Any) -> None: + if value is None or value_key in mapped: + return + text = value if isinstance(value, str) else json.dumps(value, default=str) + mapped[value_key] = text + mapped[mime_key] = _json_mime(text) + + +def _surface_content(mapped: Dict[str, Any], span_type: str) -> None: + if span_type == "tool": + _set_io(mapped, SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE, mapped.get(TOOL_ARGUMENTS)) + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(TOOL_RESULT)) + elif span_type == "human_input": + _set_io(mapped, SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE, mapped.get(HUMAN_PROMPT)) + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(HUMAN_RESPONSE)) + elif span_type == "code_execution": + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(CODE_OUTPUT)) + elif span_type == "multi_conversation": + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(CHAT_SUMMARIES)) + else: + _set_io(mapped, SpanAttributes.INPUT_VALUE, SpanAttributes.INPUT_MIME_TYPE, mapped.get(INPUT_MESSAGES)) + _set_io(mapped, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, mapped.get(OUTPUT_MESSAGES)) + + +def _bundle_request_parameters(mapped: Dict[str, Any]) -> None: + if SpanAttributes.GEN_AI_REQUEST_PARAMETERS in mapped: + return + params = { + key.split("gen_ai.request.", 1)[1]: mapped[key] for key in REQUEST_PARAMETER_KEYS if key in mapped + } + if params: + mapped[SpanAttributes.GEN_AI_REQUEST_PARAMETERS] = json.dumps(params) + + +def map_ag2_attributes( + attributes: Mapping[str, Any], + *, + capture_content: bool = False, + root_conversation: bool = True, +) -> Dict[str, Any]: + """Return a new attribute dict with Future AGI keys added. + + ``root_conversation`` is False for a ``conversation`` span nested inside + another live AG2 conversation; its ``gen_ai.conversation.id`` is then not + copied to ``session.id``. + + Attributes without an upstream ``ag2.span.type`` are returned unchanged. + """ + mapped = dict(attributes or {}) + span_type = mapped.get(SPAN_TYPE_KEY) + kind = kind_for_span_type(span_type) + if kind is None: + return mapped + + mapped[SpanAttributes.GEN_AI_SPAN_KIND] = kind + + conversation_id = mapped.get(CONVERSATION_ID) + if ( + span_type == "conversation" + and root_conversation + and conversation_id + and SpanAttributes.SESSION_ID not in mapped + ): + mapped[SpanAttributes.SESSION_ID] = str(conversation_id) + + if kind == FiSpanKindValues.LLM.value: + cost = mapped.get(UPSTREAM_COST) + if isinstance(cost, (int, float)) and SpanAttributes.GEN_AI_COST_TOTAL not in mapped: + mapped[SpanAttributes.GEN_AI_COST_TOTAL] = cost + inp = mapped.get(INPUT_TOKENS) + out = mapped.get(OUTPUT_TOKENS) + if ( + isinstance(inp, int) + and isinstance(out, int) + and SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS not in mapped + ): + mapped[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = inp + out + _bundle_request_parameters(mapped) + + if capture_content: + _surface_content(mapped, str(span_type)) + else: + for key in CONTENT_KEYS: + mapped.pop(key, None) + + return mapped + + +def _replace_attributes(span: ReadableSpan, mapped: Dict[str, Any]) -> None: + """Write ``mapped`` back onto the ended span so later processors see it. + + ``ReadableSpan.attributes`` is a read-only view over ``span._attributes``. + The SDK passes one ReadableSpan object to every processor in order, so + replacing ``_attributes`` here is visible to the exporting processor that + runs after this one. ``BoundedAttributes`` limits are preserved. + """ + old = getattr(span, "_attributes", None) + try: + from opentelemetry.attributes import BoundedAttributes + + if isinstance(old, BoundedAttributes): + new = BoundedAttributes( + maxlen=old.maxlen, + attributes=mapped, + immutable=True, + max_value_len=old.max_value_len, + ) + new.dropped = old.dropped + span._attributes = new # type: ignore[attr-defined] + return + except Exception: # pragma: no cover - fall back to a plain mapping + pass + span._attributes = mapped # type: ignore[attr-defined] + + +class AG2ClassicSpanProcessor(SpanProcessor): + """Normalize ``autogen.opentelemetry`` spans before they are exported. + + Install it ahead of the exporting processor (``setup()`` does this), so the + exporter sees the mapped attributes. + """ + + def __init__(self, capture_content: bool = False) -> None: + self.capture_content = bool(capture_content) + self._live: Dict[int, Span] = {} + self._lock = threading.Lock() + self._disabled = False + + # SpanProcessor interface ---------------------------------------------- + + def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None: + if self._disabled or not _is_ag2(span): + return + try: + with self._lock: + if len(self._live) < _MAX_LIVE_SPANS: + self._live[span.context.span_id] = span + except Exception: # pragma: no cover - never break the SDK + return + + def on_end(self, span: ReadableSpan) -> None: + if self._disabled or not _is_ag2(span): + return + try: + with self._lock: + self._live.pop(span.context.span_id, None) + attributes = dict(span.attributes or {}) + root = True + if attributes.get(SPAN_TYPE_KEY) == "conversation": + root = not self._has_live_conversation_ancestor(span) + mapped = map_ag2_attributes( + attributes, capture_content=self.capture_content, root_conversation=root + ) + if mapped != attributes: + _replace_attributes(span, mapped) + + error_type = attributes.get(ERROR_TYPE) + if error_type and span.status.status_code is StatusCode.UNSET: + span._status = Status(StatusCode.ERROR, str(error_type)) # type: ignore[attr-defined] + except Exception: # pragma: no cover - never break the SDK + logger.debug("traceai-ag2-classic: span mapping failed", exc_info=True) + + def shutdown(self) -> None: + self._disabled = True + try: + with self._lock: + self._live.clear() + except Exception: # pragma: no cover - never break SDK shutdown + pass + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True + + # Internals -------------------------------------------------------------- + + def _has_live_conversation_ancestor(self, span: ReadableSpan) -> bool: + parent = span.parent + seen = 0 + with self._lock: + while parent is not None and seen < 256: + live = self._live.get(parent.span_id) + if live is None: + return False + attrs = live.attributes or {} + if attrs.get(SPAN_TYPE_KEY) in _CONVERSATION_TYPES: + return True + parent = live.parent + seen += 1 + return False + + +def _is_ag2(span: Any) -> bool: + scope = getattr(span, "instrumentation_scope", None) + return getattr(scope, "name", None) == AG2_SCOPE diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py new file mode 100644 index 000000000..d4f8631b2 --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -0,0 +1,234 @@ +"""One-call enabler for AG2 Classic's built-in ``autogen.opentelemetry``. + +``setup()`` does four things, in order: + +1. Checks that the importable ``autogen`` is AG2 Classic 0.14.x (``_guard``). +2. Prepends :class:`AG2ClassicSpanProcessor` to the provider you pass, ahead + of the exporting processor ``fi_instrumentation.register()`` installed. +3. Calls upstream ``instrument_llm_wrapper`` (global, patches + ``OpenAIWrapper.create``) with ``capture_messages=capture_content``. +4. Calls upstream ``instrument_agent`` / ``instrument_pattern`` for the agents + and patterns you pass. + +It never creates a ``TracerProvider``, never sets the global provider, and +never constructs an exporter (in particular not the gRPC exporter the +``autogen[tracing]`` extra installs). One provider, the one you pass. +""" + +from __future__ import annotations + +import logging +from typing import Any, Iterable, Optional + +from opentelemetry import trace as trace_api + +from ._guard import check_autogen_classic +from ._processor import AG2ClassicSpanProcessor + +logger = logging.getLogger(__name__) + + +def _active_multi_processor(provider: Any) -> Any: + active = getattr(provider, "_active_span_processor", None) + if active is None or not hasattr(active, "_span_processors"): + raise TypeError( + "traceai_ag2_classic.setup() needs an OpenTelemetry SDK TracerProvider, for example " + "the one returned by fi_instrumentation.register(project_type=ProjectType.OBSERVE, " + "project_name=...). It does not create or register a provider itself. Got {0}.".format( + type(provider).__name__ + ) + ) + return active + + +def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanProcessor: + """Prepend our processor so it runs before the exporting processor. + + ``provider.add_span_processor`` is deliberately not used: Future AGI's + ``TracerProvider`` drops its default exporting processor on the first + ``add_span_processor`` call after ``register()``, which would lose spans. + Prepending also guarantees the exporter sees the mapped attributes, and + that the ``error.type`` -> ERROR promotion runs before Future AGI's + exporting processor turns every UNSET status into OK + (``fi_instrumentation/otel.py`` ``BatchSpanProcessor.on_end``). + """ + active = _active_multi_processor(provider) + existing = tuple(active._span_processors) + for processor in existing: + if isinstance(processor, AG2ClassicSpanProcessor): + if processor.capture_content != bool(capture_content): + logger.warning( + "traceai-ag2-classic is already installed on this provider with " + "capture_content=%s; keeping that setting.", + processor.capture_content, + ) + return processor + processor = AG2ClassicSpanProcessor(capture_content=capture_content) + with getattr(active, "_lock", _NullLock()): + active._span_processors = (processor,) + tuple(active._span_processors) + return processor + + +class _NullLock: + def __enter__(self) -> None: + return None + + def __exit__(self, *_exc: Any) -> None: + return None + + +class AG2ClassicTracing: + """Handle returned by :func:`setup`. + + Use it to instrument agents or patterns created after ``setup()``. + """ + + def __init__( + self, + *, + tracer_provider: Any, + processor: AG2ClassicSpanProcessor, + autogen_version: str, + capture_content: bool, + ) -> None: + self.tracer_provider = tracer_provider + self.processor = processor + self.autogen_version = autogen_version + self.capture_content = capture_content + self._original_create: Any = None + self.owns_llm_wrapper = False + + # Upstream calls --------------------------------------------------------- + + def _instrument_llm_wrapper(self) -> None: + from autogen.oai.client import OpenAIWrapper + from autogen.opentelemetry import instrument_llm_wrapper + + current = OpenAIWrapper.create + if hasattr(current, "__otel_wrapped__"): + # Upstream returns early when already wrapped (llm_wrapper.py:64-65), + # so LLM spans keep going to whichever provider wrapped it first. + logger.warning( + "autogen OpenAIWrapper.create is already instrumented; LLM spans keep using the " + "tracer provider and capture setting from that earlier call." + ) + return + self._original_create = current + instrument_llm_wrapper( + tracer_provider=self.tracer_provider, capture_messages=self.capture_content + ) + self.owns_llm_wrapper = True + + def instrument_agent(self, agent: Any) -> Any: + """Call upstream ``instrument_agent`` on ``agent``. + + For a ``GroupChatManager`` it also instruments every agent in its group + chat and the group chat itself (speaker-selection spans), mirroring what + upstream ``instrument_pattern`` does for patterns (pattern.py:93-104). + """ + from autogen.agentchat.groupchat import GroupChat, GroupChatManager + from autogen.opentelemetry import instrument_agent + from autogen.opentelemetry.instrumentators.pattern import instrument_groupchat + + provider = self.tracer_provider + instrument_agent(agent, tracer_provider=provider) + if isinstance(agent, GroupChatManager): + groupchat = agent.groupchat + for member in list(getattr(groupchat, "agents", []) or []): + instrument_agent(member, tracer_provider=provider) + instrument_groupchat(groupchat, tracer_provider=provider) + # GroupChatManager.__init__ registers a shallow copy of the group chat + # in _reply_func_list; instrument that copy too (pattern.py:98-104). + for entry in getattr(agent, "_reply_func_list", []) or []: + config = entry.get("config") if isinstance(entry, dict) else None + if isinstance(config, GroupChat) and config is not groupchat: + instrument_groupchat(config, tracer_provider=provider) + return agent + + def instrument_pattern(self, pattern: Any) -> Any: + """Call upstream ``instrument_pattern`` on ``pattern``.""" + from autogen.opentelemetry import instrument_pattern + + return instrument_pattern(pattern, tracer_provider=self.tracer_provider) + + def instrument_a2a_server(self, server: Any) -> Any: + """Call upstream ``instrument_a2a_server``. Optional; only if you ask. + + Upstream exports it only when its imports succeed (``autogen[a2a]``). + """ + try: + from autogen.opentelemetry import instrument_a2a_server # type: ignore[attr-defined] + except ImportError as error: + raise ImportError( + "autogen.opentelemetry.instrument_a2a_server is not available; install the " + "`autogen[a2a]` extra to trace an A2A server." + ) from error + return instrument_a2a_server(server, tracer_provider=self.tracer_provider) + + # Teardown --------------------------------------------------------------- + + def uninstrument(self) -> None: + """Remove the processor and restore ``OpenAIWrapper.create`` if we patched it. + + Upstream patches agents per instance and offers no undo, so agents + instrumented through this handle stay instrumented. + """ + if self.owns_llm_wrapper and self._original_create is not None: + from autogen.oai import client as oai_client_module + from autogen.oai.client import OpenAIWrapper + + OpenAIWrapper.create = self._original_create + oai_client_module.OpenAIWrapper.create = self._original_create + self.owns_llm_wrapper = False + self._original_create = None + active = getattr(self.tracer_provider, "_active_span_processor", None) + if active is not None and hasattr(active, "_span_processors"): + with getattr(active, "_lock", _NullLock()): + active._span_processors = tuple( + p for p in active._span_processors if p is not self.processor + ) + self.processor.shutdown() + + +def setup( + tracer_provider: Optional[Any] = None, + *, + agents: Iterable[Any] = (), + patterns: Iterable[Any] = (), + capture_content: bool = False, + instrument_llm: bool = True, +) -> AG2ClassicTracing: + """Enable AG2 Classic's native tracing on a Future AGI tracer provider. + + Args: + tracer_provider: The provider returned by ``fi_instrumentation.register``. + Defaults to the global provider, which must be an SDK provider. + agents: Agents to pass to upstream ``instrument_agent``. + patterns: Group patterns to pass to upstream ``instrument_pattern``. + capture_content: Keep message bodies, tool arguments/results, human + input and code output on spans. Off by default. + instrument_llm: Call upstream ``instrument_llm_wrapper`` (LLM spans). + + Returns: + An :class:`AG2ClassicTracing` handle for agents created later. + + Raises: + AG2ClassicCompatibilityError: ``autogen`` is not AG2 Classic 0.14.x. + TypeError: The provider is not an OpenTelemetry SDK provider. + """ + version = check_autogen_classic() + provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + processor = _install_processor(provider, capture_content) + handle = AG2ClassicTracing( + tracer_provider=provider, + processor=processor, + autogen_version=version, + capture_content=bool(capture_content), + ) + if instrument_llm: + handle._instrument_llm_wrapper() + for agent in agents: + handle.instrument_agent(agent) + for pattern in patterns: + handle.instrument_pattern(pattern) + return handle diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/version.py b/python/frameworks/ag2-classic/traceai_ag2_classic/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" From 96ef5b3a683cd1d5c80c4001d16a96c596535c4d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:07:35 -0700 Subject: [PATCH 05/22] fix(ag2-classic): keep promoted token keys on LLM spans only Conversation spans carry the chat-wide aggregate_usage sum as gen_ai.usage.input_tokens/output_tokens (autogen 0.14.1 opentelemetry/instrumentators/agent_instrumentators/chat.py 81-82, 146-147) on top of the per-call tokens on the LLM spans (llm_wrapper.py 123-125). fi-collector promotes those keys on any span (adapter.go) and Observe sums total_tokens over a trace, so totals doubled. Non-LLM spans now keep the values under ag2.usage.*. Tests first: new processor test plus a trace-sum assertion in the two-agent setup test. 59 passed on autogen 0.14.1 and 0.14.0. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 2 +- .../ag2-classic/tests/test_processor.py | 18 ++++++++++++++++++ .../frameworks/ag2-classic/tests/test_setup.py | 12 ++++++++++-- .../traceai_ag2_classic/_processor.py | 8 ++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 7ebe8f024..895407664 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -104,7 +104,7 @@ are covered by mapping unit tests only. |---|---|---| | Span kind | `gen_ai.span.kind` from upstream `ag2.span.type`: `llm` → `LLM`; `agent` → `AGENT`; `tool`, `code_execution` → `TOOL`; `conversation`, `multi_conversation`, `speaker_selection`, `human_input` → `CHAIN`; `handoff` → `CHAIN` (declared upstream as TODO, never emitted at 0.14.x). `ag2.span.type` and `gen_ai.operation.name` are kept. | The `a2a-execution` span from `instrument_a2a_server` has no `ag2.span.type` and gets no kind. | | Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | -| Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans also carry upstream's aggregate input/output tokens for the whole chat; they are passed through unchanged and get no `total_tokens`, so do not sum them with the LLM spans. | +| Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans carry upstream's aggregate input/output tokens for the whole chat. Future AGI sums tokens over every span in a trace, so those move to `ag2.usage.input_tokens` / `ag2.usage.output_tokens`; each LLM call counts once. | | Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). | Conversation spans keep only upstream `gen_ai.usage.cost` (aggregate). | | Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. | Nested chats (for example the internal chat behind group-chat speaker selection) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. | | User | Not emitted upstream. | Always. | diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index 3ea5661b5..fb7fa5b24 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -100,6 +100,24 @@ def test_conversation_aggregate_usage_is_not_relabelled(): assert "gen_ai.cost.total" not in mapped +def test_conversation_aggregate_tokens_do_not_double_count_the_trace(): + """chat.py:81-82 puts the chat-wide sum on the conversation span. + + fi-collector promotes gen_ai.usage.* on any span and Observe sums + total_tokens over a trace, so only LLM spans keep the promoted keys. + """ + attrs = { + "ag2.span.type": "conversation", + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 14, + } + mapped = map_ag2_attributes(attrs) + assert "gen_ai.usage.input_tokens" not in mapped + assert "gen_ai.usage.output_tokens" not in mapped + assert mapped["ag2.usage.input_tokens"] == 22 + assert mapped["ag2.usage.output_tokens"] == 14 + + # Session ---------------------------------------------------------------------- diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index e8cf99683..8d6430efc 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -226,8 +226,16 @@ def test_two_agent_chat_kinds_usage_session_and_no_content(pipeline, fake): conversation = spans["conversation user"][0].attributes assert conversation["gen_ai.span.kind"] == "CHAIN" assert conversation["session.id"] == str(result.chat_id) - # Upstream aggregate on the conversation span (chat.py:73-82) is passed through. - assert conversation["gen_ai.usage.input_tokens"] == 2 * PROMPT_TOKENS + # Upstream aggregate on the conversation span (chat.py:73-82) is kept under + # ag2.usage.* so the trace total counts each LLM call once. + assert "gen_ai.usage.input_tokens" not in conversation + assert conversation["ag2.usage.input_tokens"] == 2 * PROMPT_TOKENS + promoted = sum( + s.attributes.get("gen_ai.usage.input_tokens", 0) + for group in spans.values() + for s in group + ) + assert promoted == 2 * PROMPT_TOKENS # LLM spans are children of the agent span, all in one trace. trace_ids = {s.context.trace_id for group in spans.values() for s in group} diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index b7e76a5fe..cf3572a08 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -210,6 +210,14 @@ def map_ag2_attributes( ): mapped[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = inp + out _bundle_request_parameters(mapped) + else: + # fi-collector promotes these keys into the token columns on any span + # (adapter.go inputTokenKeys/outputTokenKeys/totalTokenKeys) and Observe + # sums total_tokens over a trace. Only LLM spans carry per-call usage; + # the conversation span's chat-wide sum (chat.py:81-82) would double it. + for key in (INPUT_TOKENS, OUTPUT_TOKENS, SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS): + if key in mapped: + mapped.setdefault("ag2.usage." + key.rsplit(".", 1)[1], mapped.pop(key)) if capture_content: _surface_content(mapped, str(span_type)) From 10e9dd775ead63d74155724a2738ae97d6faaed6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:45:35 -0700 Subject: [PATCH 06/22] fix(ag2-classic): keep content dropping after uninstrument uninstrument() removed AG2ClassicSpanProcessor and shut it down, but upstream instrument_agent patches each agent once with no undo. Agents instrumented earlier kept emitting spans on the same provider, now unfiltered: gen_ai.input/output.messages were exported on invoke_agent and conversation spans, and conversation spans regained the chat-wide gen_ai.usage.* tokens (double count). The same happened to a second handle sharing the processor when the first handle uninstrumented. uninstrument() now restores OpenAIWrapper.create only. The processor stays installed, shared by every handle on the provider, and shuts down with the provider. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 7 +- .../ag2-classic/tests/test_setup.py | 70 ++++++++++++++++++- .../ag2-classic/traceai_ag2_classic/_setup.py | 20 +++--- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 895407664..8b25514bc 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -89,8 +89,11 @@ the default; pass `transport=Transport.GRPC` (from `fi_instrumentation.otel`) to also install the `[tracing]` extra's gRPC exporter as a global provider, or spans are duplicated. -`tracing.uninstrument()` removes the processor and restores -`OpenAIWrapper.create`. Agents stay instrumented; upstream has no per-agent undo. +`tracing.uninstrument()` restores `OpenAIWrapper.create`, so no new LLM spans. +Agents stay instrumented; upstream has no per-agent undo. The processor stays on +the provider, so spans from those agents still have content removed and +aggregate usage moved off non-LLM spans. It is shared by every handle on that +provider and shuts down with `trace_provider.shutdown()`. ## Conformance diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 8d6430efc..4d9d1371c 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -182,9 +182,75 @@ def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline): assert second.owns_llm_wrapper is False # upstream wraps once (llm_wrapper.py:64-65) first.uninstrument() assert OpenAIWrapper.create is original - assert not any( - isinstance(p, AG2ClassicSpanProcessor) for p in provider._active_span_processor._span_processors + # Upstream has no per-agent undo, so agents instrumented earlier keep + # emitting spans on this provider. The processor stays installed and live + # so those spans are still filtered; the second handle keeps working too. + ours = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2ClassicSpanProcessor)] + assert ours == [first.processor] + assert provider._active_span_processor._span_processors[0] is first.processor + assert first.processor._disabled is False + + +PROMOTED_USAGE_KEYS = ( + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.cost.total", + "llm.cost.total", +) + + +def _assert_no_content_and_no_promoted_usage_off_llm(spans): + assert spans, "no spans were exported" + for span in spans: + attrs = dict(span.attributes) + for key in CONTENT_KEYS: + assert key not in attrs, (key, span.name) + blob = json.dumps(attrs, default=str) + assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob, span.name + if attrs.get("gen_ai.span.kind") != "LLM": + leaked = [k for k in attrs if k in PROMOTED_USAGE_KEYS] + assert not leaked, (span.name, leaked) + + +def test_previously_instrumented_agents_stay_filtered_after_uninstrument(pipeline, fake): + from autogen import ConversableAgent + + _provider, exporter, make = pipeline + tracing = make() + assistant = ConversableAgent( + "assistant", + llm_config=fake.llm_config(), + human_input_mode="NEVER", + is_termination_msg=lambda m: "TERMINATE" in str(m.get("content") or ""), ) + user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=1) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + tracing.uninstrument() + exporter.clear() + + user.initiate_chat(assistant, message=SECRET_PROMPT, max_turns=1, silent=True) + spans = exporter.get_finished_spans() + names = {s.name for s in spans} + # Upstream patched these agents for good; their spans still arrive. + assert {"conversation user", "invoke_agent assistant"} <= names + _assert_no_content_and_no_promoted_usage_off_llm(spans) + conversation = next(s for s in spans if s.name == "conversation user").attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" + assert conversation["ag2.usage.input_tokens"] == PROMPT_TOKENS + + +def test_second_handle_keeps_filtering_after_first_handle_uninstruments(pipeline, fake): + _provider, exporter, make = pipeline + first = make() + second = make() + first.uninstrument() + exporter.clear() + two_agent_tool_chat(second, fake) + spans = exporter.get_finished_spans() + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) def test_setup_rejects_a_non_sdk_provider(): diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index d4f8631b2..01fa2b0af 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -168,10 +168,15 @@ def instrument_a2a_server(self, server: Any) -> Any: # Teardown --------------------------------------------------------------- def uninstrument(self) -> None: - """Remove the processor and restore ``OpenAIWrapper.create`` if we patched it. - - Upstream patches agents per instance and offers no undo, so agents - instrumented through this handle stay instrumented. + """Restore ``OpenAIWrapper.create`` if this handle patched it. + + The span processor stays on the provider. Upstream patches agents per + instance and offers no undo, so agents instrumented earlier keep + emitting spans on this provider; the processor keeps dropping their + content and moving their aggregate usage off non-LLM spans. Every + handle on a provider shares that one processor, so removing it here + would also unfilter the other handles. It is shut down with the + provider (``provider.shutdown()``). """ if self.owns_llm_wrapper and self._original_create is not None: from autogen.oai import client as oai_client_module @@ -181,13 +186,6 @@ def uninstrument(self) -> None: oai_client_module.OpenAIWrapper.create = self._original_create self.owns_llm_wrapper = False self._original_create = None - active = getattr(self.tracer_provider, "_active_span_processor", None) - if active is not None and hasattr(active, "_span_processors"): - with getattr(active, "_lock", _NullLock()): - active._span_processors = tuple( - p for p in active._span_processors if p is not self.processor - ) - self.processor.shutdown() def setup( From e4540e41c8cc8aaa6d928e2089372cf0a9b0a73a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:46:21 -0700 Subject: [PATCH 07/22] fix(ag2-classic): roll back the LLM patch when setup() fails setup() installed the processor and patched OpenAIWrapper.create before instrumenting agents and patterns. If one of those raised, the caller never got the handle and the global patch could not be undone. setup() now restores OpenAIWrapper.create and re-raises; the processor stays, because agents instrumented before the failure keep emitting spans. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 4 +++- .../ag2-classic/tests/test_setup.py | 24 +++++++++++++++++++ .../ag2-classic/traceai_ag2_classic/_setup.py | 20 ++++++++++++---- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 8b25514bc..968a861d4 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -81,7 +81,9 @@ never called unless you ask. 2. Puts `AG2ClassicSpanProcessor` first on the provider you pass, ahead of the Future AGI exporter. 3. Calls upstream `instrument_llm_wrapper(capture_messages=capture_content)`, - then `instrument_agent` and `instrument_pattern` for what you pass. + then `instrument_agent` and `instrument_pattern` for what you pass. If one + of those raises (for example a non-agent in `agents=`), `setup()` restores + `OpenAIWrapper.create` and re-raises. It never creates a provider, never sets the global provider, and never adds an exporter. One provider, the one `register()` returned. HTTP to the collector is diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 4d9d1371c..900251dbd 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -259,6 +259,30 @@ def test_setup_rejects_a_non_sdk_provider(): setup(tracer_provider=trace_api.NoOpTracerProvider()) +def test_setup_rolls_back_the_llm_patch_when_an_agent_fails(fake): + from autogen import ConversableAgent + from autogen.oai import client as oai_client_module + from autogen.oai.client import OpenAIWrapper + + provider = TracerProvider() + original = OpenAIWrapper.create + good = ConversableAgent("good", llm_config=fake.llm_config(), human_input_mode="NEVER") + try: + # Upstream instrument_agent raises AttributeError on a non-agent. + with pytest.raises(AttributeError): + setup(tracer_provider=provider, agents=[good, object()]) + assert OpenAIWrapper.create is original + assert oai_client_module.OpenAIWrapper.create is original + # A later setup() can still own the global LLM patch. + retry = setup(tracer_provider=provider) + assert retry.owns_llm_wrapper is True + retry.uninstrument() + assert OpenAIWrapper.create is original + finally: + OpenAIWrapper.create = original + oai_client_module.OpenAIWrapper.create = original + + # AC-01 / AC-03 / AC-06: two-agent chat with a tool, content off -------------------- diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index 01fa2b0af..32847f2a7 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -213,6 +213,10 @@ def setup( Raises: AG2ClassicCompatibilityError: ``autogen`` is not AG2 Classic 0.14.x. TypeError: The provider is not an OpenTelemetry SDK provider. + + If instrumenting an agent or pattern raises, ``setup()`` restores + ``OpenAIWrapper.create`` before re-raising, so no global patch is left + without a handle to undo it. """ version = check_autogen_classic() provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() @@ -225,8 +229,16 @@ def setup( ) if instrument_llm: handle._instrument_llm_wrapper() - for agent in agents: - handle.instrument_agent(agent) - for pattern in patterns: - handle.instrument_pattern(pattern) + try: + for agent in agents: + handle.instrument_agent(agent) + for pattern in patterns: + handle.instrument_pattern(pattern) + except BaseException: + # The caller never receives the handle, so undo the global + # OpenAIWrapper.create patch here. The processor stays (see + # AG2ClassicTracing.uninstrument): agents instrumented before the + # failure keep emitting spans and still need filtering. + handle.uninstrument() + raise return handle From 911cabfb60a41542f6c460248aff1d4d62fa3767 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:48:58 -0700 Subject: [PATCH 08/22] fix(ag2-classic): keep the processor first across add_span_processor Future AGI's TracerProvider.add_span_processor shuts down and clears every processor on the first call after register() (fi_instrumentation/otel.py 336-339), including AG2ClassicSpanProcessor. Content then leaked to the new processor and conversation spans kept the promoted gen_ai.usage.* tokens. setup() now wraps add_span_processor on the provider instance once. After the original runs, our processor is re-enabled if that call shut it down and re-prepended ahead of every other processor. setup() finds the processor through the wrapper, so repeated setup() calls never stack a second guard or processor. README Privacy and Troubleshooting updated. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 19 ++++- .../ag2-classic/tests/test_setup.py | 57 +++++++++++++ .../traceai_ag2_classic/_processor.py | 8 ++ .../ag2-classic/traceai_ag2_classic/_setup.py | 83 ++++++++++++++++--- 4 files changed, 152 insertions(+), 15 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 968a861d4..ff7cb591f 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -132,6 +132,20 @@ summaries. With content off this package removes these keys before export: `output.value`. That is stricter than upstream. With content on they are kept and lifted into `input.value` / `output.value`. +The filtering lives in `AG2ClassicSpanProcessor`, which must run before any +exporting processor. `setup()` keeps it first on the provider: + +- `trace_provider.add_span_processor(...)` after `setup()`: Future AGI's + provider shuts down and removes every processor on the first such call after + `register()` (`fi_instrumentation/otel.py:336-339`), this one included. + `setup()` wraps that method on your provider instance, so after the call this + processor is put back first and re-enabled. Your processor sees filtered + spans. +- `tracing.uninstrument()` leaves the processor in place, because agents + instrumented earlier keep emitting spans. +- Editing `_active_span_processor._span_processors` yourself bypasses this; + call `setup()` again to put the processor back first. + ## Attribute inventory (autogen 0.14.1) Paths are relative to `autogen/opentelemetry/`. Scope name @@ -160,8 +174,9 @@ Paths are relative to `autogen/opentelemetry/`. Scope name `OpenAIWrapper.create` once and keeps the first provider (`llm_wrapper.py:64-65`). Call `uninstrument()` on the first handle first. - Calling `trace_provider.add_span_processor(...)` after `register()` drops - Future AGI's default exporter (and this processor); that is - `fi_instrumentation` behaviour. `setup()` does not use it. + Future AGI's default exporter; that is `fi_instrumentation` behaviour. + `setup()` does not use it, and puts this package's processor back first + when you call it (see Privacy). - `setup()` raises and names three packages: you have Microsoft AutoGen, `ag2` 1.x, or `pyautogen` instead of `autogen` 0.14.x. - Content missing: content is off by default here; pass `capture_content=True`. diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 900251dbd..356a6cd88 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -169,6 +169,63 @@ def test_register_then_setup_keeps_one_provider_resource_and_headers(monkeypatch signal.signal(sig, handler) +def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(monkeypatch, fake): + """fi's TracerProvider.add_span_processor shuts down and clears every + processor, ours included (fi_instrumentation/otel.py:336-339). setup() + re-prepends and re-enables ours so the new processor still sees filtered spans. + """ + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") + saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + provider = register( + project_type=ProjectType.OBSERVE, + project_name="ag2-classic-add", + verbose=False, + set_global_tracer_provider=False, + ) + tracing = setup(tracer_provider=provider) + memory = InMemorySpanExporter() + try: + added = SimpleSpanProcessor(memory) + provider.add_span_processor(added) + processors = provider._active_span_processor._span_processors + assert processors == (tracing.processor, added) + assert tracing.processor._disabled is False + + two_agent_tool_chat(tracing, fake) + spans = memory.get_finished_spans() + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) + conversation = next(s for s in spans if s.name == "conversation user").attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" + + # A second add keeps exactly one copy of ours, still first. + second = SimpleSpanProcessor(InMemorySpanExporter()) + provider.add_span_processor(second) + assert provider._active_span_processor._span_processors == (tracing.processor, added, second) + finally: + tracing.uninstrument() + provider.shutdown() + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +def test_add_span_processor_on_a_plain_sdk_provider_keeps_one_processor_first(pipeline): + provider, _exporter, make = pipeline + first = make() + make() # a second setup() must not stack a second guard or processor + extra = SimpleSpanProcessor(InMemorySpanExporter()) + provider.add_span_processor(extra) + processors = provider._active_span_processor._span_processors + assert processors[0] is first.processor + assert processors[-1] is extra + assert sum(isinstance(p, AG2ClassicSpanProcessor) for p in processors) == 1 + + def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline): from autogen.oai.client import OpenAIWrapper diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index cf3572a08..86c417e56 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -310,6 +310,14 @@ def shutdown(self) -> None: except Exception: # pragma: no cover - never break SDK shutdown pass + def _reopen(self) -> None: + """Undo :meth:`shutdown` after Future AGI's ``add_span_processor`` ran. + + That method shuts down every processor on the provider, not only its + own default exporter; ``setup()`` re-installs this one and calls this. + """ + self._disabled = False + def force_flush(self, timeout_millis: int = 30000) -> bool: return True diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index 32847f2a7..8b580f896 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -51,24 +51,81 @@ def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanPr that the ``error.type`` -> ERROR promotion runs before Future AGI's exporting processor turns every UNSET status into OK (``fi_instrumentation/otel.py`` ``BatchSpanProcessor.on_end``). + + The provider instance's ``add_span_processor`` is wrapped once (see + :func:`_guard_add_span_processor`) so a later call by the user keeps this + processor installed, first and enabled. """ active = _active_multi_processor(provider) - existing = tuple(active._span_processors) - for processor in existing: - if isinstance(processor, AG2ClassicSpanProcessor): - if processor.capture_content != bool(capture_content): - logger.warning( - "traceai-ag2-classic is already installed on this provider with " - "capture_content=%s; keeping that setting.", - processor.capture_content, - ) - return processor - processor = AG2ClassicSpanProcessor(capture_content=capture_content) - with getattr(active, "_lock", _NullLock()): - active._span_processors = (processor,) + tuple(active._span_processors) + processor = getattr(getattr(provider, "add_span_processor", None), _GUARD_ATTR, None) + if not isinstance(processor, AG2ClassicSpanProcessor): + processor = next( + (p for p in tuple(active._span_processors) if isinstance(p, AG2ClassicSpanProcessor)), + None, + ) + if processor is not None: + if processor.capture_content != bool(capture_content): + logger.warning( + "traceai-ag2-classic is already installed on this provider with " + "capture_content=%s; keeping that setting.", + processor.capture_content, + ) + if processor not in tuple(active._span_processors): + _prepend(active, processor) + else: + processor = AG2ClassicSpanProcessor(capture_content=capture_content) + _prepend(active, processor) + _guard_add_span_processor(provider, processor) return processor +# Attribute on our add_span_processor wrapper; holds the provider's processor. +_GUARD_ATTR = "_traceai_ag2_classic_processor" + + +def _prepend(active: Any, processor: AG2ClassicSpanProcessor) -> None: + with getattr(active, "_lock", _NullLock()): + rest = tuple(p for p in active._span_processors if p is not processor) + active._span_processors = (processor,) + rest + + +def _guard_add_span_processor(provider: Any, processor: AG2ClassicSpanProcessor) -> None: + """Keep ``processor`` first and enabled across ``provider.add_span_processor``. + + Future AGI's ``TracerProvider.add_span_processor`` shuts down and clears + every processor while ``register()``'s default exporter is still in place + (``fi_instrumentation/otel.py:336-339``), ours included. Without this, + content would be exported and conversation spans would keep the promoted + token keys after a user adds their own processor. Only this provider + instance is wrapped, once. + """ + current = getattr(provider, "add_span_processor", None) + if current is None or getattr(current, _GUARD_ATTR, None) is not None: + return + + def add_span_processor(*args: Any, **kwargs: Any) -> Any: + was_enabled = not processor._disabled + result = current(*args, **kwargs) + try: + if was_enabled and processor._disabled: + processor._reopen() + _prepend(_active_multi_processor(provider), processor) + except Exception: # pragma: no cover - never break the caller's add + logger.warning("traceai-ag2-classic could not re-install its span processor.", exc_info=True) + return result + + setattr(add_span_processor, _GUARD_ATTR, processor) + add_span_processor.__doc__ = getattr(current, "__doc__", None) + try: + provider.add_span_processor = add_span_processor + except Exception: # pragma: no cover - provider without instance attributes + logger.warning( + "traceai-ag2-classic could not guard %s.add_span_processor; calling it after setup() " + "may remove this package's span processor.", + type(provider).__name__, + ) + + class _NullLock: def __enter__(self) -> None: return None From 5805ca0d8b483f9e5bdd837445d6f1f45625d725 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:53:12 -0700 Subject: [PATCH 09/22] fix(ag2-classic): follow non-AG2 spans when finding the outer conversation The processor tracked only AG2-scope spans and stopped the ancestor walk at the first untracked parent. A user span between two AG2 conversations (an inner initiate_chat inside the user's own span, agent-as-tool style) hid the outer conversation, so the inner chat id became a second session.id in the same trace. on_start now records (parent span id, AG2 span or None) for every live span on the provider, still bounded by _MAX_LIVE_SPANS, and on_end releases every span. The walk continues through non-AG2 spans. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- .../ag2-classic/tests/test_processor.py | 35 +++++++++++++++ .../ag2-classic/tests/test_setup.py | 41 ++++++++++++++++++ .../traceai_ag2_classic/_processor.py | 43 ++++++++++++------- 3 files changed, 103 insertions(+), 16 deletions(-) diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index fb7fa5b24..5603c0061 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -260,6 +260,41 @@ def test_processor_keeps_session_on_outer_conversation_only(): assert by_name["speaker_selection"].attributes["gen_ai.span.kind"] == "CHAIN" +def test_non_ag2_span_between_conversations_does_not_split_the_session(): + """Agent-as-tool: the user's own span wraps an inner initiate_chat.""" + provider, exporter = _pipeline() + processor = provider._active_span_processor._span_processors[0] + ag2 = provider.get_tracer(AG2_SCOPE) + user = provider.get_tracer("user.app") + with ag2.start_as_current_span("conversation outer") as outer: + outer.set_attribute("ag2.span.type", "conversation") + with user.start_as_current_span("user work"): + with user.start_as_current_span("user inner work"): + with ag2.start_as_current_span("conversation inner") as inner: + inner.set_attribute("ag2.span.type", "conversation") + inner.set_attribute("gen_ai.conversation.id", "inner-id") + outer.set_attribute("gen_ai.conversation.id", "outer-id") + by_name = {s.name: s for s in exporter.get_finished_spans()} + assert by_name["conversation outer"].attributes["session.id"] == "outer-id" + assert "session.id" not in by_name["conversation inner"].attributes + assert "session.id" not in by_name["user work"].attributes + # Every tracked span, AG2 or not, is released when it ends. + assert processor._live == {} + + +def test_live_span_tracking_is_bounded(monkeypatch): + from traceai_ag2_classic import _processor + + monkeypatch.setattr(_processor, "_MAX_LIVE_SPANS", 2) + provider, exporter = _pipeline() + processor = provider._active_span_processor._span_processors[0] + user = provider.get_tracer("user.app") + with user.start_as_current_span("a"), user.start_as_current_span("b"), user.start_as_current_span("c"): + assert len(processor._live) == 2 + assert processor._live == {} + assert len(exporter.get_finished_spans()) == 3 + + def test_processor_preserves_bounded_attribute_limits(): from opentelemetry.attributes import BoundedAttributes diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 356a6cd88..3e2328ddf 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -443,6 +443,47 @@ def test_group_chat_is_one_trace_with_one_session(pipeline, fake): assert selection.attributes["ag2.speaker_selection.selected"] == "critic" +def test_user_span_around_an_inner_chat_keeps_one_session(pipeline, fake): + """Agent-as-tool: a tool runs an inner chat inside the user's own span.""" + from autogen import ConversableAgent + + provider, exporter, make = pipeline + tracing = make() + user_tracer = provider.get_tracer("user.app") + terminates = lambda m: "TERMINATE" in str(m.get("content") or "") # noqa: E731 + expert = ConversableAgent("expert", llm_config=fake.llm_config(), human_input_mode="NEVER") + asker = ConversableAgent("asker", llm_config=False, human_input_mode="NEVER") + tracing.instrument_agent(expert) + tracing.instrument_agent(asker) + + def ask_expert(city: str) -> str: + """Ask the expert about a city.""" + with user_tracer.start_as_current_span("ask_expert work"): + inner = asker.initiate_chat(expert, message="about " + city, max_turns=1, silent=True) + return str(inner.summary) + + assistant = ConversableAgent( + "assistant", llm_config=fake.llm_config(), human_input_mode="NEVER", is_termination_msg=terminates + ) + user = ConversableAgent( + "user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=3, is_termination_msg=terminates + ) + assistant.register_for_llm(description="Ask the expert")(ask_expert) + user.register_for_execution()(ask_expert) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + result = user.initiate_chat(assistant, message="hi", max_turns=3, silent=True) + + finished = exporter.get_finished_spans() + names = {s.name for s in finished} + assert {"conversation user", "execute_tool ask_expert", "ask_expert work", "conversation asker"} <= names + assert len({s.context.trace_id for s in finished}) == 1 + assert {s.attributes.get("session.id") for s in finished} - {None} == {str(result.chat_id)} + inner = next(s for s in finished if s.name == "conversation asker") + assert inner.attributes["gen_ai.conversation.id"] != str(result.chat_id) + assert "session.id" not in inner.attributes + + def test_patterns_are_passed_to_upstream_instrument_pattern(pipeline, fake): from autogen import ConversableAgent from autogen.agentchat import initiate_group_chat diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index 86c417e56..e399df480 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -12,8 +12,9 @@ * ``gen_ai.span.kind`` from upstream ``ag2.span.type`` (the ``SpanType`` enum). * ``session.id`` from ``gen_ai.conversation.id`` on the outermost ``conversation`` span only. Nested chats (for example group-chat speaker - selection) get their own ``chat_id`` upstream; those are not copied, so one - run maps to one session. + selection, or an inner chat a tool starts inside the user's own span) get + their own ``chat_id`` upstream; those are not copied, so one run maps to one + session. The ancestor walk follows every live span, AG2 or not. * ``gen_ai.cost.total`` from upstream ``gen_ai.usage.cost`` on LLM spans, and ``gen_ai.usage.total_tokens`` = input + output on LLM spans. * ``gen_ai.request.parameters`` JSON from upstream ``gen_ai.request.*`` @@ -37,7 +38,7 @@ import json import logging import threading -from typing import Any, Dict, Mapping, Optional +from typing import Any, Dict, Mapping, Optional, Tuple from opentelemetry.context import Context from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor @@ -111,8 +112,9 @@ # its own upstream chat with its own ``chat_id``. _CONVERSATION_TYPES = ("conversation",) -# Upper bound on live AG2 spans tracked for the nested-conversation check, so a -# span that is started and never ended cannot grow memory without limit. +# Upper bound on live spans tracked (AG2 or not) for the nested-conversation +# check, so a span that is started and never ended cannot grow memory without +# limit. Spans past the bound are not tracked; an ancestor walk stops there. _MAX_LIVE_SPANS = 10000 @@ -264,28 +266,35 @@ class AG2ClassicSpanProcessor(SpanProcessor): def __init__(self, capture_content: bool = False) -> None: self.capture_content = bool(capture_content) - self._live: Dict[int, Span] = {} + # span_id -> (parent span_id, the span if it is an AG2 span else None), + # for every live span on the provider, AG2 or not. A user span between + # two AG2 conversations must not hide the outer one. + self._live: Dict[int, Tuple[Optional[int], Optional[Span]]] = {} self._lock = threading.Lock() self._disabled = False # SpanProcessor interface ---------------------------------------------- def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None: - if self._disabled or not _is_ag2(span): + if self._disabled: return try: + parent = span.parent + entry = (parent.span_id if parent is not None else None, span if _is_ag2(span) else None) with self._lock: if len(self._live) < _MAX_LIVE_SPANS: - self._live[span.context.span_id] = span + self._live[span.context.span_id] = entry except Exception: # pragma: no cover - never break the SDK return def on_end(self, span: ReadableSpan) -> None: - if self._disabled or not _is_ag2(span): + if self._disabled: return try: with self._lock: self._live.pop(span.context.span_id, None) + if not _is_ag2(span): + return attributes = dict(span.attributes or {}) root = True if attributes.get(SPAN_TYPE_KEY) == "conversation": @@ -325,16 +334,18 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: def _has_live_conversation_ancestor(self, span: ReadableSpan) -> bool: parent = span.parent + parent_id = parent.span_id if parent is not None else None seen = 0 with self._lock: - while parent is not None and seen < 256: - live = self._live.get(parent.span_id) - if live is None: + while parent_id is not None and seen < 256: + entry = self._live.get(parent_id) + if entry is None: return False - attrs = live.attributes or {} - if attrs.get(SPAN_TYPE_KEY) in _CONVERSATION_TYPES: - return True - parent = live.parent + parent_id, ag2_span = entry + if ag2_span is not None: + attrs = ag2_span.attributes or {} + if attrs.get(SPAN_TYPE_KEY) in _CONVERSATION_TYPES: + return True seen += 1 return False From a383f20d98f094175b82f752c16acd1342dac4f9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 17:01:55 -0700 Subject: [PATCH 10/22] fix(ag2-classic): apply using_attributes context to AG2 spans Upstream spans come from a plain OpenTelemetry tracer; only Future AGI's own tracer applies context attributes (fi_instrumentation/instrumentation/ _tracers.py:165). using_session / using_user / using_attributes were therefore ignored: session.id was the chat id and user.id was absent. on_start now copies get_attributes_from_context() (read from the span's parent context) onto AG2 spans, without overwriting keys upstream already set. A context session.id is on the span before it ends, so it wins over the gen_ai.conversation.id copy; the conversation span keeps the real chat id in gen_ai.conversation.id. initiate_chats keeps one session per outermost conversation and no session on the multi_conversation parent. The README Session row now says so and points to using_session(...) for one session over the whole call; tests cover both. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 12 ++-- .../ag2-classic/tests/test_processor.py | 44 ++++++++++++++ .../ag2-classic/tests/test_setup.py | 58 +++++++++++++++++++ .../traceai_ag2_classic/_processor.py | 45 +++++++++++++- 4 files changed, 152 insertions(+), 7 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index ff7cb591f..917ee3fef 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -100,10 +100,10 @@ provider and shuts down with `trace_provider.shutdown()`. ## Conformance Every row was read from `autogen/opentelemetry/instrumentators/` at 0.14.1. -LLM, agent, tool, conversation, group-chat speaker selection and pattern spans -are exercised by real `autogen` runs against a loopback fake model in -`tests/`; code execution, human input, `initiate_chats` and remote-agent spans -are covered by mapping unit tests only. +LLM, agent, tool, conversation, group-chat speaker selection, pattern and +`initiate_chats` spans are exercised by real `autogen` runs against a loopback +fake model in `tests/`; code execution, human input and remote-agent spans are +covered by mapping unit tests only. | Field | What you get | When it is omitted | |---|---|---| @@ -111,8 +111,8 @@ are covered by mapping unit tests only. | Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | | Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans carry upstream's aggregate input/output tokens for the whole chat. Future AGI sums tokens over every span in a trace, so those move to `ag2.usage.input_tokens` / `ag2.usage.output_tokens`; each LLM call counts once. | | Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). | Conversation spans keep only upstream `gen_ai.usage.cost` (aggregate). | -| Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. | Nested chats (for example the internal chat behind group-chat speaker selection) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. | -| User | Not emitted upstream. | Always. | +| Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id; `gen_ai.conversation.id` keeps the chat id. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | +| User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | | Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | | Errors | Exceptions keep OTel's ERROR status. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | | Retrieval | Not emitted upstream. | Always. | diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index 5603c0061..fef478e67 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -295,6 +295,50 @@ def test_live_span_tracking_is_bounded(monkeypatch): assert len(exporter.get_finished_spans()) == 3 +# Context attributes (using_attributes / using_session / using_user) ------------ + + +def test_context_attributes_are_copied_onto_ag2_spans(): + from fi_instrumentation import using_attributes + + provider, exporter = _pipeline() + ag2 = provider.get_tracer(AG2_SCOPE) + other = provider.get_tracer("someone.else") + with using_attributes(session_id="my-session", user_id="u-1", metadata={"k": "v"}, tags=["t1"]): + with ag2.start_as_current_span("conversation c") as conversation: + conversation.set_attribute("ag2.span.type", "conversation") + with ag2.start_as_current_span("chat m") as llm: + llm.set_attribute("ag2.span.type", "llm") + conversation.set_attribute("gen_ai.conversation.id", "chat-1") + with other.start_as_current_span("x"): + pass + by_name = {s.name: s.attributes for s in exporter.get_finished_spans()} + for name in ("conversation c", "chat m"): + assert by_name[name]["session.id"] == "my-session", name + assert by_name[name]["user.id"] == "u-1", name + assert json.loads(by_name[name]["metadata"]) == {"k": "v"}, name + assert tuple(by_name[name]["tag.tags"]) == ("t1",), name + # The user's session wins over the upstream chat id; the chat id is kept. + assert by_name["conversation c"]["gen_ai.conversation.id"] == "chat-1" + # Spans from other instrumentations are left to their own tracer. + assert "session.id" not in by_name["x"] and "user.id" not in by_name["x"] + + +def test_context_attributes_do_not_overwrite_upstream_values(): + from fi_instrumentation import using_session + + provider, exporter = _pipeline() + ag2 = provider.get_tracer(AG2_SCOPE) + with using_session("my-session"): + with ag2.start_as_current_span( + "conversation c", attributes={"ag2.span.type": "conversation", "gen_ai.conversation.id": "chat-1"} + ): + pass + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["session.id"] == "my-session" + assert attrs["gen_ai.conversation.id"] == "chat-1" + + def test_processor_preserves_bounded_attribute_limits(): from opentelemetry.attributes import BoundedAttributes diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 3e2328ddf..b433b6805 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -484,6 +484,64 @@ def ask_expert(city: str) -> str: assert "session.id" not in inner.attributes +# Context attributes and initiate_chats sessions ----------------------------------- + + +def test_using_attributes_sets_session_and_user_on_ag2_spans(pipeline, fake): + from fi_instrumentation import using_attributes + + _provider, exporter, make = pipeline + tracing = make() + with using_attributes(session_id="my-session", user_id="u-1"): + result = two_agent_tool_chat(tracing, fake) + finished = exporter.get_finished_spans() + assert {s.name for s in finished} >= {"conversation user", "chat {0}".format(FAKE_MODEL), "execute_tool get_weather"} + for span in finished: + assert span.attributes["session.id"] == "my-session", span.name + assert span.attributes["user.id"] == "u-1", span.name + conversation = next(s for s in finished if s.name == "conversation user").attributes + assert conversation["gen_ai.conversation.id"] == str(result.chat_id) + + +def _initiate_chats(tracing, fake): + from autogen import ConversableAgent + + first = ConversableAgent("first", llm_config=fake.llm_config(), human_input_mode="NEVER") + second = ConversableAgent("second", llm_config=fake.llm_config(), human_input_mode="NEVER") + sender = ConversableAgent("sender", llm_config=False, human_input_mode="NEVER") + for agent in (first, second, sender): + tracing.instrument_agent(agent) + return sender.initiate_chats( + [ + {"recipient": first, "message": "one", "max_turns": 1, "silent": True}, + {"recipient": second, "message": "two", "max_turns": 1, "silent": True}, + ] + ) + + +def test_initiate_chats_has_one_session_per_chat_by_default(pipeline, fake): + _provider, exporter, make = pipeline + results = _initiate_chats(make(), fake) + finished = exporter.get_finished_spans() + assert len({s.context.trace_id for s in finished}) == 1 + sessions = {s.attributes.get("session.id") for s in finished} - {None} + assert sessions == {str(r.chat_id) for r in results} and len(sessions) == 2 + multi = next(s for s in finished if s.attributes.get("ag2.span.type") == "multi_conversation") + assert "session.id" not in multi.attributes + + +def test_using_session_gives_initiate_chats_one_session(pipeline, fake): + from fi_instrumentation import using_session + + _provider, exporter, make = pipeline + tracing = make() + with using_session("batch-1"): + _initiate_chats(tracing, fake) + finished = exporter.get_finished_spans() + assert any(s.attributes.get("ag2.span.type") == "multi_conversation" for s in finished) + assert {s.attributes.get("session.id") for s in finished} == {"batch-1"} + + def test_patterns_are_passed_to_upstream_instrument_pattern(pipeline, fake): from autogen import ConversableAgent from autogen.agentchat import initiate_group_chat diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index e399df480..eedd41ff5 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -15,6 +15,10 @@ selection, or an inner chat a tool starts inside the user's own span) get their own ``chat_id`` upstream; those are not copied, so one run maps to one session. The ancestor walk follows every live span, AG2 or not. +* Future AGI context attributes (``using_session``, ``using_user``, + ``using_attributes``: ``session.id``, ``user.id``, ``metadata``, ...) copied + onto AG2 spans when they start; a context ``session.id`` wins over the + conversation id. * ``gen_ai.cost.total`` from upstream ``gen_ai.usage.cost`` on LLM spans, and ``gen_ai.usage.total_tokens`` = input + output on LLM spans. * ``gen_ai.request.parameters`` JSON from upstream ``gen_ai.request.*`` @@ -40,11 +44,13 @@ import threading from typing import Any, Dict, Mapping, Optional, Tuple +from opentelemetry import context as context_api from opentelemetry.context import Context from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.trace import Status, StatusCode from fi_instrumentation.fi_types import FiMimeTypeValues, FiSpanKindValues, SpanAttributes +from fi_instrumentation import get_attributes_from_context logger = logging.getLogger(__name__) @@ -279,11 +285,14 @@ def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None if self._disabled: return try: + ag2 = _is_ag2(span) parent = span.parent - entry = (parent.span_id if parent is not None else None, span if _is_ag2(span) else None) + entry = (parent.span_id if parent is not None else None, span if ag2 else None) with self._lock: if len(self._live) < _MAX_LIVE_SPANS: self._live[span.context.span_id] = entry + if ag2: + _apply_context_attributes(span, parent_context) except Exception: # pragma: no cover - never break the SDK return @@ -353,3 +362,37 @@ def _has_live_conversation_ancestor(self, span: ReadableSpan) -> bool: def _is_ag2(span: Any) -> bool: scope = getattr(span, "instrumentation_scope", None) return getattr(scope, "name", None) == AG2_SCOPE + + +def _context_attributes(parent_context: Optional[Context]) -> Dict[str, Any]: + """Return Future AGI context attributes (``using_attributes`` & co.). + + Upstream spans come from a plain OpenTelemetry tracer, which never reads + these; only Future AGI's own tracer applies them + (``fi_instrumentation/instrumentation/_tracers.py:165``). + """ + if parent_context is None: + return dict(get_attributes_from_context()) + token = context_api.attach(parent_context) + try: + return dict(get_attributes_from_context()) + finally: + context_api.detach(token) + + +def _apply_context_attributes(span: Span, parent_context: Optional[Context]) -> None: + """Copy context attributes onto a starting AG2 span. + + Keys upstream already set are left alone. A ``session.id`` from + ``using_session`` / ``using_attributes`` is therefore on the span before + it ends, and wins over the ``gen_ai.conversation.id`` copy in + :func:`map_ag2_attributes`. Upstream sets the conversation span's + ``gen_ai.conversation.id`` later, so it keeps the real chat id. + """ + values = _context_attributes(parent_context) + if not values: + return + existing = span.attributes or {} + missing = {key: value for key, value in values.items() if key not in existing} + if missing: + span.set_attributes(missing) From cb44a2f9da58325178eccd4daca554033de108c6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:29:56 -0700 Subject: [PATCH 11/22] fix(ag2-classic): keep every gen_ai.usage.* key off non-LLM spans The conversation span still carried upstream's chat-wide gen_ai.usage.cost after the token move. The contract test now asserts, on the OTLP the shared Receiver decoded, that no gen_ai.usage.* / gen_ai.cost.total / llm.cost.total key appears on a non-LLM span and that each trace's gen_ai.usage.total_tokens sum equals its LLM calls x 18 (the reviewer's M1 and M5 mutations passed the contract test before). map_ag2_attributes now moves every gen_ai.usage.* key on a non-LLM span to ag2.usage.*, so the aggregate cost lands in ag2.usage.cost next to the aggregate tokens. README Cost row updated, and notes that upstream reports response.cost on cache hits (oai/client.py:1230-1241), so cached replies over-report cost (upstream behaviour, not fixed here). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 2 +- .../tests/test_contract_harness.py | 25 +++++++++++++++++++ .../ag2-classic/tests/test_processor.py | 17 +++++++++++++ .../ag2-classic/tests/test_setup.py | 2 +- .../traceai_ag2_classic/_processor.py | 12 +++++---- 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 917ee3fef..18e50c202 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -110,7 +110,7 @@ covered by mapping unit tests only. | Span kind | `gen_ai.span.kind` from upstream `ag2.span.type`: `llm` → `LLM`; `agent` → `AGENT`; `tool`, `code_execution` → `TOOL`; `conversation`, `multi_conversation`, `speaker_selection`, `human_input` → `CHAIN`; `handoff` → `CHAIN` (declared upstream as TODO, never emitted at 0.14.x). `ag2.span.type` and `gen_ai.operation.name` are kept. | The `a2a-execution` span from `instrument_a2a_server` has no `ag2.span.type` and gets no kind. | | Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | | Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans carry upstream's aggregate input/output tokens for the whole chat. Future AGI sums tokens over every span in a trace, so those move to `ag2.usage.input_tokens` / `ag2.usage.output_tokens`; each LLM call counts once. | -| Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). | Conversation spans keep only upstream `gen_ai.usage.cost` (aggregate). | +| Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). Upstream also reports `response.cost` on a cache hit (`oai/client.py:1230-1241` returns the cached response with its stored cost), so with `cache_seed` set a cached reply carries the cost of the original call and Future AGI over-reports cost for cached responses; its token counts are the original call's too. Not fixed here; upstream behaviour. | Conversation spans carry upstream's aggregate `gen_ai.usage.cost`; it moves to `ag2.usage.cost`, so no `gen_ai.usage.*` or `gen_ai.cost.total` key is left on any non-LLM span. | | Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id; `gen_ai.conversation.id` keeps the chat id. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | | User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | | Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | diff --git a/python/frameworks/ag2-classic/tests/test_contract_harness.py b/python/frameworks/ag2-classic/tests/test_contract_harness.py index 32bb5f38c..79b0cc32d 100644 --- a/python/frameworks/ag2-classic/tests/test_contract_harness.py +++ b/python/frameworks/ag2-classic/tests/test_contract_harness.py @@ -188,6 +188,31 @@ def test_ag2_classic_contract_content_off_round_trip(): assert SECRET_PROMPT not in blob, span["name"] assert TOOL_SECRET_CITY not in blob, span["name"] + # Usage and cost live on LLM spans only. fi-collector promotes token and + # cost keys on any span and Observe sums them over a trace, so an + # aggregate on a conversation/agent span would count every call twice. + llm_spans = [s for s in spans if _flatten(s).get("gen_ai.span.kind") == "LLM"] + for span in spans: + attrs = _flatten(span) + if attrs.get("gen_ai.span.kind") == "LLM": + continue + leaked = [ + key + for key in attrs + if key.startswith("gen_ai.usage.") or key in ("gen_ai.cost.total", "llm.cost.total") + ] + assert not leaked, "usage/cost keys {0} on non-LLM span {1}".format(leaked, span["name"]) + assert len(llm_spans) == app["llm_requests"] + per_trace: Dict[str, List[int]] = defaultdict(lambda: [0, 0]) + for span in spans: + tokens = _flatten(span).get("gen_ai.usage.total_tokens", 0) + per_trace[span["traceId"]][0] += tokens + for span in llm_spans: + per_trace[span["traceId"]][1] += 1 + for trace_id, (total, calls) in per_trace.items(): + assert total == calls * 18, (trace_id, total, calls) + assert sum(total for total, _calls in per_trace.values()) == app["llm_requests"] * 18 + def test_ag2_classic_contract_content_on_round_trip(): with Receiver() as receiver: diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index fef478e67..6a117e7d7 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -98,6 +98,23 @@ def test_conversation_aggregate_usage_is_not_relabelled(): mapped = map_ag2_attributes(attrs) assert "gen_ai.usage.total_tokens" not in mapped assert "gen_ai.cost.total" not in mapped + # The chat-wide cost moves off the gen_ai.usage.* namespace like the tokens. + assert "gen_ai.usage.cost" not in mapped + assert mapped["ag2.usage.cost"] == 0.0002 + + +@pytest.mark.parametrize("span_type", sorted(set(EXPECTED_KINDS) - {"llm"})) +def test_no_gen_ai_usage_key_survives_on_a_non_llm_span(span_type): + attrs = { + "ag2.span.type": span_type, + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 14, + "gen_ai.usage.total_tokens": 36, + "gen_ai.usage.cost": 0.0002, + } + mapped = map_ag2_attributes(attrs) + assert not [key for key in mapped if key.startswith("gen_ai.usage.") or key == "gen_ai.cost.total"] + assert mapped["ag2.usage.total_tokens"] == 36 def test_conversation_aggregate_tokens_do_not_double_count_the_trace(): diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index b433b6805..44687020d 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -266,7 +266,7 @@ def _assert_no_content_and_no_promoted_usage_off_llm(spans): blob = json.dumps(attrs, default=str) assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob, span.name if attrs.get("gen_ai.span.kind") != "LLM": - leaked = [k for k in attrs if k in PROMOTED_USAGE_KEYS] + leaked = [k for k in attrs if k in PROMOTED_USAGE_KEYS or k.startswith("gen_ai.usage.")] assert not leaked, (span.name, leaked) diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index eedd41ff5..0b65c7c57 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -87,6 +87,7 @@ ERROR_TYPE = "error.type" INPUT_TOKENS = "gen_ai.usage.input_tokens" OUTPUT_TOKENS = "gen_ai.usage.output_tokens" +USAGE_PREFIX = "gen_ai.usage." # Every key that carries user or model content. Removed unless capture is on. CONTENT_KEYS = ( @@ -219,13 +220,14 @@ def map_ag2_attributes( mapped[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = inp + out _bundle_request_parameters(mapped) else: - # fi-collector promotes these keys into the token columns on any span + # fi-collector promotes token keys into the token columns on any span # (adapter.go inputTokenKeys/outputTokenKeys/totalTokenKeys) and Observe # sums total_tokens over a trace. Only LLM spans carry per-call usage; - # the conversation span's chat-wide sum (chat.py:81-82) would double it. - for key in (INPUT_TOKENS, OUTPUT_TOKENS, SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS): - if key in mapped: - mapped.setdefault("ag2.usage." + key.rsplit(".", 1)[1], mapped.pop(key)) + # the conversation span's chat-wide sum (chat.py:73-82) would double it. + # Every gen_ai.usage.* key, including upstream's aggregate + # gen_ai.usage.cost, moves to ag2.usage.* on non-LLM spans. + for key in [k for k in mapped if k.startswith(USAGE_PREFIX)]: + mapped.setdefault("ag2.usage." + key[len(USAGE_PREFIX) :], mapped.pop(key)) if capture_content: _surface_content(mapped, str(span_type)) From 54a587c9f26a9fdcc11656d6691ba56e3ffc88b7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:31:34 -0700 Subject: [PATCH 12/22] test(ag2-classic): cover the LLM exception path A model call that raises was untested. The loopback fake model now answers HTTP 400 to a request carrying LLM_FAIL_TRIGGER (OpenAI raises BadRequestError, no retry), and failing_llm_chat runs a chat whose only model call is rejected. - test_setup.py: the LLM span keeps status ERROR, carries error.type=BadRequestError and gen_ai.span.kind=LLM, has no usage or cost, and no content leaks on the error path. - Contract test: the same through register()'s exporting processor (which turns only UNSET into OK) and the shared Receiver; the token-sum check counts successful LLM spans only. Upstream sets error.type and re-raises (llm_wrapper.py:97-101) and the SDK sets ERROR, so these pass on the pre-fix code by design; README Errors row documents the behaviour. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 2 +- .../ag2-classic/tests/_contract_app.py | 7 +++-- .../ag2-classic/tests/_fake_openai.py | 31 +++++++++++++++++-- .../ag2-classic/tests/_scenarios.py | 23 ++++++++++++++ .../tests/test_contract_harness.py | 29 ++++++++++++++--- .../ag2-classic/tests/test_setup.py | 20 ++++++++++++ 6 files changed, 102 insertions(+), 10 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 18e50c202..3c345191f 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -114,7 +114,7 @@ covered by mapping unit tests only. | Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id; `gen_ai.conversation.id` keeps the chat id. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | | User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | | Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | -| Errors | Exceptions keep OTel's ERROR status. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | +| Errors | Exceptions keep OTel's ERROR status. A model call that raises keeps ERROR, and upstream sets `error.type` to the exception class (for example `BadRequestError`, `llm_wrapper.py:97-101`); the span has no usage or cost. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | | Retrieval | Not emitted upstream. | Always. | ## Privacy diff --git a/python/frameworks/ag2-classic/tests/_contract_app.py b/python/frameworks/ag2-classic/tests/_contract_app.py index a8f301328..0134f8094 100644 --- a/python/frameworks/ag2-classic/tests/_contract_app.py +++ b/python/frameworks/ag2-classic/tests/_contract_app.py @@ -25,7 +25,7 @@ def main(argv: list) -> int: from opentelemetry import trace as trace_api from _fake_openai import FakeOpenAI - from _scenarios import TOOL_SECRET_CITY, broken_tool_chat, group_chat, two_agent_tool_chat + from _scenarios import TOOL_SECRET_CITY, broken_tool_chat, failing_llm_chat, group_chat, two_agent_tool_chat from traceai_ag2_classic import setup global_before = type(trace_api.get_tracer_provider()).__name__ @@ -43,7 +43,10 @@ def main(argv: list) -> int: broken_tool_chat(tracing, fake) group = group_chat(tracing, fake) out["group_chat_id"] = str(group.chat_id) - out["llm_requests"] = len(fake.requests) + # The model rejects this one (HTTP 400): an LLM span that ends in an exception. + out["llm_error"] = failing_llm_chat(tracing, fake) + out["llm_requests"] = len(fake.requests) # successful model calls only + out["llm_failed_requests"] = len(fake.failed_requests) flushed = provider.force_flush(timeout_millis=10000) out["flushed"] = bool(flushed) diff --git a/python/frameworks/ag2-classic/tests/_fake_openai.py b/python/frameworks/ag2-classic/tests/_fake_openai.py index 8195b852a..13037653c 100644 --- a/python/frameworks/ag2-classic/tests/_fake_openai.py +++ b/python/frameworks/ag2-classic/tests/_fake_openai.py @@ -4,12 +4,15 @@ answers ``POST /v1/chat/completions`` with deterministic replies chosen from the request body: +* a request whose messages contain ``LLM_FAIL_TRIGGER`` gets HTTP 400 (the + OpenAI client raises ``BadRequestError`` and does not retry); it is recorded + in ``failed_requests``, not ``requests``; * a group-chat speaker-selection prompt gets ``speaker_reply`` (an agent name); * a request whose last message is a tool result gets ``final_reply``; * a request that offers tools gets one tool call to the first offered tool; * anything else gets ``text_reply``. -Every reply carries ``usage`` so token attributes can be asserted. +Every successful reply carries ``usage`` so token attributes can be asserted. """ from __future__ import annotations @@ -23,6 +26,7 @@ PROMPT_TOKENS = 11 COMPLETION_TOKENS = 7 FAKE_MODEL = "gpt-4o-mini" +LLM_FAIL_TRIGGER = "FAKE-LLM-REJECT" class FakeOpenAI: @@ -39,6 +43,7 @@ def __init__( self.speaker_reply = speaker_reply self.tool_arguments = tool_arguments if tool_arguments is not None else {"city": "Paris"} self.requests: List[Dict[str, Any]] = [] + self.failed_requests: List[Dict[str, Any]] = [] self._lock = threading.Lock() self._counter = 0 owner = self @@ -50,9 +55,21 @@ def do_POST(self) -> None: # noqa: N802 return length = int(self.headers.get("Content-Length", "0")) body = json.loads(self.rfile.read(length).decode("utf-8") or "{}") - payload = owner._reply(body) + status = HTTPStatus.OK + if owner._rejects(body): + status = HTTPStatus.BAD_REQUEST + payload = { + "error": { + "message": "fake model rejected the request", + "type": "invalid_request_error", + "param": None, + "code": "fake_rejected", + } + } + else: + payload = owner._reply(body) data = json.dumps(payload).encode("utf-8") - self.send_response(HTTPStatus.OK) + self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(data))) self.end_headers() @@ -91,6 +108,14 @@ def llm_config(self, **extra: Any) -> Dict[str, Any]: entry.update(extra) return {"config_list": [entry], "cache_seed": None} + def _rejects(self, body: Dict[str, Any]) -> bool: + messages = body.get("messages") or [] + if not any(LLM_FAIL_TRIGGER in str(m.get("content") or "") for m in messages if isinstance(m, dict)): + return False + with self._lock: + self.failed_requests.append(body) + return True + def _reply(self, body: Dict[str, Any]) -> Dict[str, Any]: with self._lock: self.requests.append(body) diff --git a/python/frameworks/ag2-classic/tests/_scenarios.py b/python/frameworks/ag2-classic/tests/_scenarios.py index b589a435a..e3e3aa7fd 100644 --- a/python/frameworks/ag2-classic/tests/_scenarios.py +++ b/python/frameworks/ag2-classic/tests/_scenarios.py @@ -73,6 +73,29 @@ def broken_tool_chat(tracing: Any, fake: Any) -> Any: return user.initiate_chat(assistant, message="call the broken tool", max_turns=2, silent=True) +def failing_llm_chat(tracing: Any, fake: Any) -> str: + """The model endpoint rejects the request (HTTP 400); the error reaches the caller. + + The prompt also carries ``SECRET_PROMPT`` so the error path is checked for + content leaks. Returns the exception class name the caller saw. + """ + from autogen import ConversableAgent + + from _fake_openai import LLM_FAIL_TRIGGER + + assistant = ConversableAgent("assistant_f", llm_config=fake.llm_config(), human_input_mode="NEVER") + user = ConversableAgent("user_f", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=1) + tracing.instrument_agent(assistant) + tracing.instrument_agent(user) + try: + user.initiate_chat( + assistant, message="{0} {1}".format(LLM_FAIL_TRIGGER, SECRET_PROMPT), max_turns=1, silent=True + ) + except Exception as exc: # the caller sees the model error + return type(exc).__name__ + raise AssertionError("the fake model was expected to reject the request") + + def group_chat(tracing: Any, fake: Any, *, message: str = "Write a haiku about tracing") -> Any: """writer -> GroupChatManager with LLM ("auto") speaker selection.""" from autogen import ConversableAgent diff --git a/python/frameworks/ag2-classic/tests/test_contract_harness.py b/python/frameworks/ag2-classic/tests/test_contract_harness.py index 79b0cc32d..d48bd3e37 100644 --- a/python/frameworks/ag2-classic/tests/test_contract_harness.py +++ b/python/frameworks/ag2-classic/tests/test_contract_harness.py @@ -135,8 +135,11 @@ def test_ag2_classic_contract_content_off_round_trip(): assert attrs["ag2.span.type"] == span_type, name assert attrs["gen_ai.span.kind"] == kind, name - # Model, provider, usage and cost on LLM spans. + # Model, provider, usage and cost on LLM spans (the one rejected call is + # asserted below). for span in names["chat gpt-4o-mini"]: + if span.get("status", {}).get("code") == "STATUS_CODE_ERROR": + continue attrs = _flatten(span) assert attrs["gen_ai.operation.name"] == "chat" assert attrs["gen_ai.provider.name"] == "openai" @@ -171,6 +174,23 @@ def test_ag2_classic_contract_content_off_round_trip(): assert broken["status"].get("code") == "STATUS_CODE_ERROR" assert names["execute_tool get_weather"][0].get("status", {}).get("code") != "STATUS_CODE_ERROR" + # A model call that raises: upstream sets error.type and re-raises + # (llm_wrapper.py:97-101). Status stays ERROR through Future AGI's + # exporting processor, which turns only UNSET into OK. + assert app["llm_error"] == "BadRequestError" + assert app["llm_failed_requests"] == 1 + llm_spans = [s for s in spans if _flatten(s).get("gen_ai.span.kind") == "LLM"] + failed_llm = [s for s in llm_spans if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] + assert len(failed_llm) == 1 + failed_attrs = _flatten(failed_llm[0]) + assert failed_attrs["error.type"] == "BadRequestError" + assert failed_attrs["ag2.span.type"] == "llm" + for key in ("gen_ai.usage.input_tokens", "gen_ai.usage.total_tokens", "gen_ai.cost.total"): + assert key not in failed_attrs, key + ok_llm = [s for s in llm_spans if s not in failed_llm] + for span in ok_llm: + assert span.get("status", {}).get("code") != "STATUS_CODE_ERROR", span["name"] + # AC-04: every span of the group chat run shares one trace id. group_trace = names["conversation writer"][0]["traceId"] for name in ("speaker_selection", "conversation checking_agent", "invoke_agent critic"): @@ -191,7 +211,6 @@ def test_ag2_classic_contract_content_off_round_trip(): # Usage and cost live on LLM spans only. fi-collector promotes token and # cost keys on any span and Observe sums them over a trace, so an # aggregate on a conversation/agent span would count every call twice. - llm_spans = [s for s in spans if _flatten(s).get("gen_ai.span.kind") == "LLM"] for span in spans: attrs = _flatten(span) if attrs.get("gen_ai.span.kind") == "LLM": @@ -202,12 +221,14 @@ def test_ag2_classic_contract_content_off_round_trip(): if key.startswith("gen_ai.usage.") or key in ("gen_ai.cost.total", "llm.cost.total") ] assert not leaked, "usage/cost keys {0} on non-LLM span {1}".format(leaked, span["name"]) - assert len(llm_spans) == app["llm_requests"] + # Each successful model call is one LLM span with 11 + 7 = 18 tokens; the + # failed call carries none. + assert len(ok_llm) == app["llm_requests"] per_trace: Dict[str, List[int]] = defaultdict(lambda: [0, 0]) for span in spans: tokens = _flatten(span).get("gen_ai.usage.total_tokens", 0) per_trace[span["traceId"]][0] += tokens - for span in llm_spans: + for span in ok_llm: per_trace[span["traceId"]][1] += 1 for trace_id, (total, calls) in per_trace.items(): assert total == calls * 18, (trace_id, total, calls) diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 44687020d..ef8face84 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -28,6 +28,7 @@ SECRET_PROMPT, TOOL_SECRET_CITY, broken_tool_chat, + failing_llm_chat, group_chat, two_agent_tool_chat, ) @@ -424,6 +425,25 @@ def test_tool_exception_sets_error_status(pipeline, fake): assert tool.status.description == "ExecutionError" +def test_llm_exception_keeps_error_status_and_error_type(pipeline, fake): + """The model call raises: upstream sets error.type and re-raises (llm_wrapper.py:97-101).""" + _provider, exporter, make = pipeline + tracing = make() + assert failing_llm_chat(tracing, fake) == "BadRequestError" + assert len(fake.failed_requests) == 1 + finished = exporter.get_finished_spans() + llm = [s for s in finished if s.attributes.get("ag2.span.type") == "llm"] + assert len(llm) == 1 + attrs = llm[0].attributes + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["error.type"] == "BadRequestError" + assert llm[0].status.status_code is StatusCode.ERROR + # A failed call has no usage: nothing to count, nothing invented. + for key in ("gen_ai.usage.total_tokens", "gen_ai.cost.total"): + assert key not in attrs + _assert_no_content_and_no_promoted_usage_off_llm(finished) + + # AC-04: group chat, one trace id ---------------------------------------------------- From af1845a17e7d190863c3b8226f8003bfb1ff5ce9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:31:43 -0700 Subject: [PATCH 13/22] docs(ag2-classic): describe the autogen 0.14.0 alias wheel correctly _guard.py called the autogen==0.14.0 wheel an "8-file alias". The PyPI wheel has 6 entries, all under autogen-0.14.0.dist-info/ (LICENSE, METADATA, NOTICE.md, WHEEL, top_level.txt, RECORD), and ships no modules. Docstring only; no behaviour change. approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py index e6b6e91bd..e6eae24d9 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_guard.py @@ -12,7 +12,8 @@ * ``autogen==0.14.1`` is a standalone distribution that ships the ``autogen`` module itself (Source: ``ag2ai/ag2classic``). -* ``autogen==0.14.0`` is an 8-file alias whose only runtime requirement is +* ``autogen==0.14.0`` is a metadata-only alias wheel (6 entries, all under + ``autogen-0.14.0.dist-info/``, no modules) whose only runtime requirement is ``ag2==0.14.0``; the ``autogen`` module is shipped by the ``ag2`` 0.14.0 distribution. Its ``autogen/opentelemetry`` tree is byte-identical to the 0.14.1 one. That exact pairing is accepted. From 2aee53b645c12e86f90bf2686dc6dcad8667bb46 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 00:56:45 -0700 Subject: [PATCH 14/22] fix(ag2-classic): reopen a disabled processor when setup() reuses it setup() on a provider that already has AG2ClassicSpanProcessor reused it and re-prepended it, but never undid a shutdown that bypassed the add_span_processor guard (an unbound TracerProvider.add_span_processor, or the multi-processor's shutdown()). A disabled processor returns early in on_end, so spans were exported unmapped and unfiltered (fail-open). Reusing the processor now reopens it. Test: test_setup_again_reopens_a_processor_that_was_shut_down fails before the fix (LLM span exported with no gen_ai.span.kind and raw gen_ai.usage.* keys) and passes after. Review finding N4 (installed review t_7aba4044). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 6 ++++-- .../ag2-classic/tests/test_setup.py | 20 +++++++++++++++++++ .../ag2-classic/traceai_ag2_classic/_setup.py | 6 ++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 3c345191f..b7dd0f58b 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -143,8 +143,10 @@ exporting processor. `setup()` keeps it first on the provider: spans. - `tracing.uninstrument()` leaves the processor in place, because agents instrumented earlier keep emitting spans. -- Editing `_active_span_processor._span_processors` yourself bypasses this; - call `setup()` again to put the processor back first. +- Editing `_active_span_processor._span_processors` yourself, or shutting the + processor down without going through `trace_provider.add_span_processor`, + bypasses this; call `setup()` again to put the processor back first and + re-enable it. ## Attribute inventory (autogen 0.14.1) diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index ef8face84..4e4ab38c5 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -227,6 +227,26 @@ def test_add_span_processor_on_a_plain_sdk_provider_keeps_one_processor_first(pi assert sum(isinstance(p, AG2ClassicSpanProcessor) for p in processors) == 1 +def test_setup_again_reopens_a_processor_that_was_shut_down(pipeline, fake): + """A shutdown that bypassed the add_span_processor guard (for example an + unbound ``TracerProvider.add_span_processor(provider, p)``) leaves the + processor disabled; on_end then skips filtering. A second setup() reuses + that processor and must turn filtering back on, not export content. + """ + _provider, exporter, make = pipeline + first = make() + first.processor.shutdown() + second = make() + assert second.processor is first.processor + + exporter.clear() + two_agent_tool_chat(second, fake) + spans = exporter.get_finished_spans() + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) + assert second.processor._disabled is False + + def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline): from autogen.oai.client import OpenAIWrapper diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index 8b580f896..421ec2da1 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -72,6 +72,12 @@ def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanPr ) if processor not in tuple(active._span_processors): _prepend(active, processor) + if processor._disabled: + # Shut down by a path that bypassed the add_span_processor guard + # (an unbound TracerProvider.add_span_processor(provider, p), or + # the multi-processor's shutdown()). A disabled processor skips + # filtering in on_end, so reusing it as is would export content. + processor._reopen() else: processor = AG2ClassicSpanProcessor(capture_content=capture_content) _prepend(active, processor) From 9df5068aa46f03d56036b01d4ef3a210683d992f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 00:58:05 -0700 Subject: [PATCH 15/22] fix(ag2-classic): warn when the provider runs span processors concurrently Content filtering relies on AG2ClassicSpanProcessor running before the exporting processor. The SDK's ConcurrentMultiSpanProcessor runs every on_end in parallel on a thread pool, so that order is not guaranteed and an exporter can serialize a span before content is removed. setup() now logs a WARNING when the provider's active processor is a ConcurrentMultiSpanProcessor; the default SynchronousMultiSpanProcessor (what fi_instrumentation.register() uses) is unchanged and silent. Test: test_setup_warns_when_span_processors_run_concurrently fails before the fix (no warning record: assert 0 == 1) and passes after. Review finding N5 (installed review t_7aba4044). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 4 +++ .../ag2-classic/tests/test_setup.py | 27 +++++++++++++++++++ .../ag2-classic/traceai_ag2_classic/_setup.py | 20 ++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index b7dd0f58b..3605239d3 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -147,6 +147,10 @@ exporting processor. `setup()` keeps it first on the provider: processor down without going through `trace_provider.add_span_processor`, bypasses this; call `setup()` again to put the processor back first and re-enable it. +- A provider built with the SDK's `ConcurrentMultiSpanProcessor` runs every + processor's `on_end` in parallel, so an exporter can send a span before its + content is removed. `setup()` logs a WARNING for it; use the default + `SynchronousMultiSpanProcessor`, which `register()` uses. ## Attribute inventory (autogen 0.14.1) diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 4e4ab38c5..0a21e7638 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import logging import os import signal import subprocess @@ -337,6 +338,32 @@ def test_setup_rejects_a_non_sdk_provider(): setup(tracer_provider=trace_api.NoOpTracerProvider()) +def test_setup_warns_when_span_processors_run_concurrently(caplog): + """ConcurrentMultiSpanProcessor runs every on_end in parallel, so an + exporter can serialize a span before this processor removed its content. + """ + from opentelemetry.sdk.trace import ConcurrentMultiSpanProcessor + + concurrent = TracerProvider(active_span_processor=ConcurrentMultiSpanProcessor()) + plain = TracerProvider() + try: + with caplog.at_level(logging.WARNING, logger="traceai_ag2_classic"): + setup(tracer_provider=concurrent, instrument_llm=False) + warnings = [ + r for r in caplog.records if r.levelno == logging.WARNING and r.name.startswith("traceai_ag2_classic") + ] + assert len(warnings) == 1, [r.getMessage() for r in warnings] + assert "ConcurrentMultiSpanProcessor" in warnings[0].getMessage() + + caplog.clear() + with caplog.at_level(logging.WARNING, logger="traceai_ag2_classic"): + setup(tracer_provider=plain, instrument_llm=False) + assert not [r for r in caplog.records if "ConcurrentMultiSpanProcessor" in r.getMessage()] + finally: + concurrent.shutdown() + plain.shutdown() + + def test_setup_rolls_back_the_llm_patch_when_an_agent_fails(fake): from autogen import ConversableAgent from autogen.oai import client as oai_client_module diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index 421ec2da1..d06c485f1 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -41,6 +41,25 @@ def _active_multi_processor(provider: Any) -> Any: return active +def _warn_if_concurrent(active: Any) -> None: + """Warn when the provider runs its span processors in parallel. + + The SDK's ``ConcurrentMultiSpanProcessor`` calls every processor's + ``on_end`` at once on a thread pool, so an exporting processor can + serialize a span before this package has removed its content. Filtering + is only ordered on the default ``SynchronousMultiSpanProcessor``. + """ + from opentelemetry.sdk.trace import ConcurrentMultiSpanProcessor + + if isinstance(active, ConcurrentMultiSpanProcessor): + logger.warning( + "traceai-ag2-classic: this tracer provider uses ConcurrentMultiSpanProcessor, which runs " + "span processors in parallel, so an exporter may send a span before message content is " + "removed and attributes are mapped. Use a provider with the default " + "SynchronousMultiSpanProcessor (for example fi_instrumentation.register())." + ) + + def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanProcessor: """Prepend our processor so it runs before the exporting processor. @@ -57,6 +76,7 @@ def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanPr processor installed, first and enabled. """ active = _active_multi_processor(provider) + _warn_if_concurrent(active) processor = getattr(getattr(provider, "add_span_processor", None), _GUARD_ATTR, None) if not isinstance(processor, AG2ClassicSpanProcessor): processor = next( From 6410498531f72048967bff258c8e78d5c32603e2 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 00:59:44 -0700 Subject: [PATCH 16/22] fix(ag2-classic): keep a later OpenAIWrapper.create wrapper on uninstrument uninstrument() restored the captured original OpenAIWrapper.create unconditionally, so a wrapper another library installed after setup() was silently dropped. The handle now records the traced function upstream installed and restores the original only while OpenAIWrapper.create is still that function; otherwise it logs a WARNING, leaves the replacement in place and keeps ownership so a later uninstrument() can still restore. setup()'s rollback path is unchanged (nothing can replace the method between the patch and the rollback). Test: test_uninstrument_leaves_a_later_third_party_wrapper_in_place fails before the fix (OpenAIWrapper.create is the original, not the third-party wrapper) and passes after. Review finding N6 (installed review t_7aba4044). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 3 ++ .../ag2-classic/tests/test_setup.py | 36 +++++++++++++++++++ .../ag2-classic/traceai_ag2_classic/_setup.py | 15 ++++++++ 3 files changed, 54 insertions(+) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 3605239d3..bc4021bf9 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -92,6 +92,9 @@ also install the `[tracing]` extra's gRPC exporter as a global provider, or spans are duplicated. `tracing.uninstrument()` restores `OpenAIWrapper.create`, so no new LLM spans. +If another library wrapped `OpenAIWrapper.create` after `setup()`, that wrapper +is left in place and a WARNING is logged; call `uninstrument()` again once it +is removed. Agents stay instrumented; upstream has no per-agent undo. The processor stays on the provider, so spans from those agents still have content removed and aggregate usage moved off non-LLM spans. It is shared by every handle on that diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 0a21e7638..704cc75b7 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -270,6 +270,42 @@ def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline): assert first.processor._disabled is False +def test_uninstrument_leaves_a_later_third_party_wrapper_in_place(pipeline, caplog): + """Another library wraps OpenAIWrapper.create after setup(). Restoring + our captured original would silently drop that wrapper. + """ + from autogen.oai import client as oai_client_module + from autogen.oai.client import OpenAIWrapper + + _provider, _exporter, make = pipeline + original = OpenAIWrapper.create + tracing = make() + traced = OpenAIWrapper.create + assert tracing.owns_llm_wrapper is True and traced is not original + + def third_party_create(self, **config): + return traced(self, **config) + + OpenAIWrapper.create = third_party_create + try: + with caplog.at_level(logging.WARNING, logger="traceai_ag2_classic"): + tracing.uninstrument() + assert OpenAIWrapper.create is third_party_create + assert oai_client_module.OpenAIWrapper.create is third_party_create + assert any( + r.levelno == logging.WARNING and "OpenAIWrapper.create" in r.getMessage() for r in caplog.records + ) + + # Once the other wrapper is gone, the same handle can still restore. + OpenAIWrapper.create = traced + tracing.uninstrument() + assert OpenAIWrapper.create is original + assert tracing.owns_llm_wrapper is False + finally: + OpenAIWrapper.create = original + oai_client_module.OpenAIWrapper.create = original + + PROMOTED_USAGE_KEYS = ( "gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index d06c485f1..817731faf 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -179,6 +179,7 @@ def __init__( self.autogen_version = autogen_version self.capture_content = capture_content self._original_create: Any = None + self._traced_create: Any = None self.owns_llm_wrapper = False # Upstream calls --------------------------------------------------------- @@ -200,6 +201,7 @@ def _instrument_llm_wrapper(self) -> None: instrument_llm_wrapper( tracer_provider=self.tracer_provider, capture_messages=self.capture_content ) + self._traced_create = OpenAIWrapper.create self.owns_llm_wrapper = True def instrument_agent(self, agent: Any) -> Any: @@ -260,15 +262,28 @@ def uninstrument(self) -> None: handle on a provider shares that one processor, so removing it here would also unfilter the other handles. It is shut down with the provider (``provider.shutdown()``). + + ``OpenAIWrapper.create`` is restored only while it is still the + traced function upstream installed for this handle. If something else + replaced it after ``setup()`` (another library's wrapper around ours), + it is left in place and a warning is logged; call ``uninstrument()`` + again once that wrapper is gone. """ if self.owns_llm_wrapper and self._original_create is not None: from autogen.oai import client as oai_client_module from autogen.oai.client import OpenAIWrapper + if OpenAIWrapper.create is not self._traced_create: + logger.warning( + "traceai-ag2-classic: OpenAIWrapper.create was replaced after setup(); leaving that " + "replacement in place. Call uninstrument() again after it is removed." + ) + return OpenAIWrapper.create = self._original_create oai_client_module.OpenAIWrapper.create = self._original_create self.owns_llm_wrapper = False self._original_create = None + self._traced_create = None def setup( From 0af3c239f4c9ed3e00e10d55239c0605c4edfc3c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:02:40 -0700 Subject: [PATCH 17/22] docs(ag2-classic): install autogen with the openai extra The documented install (`pip install "autogen>=0.14.0,<0.15"` then `pip install traceAI-ag2-classic`) has no model client: autogen's base requirements do not include openai, so the README quick start raised ImportError ('openai' is not installed) when it created an agent with an llm_config. Install and the example docstring now use `pip install traceAI-ag2-classic "autogen[openai]"`; the package itself pins autogen to >=0.14.0,<0.15, and the 0.14.0 alias forwards the extra to ag2[openai]==0.14.0. Evidence (scratch venvs, Python 3.11, quick start up to setup(), no model call): old two-line install -> ImportError from create_openai_client; new command -> QUICKSTART_OK on autogen 0.14.1 with openai 2.54.0. Review finding N1 (installed review t_7aba4044). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 15 +++++++++++---- .../frameworks/ag2-classic/examples/basic_chat.py | 3 +++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index bc4021bf9..65b670f5a 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -27,12 +27,19 @@ itself and installs no `ag2`. `ag2` 1.x ships only an `ag2` module. ## Install ```bash -pip install "autogen>=0.14.0,<0.15" -pip install traceAI-ag2-classic +pip install traceAI-ag2-classic "autogen[openai]" ``` -`autogen[tracing]` also works. That extra adds `opentelemetry-sdk` and the OTLP -gRPC exporter; this package never constructs that exporter. +`traceAI-ag2-classic` pins `autogen` to `>=0.14.0,<0.15`. Plain `autogen` +installs no model client; the `openai` extra adds the OpenAI client the quick +start below needs (without it, creating an agent with an `llm_config` raises +`ImportError: ... 'openai' is not installed`). For another provider, use its +extra instead or as well, for example `"autogen[anthropic]"` or +`"autogen[gemini]"`. + +`autogen[openai,tracing]` also works. The `tracing` extra adds +`opentelemetry-sdk` and the OTLP gRPC exporter; this package never constructs +that exporter. ```bash export FI_API_KEY="YOUR_API_KEY" diff --git a/python/frameworks/ag2-classic/examples/basic_chat.py b/python/frameworks/ag2-classic/examples/basic_chat.py index c7092b862..1c8afd393 100644 --- a/python/frameworks/ag2-classic/examples/basic_chat.py +++ b/python/frameworks/ag2-classic/examples/basic_chat.py @@ -1,5 +1,8 @@ """Two-agent AG2 Classic chat with a tool, traced to Future AGI. +Install: pip install traceAI-ag2-classic "autogen[openai]" +(plain autogen has no OpenAI client; the openai extra adds it). + Requires FI_API_KEY, FI_SECRET_KEY and OPENAI_API_KEY in the environment. This example calls a real model; the package tests use a loopback fake instead. """ From 9c2d57dbf1d0c91632efc249d6e4ba832e960643 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:04:02 -0700 Subject: [PATCH 18/22] docs(ag2-classic): say which spans keep the chat id under using_session The Session row said gen_ai.conversation.id "keeps the chat id" inside using_session / using_attributes. That holds only for the conversation spans initiate_chat / a_initiate_chat start (upstream sets the key after start, chat.py:65,131). fi's using_session also puts the session id in context under gen_ai.conversation.id, and the processor copies missing context keys at on_start, so LLM, agent, tool, initiate_chats and resumed conversation spans carry the session id there. README reworded. Tests pin the documented behaviour (unit: chat m span; real autogen run: every non-conversation span). They pass before and after by design (doc fix, no code change); a mutation that skips copying gen_ai.conversation.id from context makes both fail with KeyError. Review finding N3 (installed review t_7aba4044). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 2 +- python/frameworks/ag2-classic/tests/test_processor.py | 2 ++ python/frameworks/ag2-classic/tests/test_setup.py | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 65b670f5a..ba02673ab 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -121,7 +121,7 @@ covered by mapping unit tests only. | Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | | Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans carry upstream's aggregate input/output tokens for the whole chat. Future AGI sums tokens over every span in a trace, so those move to `ag2.usage.input_tokens` / `ag2.usage.output_tokens`; each LLM call counts once. | | Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). Upstream also reports `response.cost` on a cache hit (`oai/client.py:1230-1241` returns the cached response with its stored cost), so with `cache_seed` set a cached reply carries the cost of the original call and Future AGI over-reports cost for cached responses; its token counts are the original call's too. Not fixed here; upstream behaviour. | Conversation spans carry upstream's aggregate `gen_ai.usage.cost`; it moves to `ag2.usage.cost`, so no `gen_ai.usage.*` or `gen_ai.cost.total` key is left on any non-LLM span. | -| Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id; `gen_ai.conversation.id` keeps the chat id. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | +| Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id. Only `conversation` spans from `initiate_chat` / `a_initiate_chat` keep their chat id in `gen_ai.conversation.id`; every other AG2 span (LLM, agent, tool, `initiate_chats`, resumed conversations) carries the session id there, as Future AGI's own tracer does. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | | User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | | Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | | Errors | Exceptions keep OTel's ERROR status. A model call that raises keeps ERROR, and upstream sets `error.type` to the exception class (for example `BadRequestError`, `llm_wrapper.py:97-101`); the span has no usage or cost. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index 6a117e7d7..9b49ade33 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -337,6 +337,8 @@ def test_context_attributes_are_copied_onto_ag2_spans(): assert tuple(by_name[name]["tag.tags"]) == ("t1",), name # The user's session wins over the upstream chat id; the chat id is kept. assert by_name["conversation c"]["gen_ai.conversation.id"] == "chat-1" + # Other AG2 spans carry the session id there (fi using_* sets it in context). + assert by_name["chat m"]["gen_ai.conversation.id"] == "my-session" # Spans from other instrumentations are left to their own tracer. assert "session.id" not in by_name["x"] and "user.id" not in by_name["x"] diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 704cc75b7..418d49bb6 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -604,6 +604,11 @@ def test_using_attributes_sets_session_and_user_on_ag2_spans(pipeline, fake): assert span.attributes["user.id"] == "u-1", span.name conversation = next(s for s in finished if s.name == "conversation user").attributes assert conversation["gen_ai.conversation.id"] == str(result.chat_id) + # Only the initiate_chat conversation span keeps the chat id there; every + # other AG2 span carries the session id that using_attributes put in context. + for span in finished: + if span.name != "conversation user": + assert span.attributes["gen_ai.conversation.id"] == "my-session", span.name def _initiate_chats(tracing, fake): From 2f1a5fc5873f3bc1081d0860ba695a7961dd1862 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:04:15 -0700 Subject: [PATCH 19/22] docs(ag2-classic): document the live-span tracking cap The nested-conversation check tracks every live span on the provider and stops tracking at _MAX_LIVE_SPANS (10000); entries are freed in on_end. A README Limits section now says so, and that the nested-session split returns only if that many spans leak or run concurrently on the provider. Upstream ends every span it starts (each instrumentator uses `with tracer.start_as_current_span`, checked in autogen 0.14.1), so a leak needs other code on the same provider. Docs only; no code change. Review finding N2 (installed review t_7aba4044). approved: Nikhil 2026-10-03 blanket Refs: TH-8237 --- python/frameworks/ag2-classic/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index ba02673ab..4e726a52b 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -127,6 +127,17 @@ covered by mapping unit tests only. | Errors | Exceptions keep OTel's ERROR status. A model call that raises keeps ERROR, and upstream sets `error.type` to the exception class (for example `BadRequestError`, `llm_wrapper.py:97-101`); the span has no usage or cost. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | | Retrieval | Not emitted upstream. | Always. | +## Limits + +- To tell a nested chat from the outermost one, `AG2ClassicSpanProcessor` + tracks every live span on the provider (AG2 or not) and frees each entry + when its span ends. Tracking caps at 10000 live spans. The nested-session + split (an inner chat getting its own `session.id`) returns only if that many + spans leak (start and never end) or run concurrently on the provider; spans + started past the cap are not tracked. Upstream ends every span it starts + (`with tracer.start_as_current_span(...)`), so a leak needs other code on + the same provider. + ## Privacy Content is off unless you opt in with `setup(..., capture_content=True)`. From d84bf89b7b84f05439079f55bf2f844d6acfe1a0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:01:41 -0700 Subject: [PATCH 20/22] fix(ag2-classic): leave a zero AG2 cost to Future AGI's pricing AG2 reports cost 0 for any model missing from its price table ("Model ... is not found. The cost will be 0."). The processor copied that 0 to gen_ai.cost.total, and Future AGI takes gen_ai.cost.total as the cost and skips its own token pricing and the org's custom model prices. A real run with claude-sonnet-5 through an OpenAI-compatible endpoint stored cost 0 on every LLM span; without the 0 Future AGI prices those calls (0.0014 each). Copy the upstream cost only when it is above 0. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/ag2-classic/CHANGELOG.md | 5 +++-- python/frameworks/ag2-classic/README.md | 2 +- .../ag2-classic/tests/test_processor.py | 19 +++++++++++++++++++ .../traceai_ag2_classic/_processor.py | 10 +++++++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/python/frameworks/ag2-classic/CHANGELOG.md b/python/frameworks/ag2-classic/CHANGELOG.md index 40fde0c86..8788e92d9 100644 --- a/python/frameworks/ag2-classic/CHANGELOG.md +++ b/python/frameworks/ag2-classic/CHANGELOG.md @@ -6,8 +6,9 @@ second provider or exporter. - `AG2ClassicSpanProcessor` maps `ag2.span.type` to `gen_ai.span.kind`, copies the outermost `gen_ai.conversation.id` to `session.id`, adds - `gen_ai.cost.total` and `gen_ai.usage.total_tokens` on LLM spans, and sets - ERROR status from upstream `error.type`. + `gen_ai.cost.total` (only when upstream's cost is above 0; AG2 reports 0 + for models it cannot price) and `gen_ai.usage.total_tokens` on LLM spans, + and sets ERROR status from upstream `error.type`. - Content off by default: upstream message, tool, human-input and code-output keys are removed unless `capture_content=True`. - Version guard: rejects non-0.14 `autogen`, Microsoft AutoGen, `ag2` 1.x and diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 4e726a52b..6ce7bc631 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -120,7 +120,7 @@ covered by mapping unit tests only. | Span kind | `gen_ai.span.kind` from upstream `ag2.span.type`: `llm` → `LLM`; `agent` → `AGENT`; `tool`, `code_execution` → `TOOL`; `conversation`, `multi_conversation`, `speaker_selection`, `human_input` → `CHAIN`; `handoff` → `CHAIN` (declared upstream as TODO, never emitted at 0.14.x). `ag2.span.type` and `gen_ai.operation.name` are kept. | The `a2a-execution` span from `instrument_a2a_server` has no `ag2.span.type` and gets no kind. | | Model, provider | `gen_ai.request.model`, `gen_ai.provider.name` on LLM, agent and conversation spans; `gen_ai.response.model` on LLM spans. Upstream keys, passed through. | Agents without an LLM config carry no model or provider. | | Tokens | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` on LLM spans (from the response `usage`), plus `gen_ai.usage.total_tokens` added here. | A response without `usage` gives no token keys. Conversation spans carry upstream's aggregate input/output tokens for the whole chat. Future AGI sums tokens over every span in a trace, so those move to `ag2.usage.input_tokens` / `ag2.usage.output_tokens`; each LLM call counts once. | -| Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table). Upstream also reports `response.cost` on a cache hit (`oai/client.py:1230-1241` returns the cached response with its stored cost), so with `cache_seed` set a cached reply carries the cost of the original call and Future AGI over-reports cost for cached responses; its token counts are the original call's too. Not fixed here; upstream behaviour. | Conversation spans carry upstream's aggregate `gen_ai.usage.cost`; it moves to `ag2.usage.cost`, so no `gen_ai.usage.*` or `gen_ai.cost.total` key is left on any non-LLM span. | +| Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table) when it is above 0. AG2 reports 0 for a model its price table lacks (it logs "The cost will be 0"); that 0 is not copied, so Future AGI prices the call from its tokens and its own model table. Upstream also reports `response.cost` on a cache hit (`oai/client.py:1230-1241` returns the cached response with its stored cost), so with `cache_seed` set a cached reply carries the cost of the original call and Future AGI over-reports cost for cached responses; its token counts are the original call's too. Not fixed here; upstream behaviour. | Conversation spans carry upstream's aggregate `gen_ai.usage.cost`; it moves to `ag2.usage.cost`, so no `gen_ai.usage.*` or `gen_ai.cost.total` key is left on any non-LLM span. | | Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id. Only `conversation` spans from `initiate_chat` / `a_initiate_chat` keep their chat id in `gen_ai.conversation.id`; every other AG2 span (LLM, agent, tool, `initiate_chats`, resumed conversations) carries the session id there, as Future AGI's own tracer does. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | | User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | | Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index 9b49ade33..cace64083 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -82,6 +82,25 @@ def test_llm_span_gets_total_tokens_cost_and_parameters(): assert json.loads(mapped["gen_ai.request.parameters"]) == {"temperature": 0.2, "max_tokens": 64} +def test_zero_upstream_cost_is_not_reported_as_the_cost(): + # AG2 reports cost 0 for any model missing from its price table ("Model + # ... is not found. The cost will be 0."). gen_ai.cost.total=0 would stop + # Future AGI from pricing the call from its own tokens and model table. + mapped = map_ag2_attributes( + { + "ag2.span.type": "llm", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "claude-sonnet-5", + "gen_ai.usage.input_tokens": 477, + "gen_ai.usage.output_tokens": 78, + "gen_ai.usage.cost": 0, + } + ) + assert "gen_ai.cost.total" not in mapped + assert mapped["gen_ai.usage.cost"] == 0 # upstream key kept + assert mapped["gen_ai.usage.total_tokens"] == 555 + + def test_llm_span_without_usage_records_no_tokens(): mapped = map_ag2_attributes({"ag2.span.type": "llm", "gen_ai.operation.name": "chat"}) assert "gen_ai.usage.total_tokens" not in mapped diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index 0b65c7c57..5959780f2 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -19,8 +19,10 @@ ``using_attributes``: ``session.id``, ``user.id``, ``metadata``, ...) copied onto AG2 spans when they start; a context ``session.id`` wins over the conversation id. -* ``gen_ai.cost.total`` from upstream ``gen_ai.usage.cost`` on LLM spans, and - ``gen_ai.usage.total_tokens`` = input + output on LLM spans. +* ``gen_ai.cost.total`` from upstream ``gen_ai.usage.cost`` on LLM spans when + it is above 0, and ``gen_ai.usage.total_tokens`` = input + output on LLM + spans. Upstream reports 0 for any model missing from its price table, and a + ``gen_ai.cost.total`` of 0 would stop Future AGI from pricing the call itself. * ``gen_ai.request.parameters`` JSON from upstream ``gen_ai.request.*`` sampling keys. * Status ``ERROR`` when upstream set ``error.type`` but left the status unset @@ -208,7 +210,9 @@ def map_ag2_attributes( if kind == FiSpanKindValues.LLM.value: cost = mapped.get(UPSTREAM_COST) - if isinstance(cost, (int, float)) and SpanAttributes.GEN_AI_COST_TOTAL not in mapped: + # 0 means "not in AG2's price table" (oai/client.py logs "The cost will + # be 0"); leave the cost to Future AGI's own pricing then. + if isinstance(cost, (int, float)) and cost > 0 and SpanAttributes.GEN_AI_COST_TOTAL not in mapped: mapped[SpanAttributes.GEN_AI_COST_TOTAL] = cost inp = mapped.get(INPUT_TOKENS) out = mapped.get(OUTPUT_TOKENS) From a68ed5b2b4240f3c1715ec0d16e3dc9943c98300 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:42:49 -0700 Subject: [PATCH 21/22] fix(ag2-classic): require fi-instrumentation-otel>=1.1.0 fi-instrumentation-otel 1.0.0's register() raises AttributeError ('HTTPSpanExporter' object has no attribute '_headers') with opentelemetry-exporter-otlp-proto-http 1.45, so a fresh install that resolves 1.0.0 crashes on startup. Use the floor the other TH-8103 packages already declare. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/ag2-classic/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/frameworks/ag2-classic/pyproject.toml b/python/frameworks/ag2-classic/pyproject.toml index 29bc4df7e..3da055998 100644 --- a/python/frameworks/ag2-classic/pyproject.toml +++ b/python/frameworks/ag2-classic/pyproject.toml @@ -29,7 +29,7 @@ python = ">=3.10,<3.14" # AG2 Classic: the `autogen` distribution only. Never vendored or bundled. # No dependency on `ag2` 1.x, `pyautogen` or `autogen-agentchat`. autogen = ">=0.14.0,<0.15" -fi-instrumentation-otel = ">=1.0.0" +fi-instrumentation-otel = ">=1.1.0" opentelemetry-api = ">=1.29.0" opentelemetry-sdk = ">=1.29.0" From 695f8b8f27ef438f5f78e7cb16ca2e691514700a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 17:54:29 -0700 Subject: [PATCH 22/22] fix(ag2-classic): follow the traceAI TraceConfig content convention Content (messages, tool arguments/results, human input, code output) is now exported by default and hidden with FI_HIDE_INPUTS / FI_HIDE_OUTPUTS (and the message/text variants) or setup(config=TraceConfig(...)), like every traceAI instrumentor. capture_content=True/False stays as an optional override. The processor hides the input and output sides independently. Re-verified 2026-10-16 end to end on autogen 0.14.1 and 0.14.0 against a local Future AGI stack; 105 tests pass on both. Co-Authored-By: Claude Sonnet 5.5 --- python/frameworks/ag2-classic/CHANGELOG.md | 7 +- python/frameworks/ag2-classic/README.md | 52 +++++++++---- .../ag2-classic/examples/basic_chat.py | 3 +- .../ag2-classic/tests/test_processor.py | 31 +++++++- .../ag2-classic/tests/test_setup.py | 52 ++++++++++++- .../traceai_ag2_classic/_processor.py | 77 ++++++++++++++----- .../ag2-classic/traceai_ag2_classic/_setup.py | 50 +++++++++--- 7 files changed, 220 insertions(+), 52 deletions(-) diff --git a/python/frameworks/ag2-classic/CHANGELOG.md b/python/frameworks/ag2-classic/CHANGELOG.md index 8788e92d9..5de6346b3 100644 --- a/python/frameworks/ag2-classic/CHANGELOG.md +++ b/python/frameworks/ag2-classic/CHANGELOG.md @@ -9,7 +9,10 @@ `gen_ai.cost.total` (only when upstream's cost is above 0; AG2 reports 0 for models it cannot price) and `gen_ai.usage.total_tokens` on LLM spans, and sets ERROR status from upstream `error.type`. -- Content off by default: upstream message, tool, human-input and code-output - keys are removed unless `capture_content=True`. +- Content follows the traceAI `TraceConfig` convention: captured by default, + hidden with `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` (and the message/text + variants) or `setup(config=TraceConfig(...))`. Upstream message, tool, + human-input and code-output keys of a hidden side are removed before export. + `capture_content=True/False` is an optional override. - Version guard: rejects non-0.14 `autogen`, Microsoft AutoGen, `ag2` 1.x and `pyautogen` layouts with an error that names the matching package. diff --git a/python/frameworks/ag2-classic/README.md b/python/frameworks/ag2-classic/README.md index 6ce7bc631..34daf61de 100644 --- a/python/frameworks/ag2-classic/README.md +++ b/python/frameworks/ag2-classic/README.md @@ -4,8 +4,9 @@ Send AG2 Classic traces to Future AGI. AG2 Classic is the `autogen` distribution at 0.14.x, imported as `autogen`, from `ag2ai/ag2classic`. It ships its own OpenTelemetry instrumentation, `autogen.opentelemetry`. This package turns that on against a Future AGI tracer provider and maps its span attributes to Future -AGI conventions. Alpha. Validated against fixtures and a loopback fake model; no -live vendor run is claimed. +AGI conventions. Alpha. Validated against fixtures and a loopback fake model, and re-verified +2026-10-16 end to end against a local Future AGI stack with a real model +(autogen 0.14.1 and 0.14.0). | You installed | Import | Package | |---|---|---| @@ -87,7 +88,7 @@ never called unless you ask. 0.14.0 alias pairing. 2. Puts `AG2ClassicSpanProcessor` first on the provider you pass, ahead of the Future AGI exporter. -3. Calls upstream `instrument_llm_wrapper(capture_messages=capture_content)`, +3. Calls upstream `instrument_llm_wrapper(capture_messages=...)` (on unless every content side is hidden), then `instrument_agent` and `instrument_pattern` for what you pass. If one of those raises (for example a non-agent in `agents=`), `setup()` restores `OpenAIWrapper.create` and re-raises. @@ -123,7 +124,7 @@ covered by mapping unit tests only. | Cost | `gen_ai.cost.total` on LLM spans, copied from upstream `gen_ai.usage.cost` (AG2's own price table) when it is above 0. AG2 reports 0 for a model its price table lacks (it logs "The cost will be 0"); that 0 is not copied, so Future AGI prices the call from its tokens and its own model table. Upstream also reports `response.cost` on a cache hit (`oai/client.py:1230-1241` returns the cached response with its stored cost), so with `cache_seed` set a cached reply carries the cost of the original call and Future AGI over-reports cost for cached responses; its token counts are the original call's too. Not fixed here; upstream behaviour. | Conversation spans carry upstream's aggregate `gen_ai.usage.cost`; it moves to `ag2.usage.cost`, so no `gen_ai.usage.*` or `gen_ai.cost.total` key is left on any non-LLM span. | | Session | `session.id` from upstream `gen_ai.conversation.id` (`ChatResult.chat_id`) on the outermost `conversation` span. Inside `using_session(...)` or `using_attributes(session_id=...)` (from `fi_instrumentation`), that id is set on every AG2 span instead and wins over the chat id. Only `conversation` spans from `initiate_chat` / `a_initiate_chat` keep their chat id in `gen_ai.conversation.id`; every other AG2 span (LLM, agent, tool, `initiate_chats`, resumed conversations) carries the session id there, as Future AGI's own tracer does. | Nested chats (for example the internal chat behind group-chat speaker selection, or an inner chat a tool starts) keep `gen_ai.conversation.id` but get no `session.id`. Child spans end before the chat id exists, so only the conversation span carries it. A `GroupChatManager` run emits no own conversation span at 0.14.x. `initiate_chats` gives one session per chat: each chat is its own outermost conversation, and the parent `initiate_chats` span (`multi_conversation`) has no `session.id`, so one trace holds several sessions. For one session over the whole call, wrap it in `with using_session("..."):`. | | User | `user.id` (and `metadata`, `tag.tags`) from `using_user(...)` / `using_attributes(...)`, set on every AG2 span started inside it. | Not emitted upstream; absent without that context. | -| Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) only with `capture_content=True`. | +| Tools | `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`. | Arguments and result (`gen_ai.tool.call.arguments` / `.result`) hidden by `FI_HIDE_INPUTS` (arguments) / `FI_HIDE_OUTPUTS` (result). | | Errors | Exceptions keep OTel's ERROR status. A model call that raises keeps ERROR, and upstream sets `error.type` to the exception class (for example `BadRequestError`, `llm_wrapper.py:97-101`); the span has no usage or cost. A failing tool is caught inside `ConversableAgent.execute_function`, so upstream only sets `error.type=ExecutionError`; this package sets status ERROR from `error.type`. A non-zero code exit (`error.type=CodeExecutionError`) also becomes ERROR. | | | Retrieval | Not emitted upstream. | Always. | @@ -140,18 +141,39 @@ covered by mapping unit tests only. ## Privacy -Content is off unless you opt in with `setup(..., capture_content=True)`. +Content follows the traceAI `TraceConfig` convention, like every traceAI +instrumentor: message bodies, tool arguments and results are exported by +default, and hidden with the standard variables (no package-specific switch): + +```bash +export FI_HIDE_INPUTS=true # input side: messages in, tool arguments, human prompt +export FI_HIDE_OUTPUTS=true # output side: messages out, tool results, code output, chat summaries +# also honoured: FI_HIDE_INPUT_MESSAGES, FI_HIDE_OUTPUT_MESSAGES, FI_HIDE_INPUT_TEXT, FI_HIDE_OUTPUT_TEXT +``` + +or in code, `setup(..., config=TraceConfig(hide_inputs=True, hide_outputs=True))` +(`from fi_instrumentation import TraceConfig`). `setup(..., capture_content=False)` +hides both sides and `capture_content=True` keeps both; either overrides `config`. Upstream has a content flag only on `instrument_llm_wrapper` -(`capture_messages`, default False). Its agent, conversation, tool, -human-input and code-execution spans always record message bodies, tool -arguments and results, the human prompt and answer, code output and chat -summaries. With content off this package removes these keys before export: -`gen_ai.input.messages`, `gen_ai.output.messages`, `gen_ai.tool.call.arguments`, -`gen_ai.tool.call.result`, `ag2.human_input.prompt`, `ag2.human_input.response`, -`ag2.code_execution.output`, `ag2.chats.summaries`, and any `input.value` / -`output.value`. That is stricter than upstream. With content on they are kept -and lifted into `input.value` / `output.value`. +(`capture_messages`). Its agent, conversation, tool, human-input and +code-execution spans always record message bodies, tool arguments and results, +the human prompt and answer, code output and chat summaries. This package +removes the keys of a hidden side before export. Input side: +`gen_ai.input.messages`, `gen_ai.tool.call.arguments`, `ag2.human_input.prompt` +and `input.value`. Output side: `gen_ai.output.messages`, +`gen_ai.tool.call.result`, `ag2.human_input.response`, +`ag2.code_execution.output`, `ag2.chats.summaries` and `output.value`. That is +stricter than upstream. Kept content is lifted into `input.value` / +`output.value`. Note that a conversation's output messages carry the whole chat +history, so the prompt also appears there unless `FI_HIDE_OUTPUTS` is set. + +Never exported in either mode: local paths, user names, host names, e-mail +addresses and stack traces (spans carry the exception class in `error.type` and +OpenTelemetry's status; checked on a real run). Exception messages from the +model provider or your tool do appear in the span status text, as with any +OpenTelemetry instrumentation; hide them by not putting secrets in exception +messages. The filtering lives in `AG2ClassicSpanProcessor`, which must run before any exporting processor. `setup()` keeps it first on the provider: @@ -206,5 +228,5 @@ Paths are relative to `autogen/opentelemetry/`. Scope name when you call it (see Privacy). - `setup()` raises and names three packages: you have Microsoft AutoGen, `ag2` 1.x, or `pyautogen` instead of `autogen` 0.14.x. -- Content missing: content is off by default here; pass `capture_content=True`. +- Content missing: check `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` (and the `_MESSAGES` / `_TEXT` variants) and any `TraceConfig` you pass to `setup()`. - Exporter errors are logged and never fail the agent. diff --git a/python/frameworks/ag2-classic/examples/basic_chat.py b/python/frameworks/ag2-classic/examples/basic_chat.py index 1c8afd393..48a1ad614 100644 --- a/python/frameworks/ag2-classic/examples/basic_chat.py +++ b/python/frameworks/ag2-classic/examples/basic_chat.py @@ -39,7 +39,8 @@ def main() -> None: assistant.register_for_llm(description="Get the weather")(get_weather) user.register_for_execution()(get_weather) - # Content (messages, tool arguments/results) is off unless capture_content=True. + # Content (messages, tool arguments/results) is exported unless FI_HIDE_INPUTS / + # FI_HIDE_OUTPUTS are set (traceAI TraceConfig convention). setup(tracer_provider=trace_provider, agents=[assistant, user]) user.initiate_chat(assistant, message="What is the weather in Paris?", max_turns=3) diff --git a/python/frameworks/ag2-classic/tests/test_processor.py b/python/frameworks/ag2-classic/tests/test_processor.py index cace64083..242ba4d29 100644 --- a/python/frameworks/ag2-classic/tests/test_processor.py +++ b/python/frameworks/ag2-classic/tests/test_processor.py @@ -193,15 +193,42 @@ def test_existing_session_id_wins(): @pytest.mark.parametrize("span_type", sorted(EXPECTED_KINDS)) -def test_content_is_dropped_by_default(span_type): +def test_content_is_dropped_when_hidden(span_type): attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": span_type, "gen_ai.agent.name": "a"}) - mapped = map_ag2_attributes(attrs) + mapped = map_ag2_attributes(attrs, capture_content=False) for key in CONTENT_KEYS: assert key not in mapped assert "secret" not in json.dumps(mapped) assert mapped["gen_ai.agent.name"] == "a" +@pytest.mark.parametrize("span_type", sorted(EXPECTED_KINDS)) +def test_content_is_kept_by_default(span_type): + # traceAI TraceConfig convention: content on unless FI_HIDE_* says otherwise. + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": span_type}) + mapped = map_ag2_attributes(attrs) + for key in CONTENT_KEYS: + if key in attrs: + assert mapped[key] == attrs[key] + + +def test_hide_inputs_only_removes_the_input_side(): + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": "tool"}) + mapped = map_ag2_attributes(attrs, hide_inputs=True) + assert "gen_ai.tool.call.arguments" not in mapped + assert "input.value" not in mapped + assert mapped["gen_ai.tool.call.result"] == attrs["gen_ai.tool.call.result"] + assert mapped["output.value"] == attrs["gen_ai.tool.call.result"] + + +def test_hide_outputs_only_removes_the_output_side(): + attrs = dict(CONTENT_SAMPLE, **{"ag2.span.type": "tool"}) + mapped = map_ag2_attributes(attrs, hide_outputs=True) + assert "gen_ai.tool.call.result" not in mapped + assert "output.value" not in mapped + assert mapped["gen_ai.tool.call.arguments"] == attrs["gen_ai.tool.call.arguments"] + + def test_capture_on_lifts_messages_to_input_output(): attrs = { "ag2.span.type": "agent", diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 418d49bb6..aa47805c2 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -48,6 +48,9 @@ def pipeline(): handles = [] def make(**kwargs): + # Most tests exercise the hidden-content path; the shown-by-default + # behaviour has its own tests below (they pass capture_content=None). + kwargs.setdefault("capture_content", False) handle = setup(tracer_provider=provider, **kwargs) handles.append(handle) return handle @@ -189,7 +192,7 @@ def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(m verbose=False, set_global_tracer_provider=False, ) - tracing = setup(tracer_provider=provider) + tracing = setup(tracer_provider=provider, capture_content=False) memory = InMemorySpanExporter() try: added = SimpleSpanProcessor(memory) @@ -495,6 +498,53 @@ def test_capture_content_on_keeps_and_lifts_content(pipeline, fake): assert SECRET_PROMPT in llm["input.value"] +def test_default_follows_traceconfig_and_keeps_content(pipeline, fake, monkeypatch): + """traceAI convention: no FI_HIDE_* means content is exported (no second switch).""" + for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_INPUT_MESSAGES", "FI_HIDE_OUTPUT_MESSAGES"): + monkeypatch.delenv(name, raising=False) + _provider, exporter, make = pipeline + tracing = make(capture_content=None) + assert tracing.capture_content is True + two_agent_tool_chat(tracing, fake) + spans = _by_name(exporter) + assert SECRET_PROMPT in spans["conversation user"][0].attributes["input.value"] + tool = spans["execute_tool get_weather"][0].attributes + assert json.loads(tool["gen_ai.tool.call.arguments"]) == {"city": TOOL_SECRET_CITY} + llm = spans["chat {0}".format(FAKE_MODEL)][0].attributes + assert SECRET_PROMPT in llm["gen_ai.input.messages"] + + +def test_fi_hide_env_vars_hide_all_content(pipeline, fake, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + _provider, exporter, make = pipeline + tracing = make(capture_content=None) + assert tracing.capture_content is False + two_agent_tool_chat(tracing, fake) + for span in exporter.get_finished_spans(): + for key in CONTENT_KEYS: + assert key not in span.attributes, (key, span.name) + blob = json.dumps(dict(span.attributes), default=str) + assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob + + +def test_traceconfig_hide_inputs_keeps_outputs(pipeline, fake): + from fi_instrumentation import TraceConfig + + _provider, exporter, make = pipeline + tracing = make(capture_content=None, config=TraceConfig(hide_inputs=True)) + two_agent_tool_chat(tracing, fake) + tool = _by_name(exporter)["execute_tool get_weather"][0].attributes + assert "gen_ai.tool.call.arguments" not in tool + assert tool["gen_ai.tool.call.result"] == "sunny in {0}".format(TOOL_SECRET_CITY) or TOOL_SECRET_CITY in tool["gen_ai.tool.call.result"] + for span in exporter.get_finished_spans(): + assert "gen_ai.input.messages" not in span.attributes + # The chat-wide output history still carries the prompt; only the + # input side is hidden. + assert "gen_ai.tool.call.arguments" not in span.attributes + assert "input.value" not in span.attributes + + # AC-07: tool exception -> ERROR --------------------------------------------------- diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py index 5959780f2..5aa6c15df 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_processor.py @@ -28,12 +28,13 @@ * Status ``ERROR`` when upstream set ``error.type`` but left the status unset (tool failures are caught inside ``ConversableAgent.execute_function``, so upstream never sets the status itself). -* Content: off by default. Upstream agent, conversation, tool, human-input and +* Content follows the traceAI ``TraceConfig`` convention: captured by default, + hidden with ``FI_HIDE_INPUTS`` / ``FI_HIDE_OUTPUTS`` (and the message/text + variants). Upstream agent, conversation, tool, human-input and code-execution spans always record message bodies, tool arguments/results, - prompts and code output; there is no upstream flag for those. With - ``capture_content=False`` (default) those keys are removed. With - ``capture_content=True`` they are kept and lifted into ``input.value`` / - ``output.value``. + prompts and code output; there is no upstream flag for those, so when a side + is hidden this processor removes its keys before export. Kept content is + lifted into ``input.value`` / ``output.value``. Spans from any other instrumentation scope pass through untouched. The processor never raises into the SDK. @@ -91,21 +92,39 @@ OUTPUT_TOKENS = "gen_ai.usage.output_tokens" USAGE_PREFIX = "gen_ai.usage." -# Every key that carries user or model content. Removed unless capture is on. -CONTENT_KEYS = ( +# Keys that carry user (input side) or model/tool (output side) content. +# Removed when the matching side is hidden. +INPUT_CONTENT_KEYS = ( INPUT_MESSAGES, - OUTPUT_MESSAGES, TOOL_ARGUMENTS, - TOOL_RESULT, HUMAN_PROMPT, + SpanAttributes.INPUT_VALUE, + SpanAttributes.INPUT_MIME_TYPE, +) +OUTPUT_CONTENT_KEYS = ( + OUTPUT_MESSAGES, + TOOL_RESULT, HUMAN_RESPONSE, CODE_OUTPUT, CHAT_SUMMARIES, - SpanAttributes.INPUT_VALUE, - SpanAttributes.INPUT_MIME_TYPE, SpanAttributes.OUTPUT_VALUE, SpanAttributes.OUTPUT_MIME_TYPE, ) +CONTENT_KEYS = INPUT_CONTENT_KEYS + OUTPUT_CONTENT_KEYS + + +def resolve_hidden( + capture_content: Optional[bool], hide_inputs: bool = False, hide_outputs: bool = False +) -> Tuple[bool, bool]: + """Return ``(hide_inputs, hide_outputs)``. + + An explicit ``capture_content`` wins: True shows both sides, False hides + both. ``None`` uses ``hide_inputs`` / ``hide_outputs`` (traceAI default: + both shown). + """ + if capture_content is None: + return bool(hide_inputs), bool(hide_outputs) + return (not capture_content), (not capture_content) # utils.py set_llm_request_params (lines 213-219). REQUEST_PARAMETER_KEYS = ( @@ -180,7 +199,9 @@ def _bundle_request_parameters(mapped: Dict[str, Any]) -> None: def map_ag2_attributes( attributes: Mapping[str, Any], *, - capture_content: bool = False, + capture_content: Optional[bool] = None, + hide_inputs: bool = False, + hide_outputs: bool = False, root_conversation: bool = True, ) -> Dict[str, Any]: """Return a new attribute dict with Future AGI keys added. @@ -233,11 +254,15 @@ def map_ag2_attributes( for key in [k for k in mapped if k.startswith(USAGE_PREFIX)]: mapped.setdefault("ag2.usage." + key[len(USAGE_PREFIX) :], mapped.pop(key)) - if capture_content: - _surface_content(mapped, str(span_type)) - else: - for key in CONTENT_KEYS: + hide_in, hide_out = resolve_hidden(capture_content, hide_inputs, hide_outputs) + if hide_in: + for key in INPUT_CONTENT_KEYS: + mapped.pop(key, None) + if hide_out: + for key in OUTPUT_CONTENT_KEYS: mapped.pop(key, None) + if not (hide_in and hide_out): + _surface_content(mapped, str(span_type)) return mapped @@ -276,8 +301,14 @@ class AG2ClassicSpanProcessor(SpanProcessor): exporter sees the mapped attributes. """ - def __init__(self, capture_content: bool = False) -> None: - self.capture_content = bool(capture_content) + def __init__( + self, + capture_content: Optional[bool] = None, + *, + hide_inputs: bool = False, + hide_outputs: bool = False, + ) -> None: + self.hide_inputs, self.hide_outputs = resolve_hidden(capture_content, hide_inputs, hide_outputs) # span_id -> (parent span_id, the span if it is an AG2 span else None), # for every live span on the provider, AG2 or not. A user span between # two AG2 conversations must not hide the outer one. @@ -315,7 +346,10 @@ def on_end(self, span: ReadableSpan) -> None: if attributes.get(SPAN_TYPE_KEY) == "conversation": root = not self._has_live_conversation_ancestor(span) mapped = map_ag2_attributes( - attributes, capture_content=self.capture_content, root_conversation=root + attributes, + hide_inputs=self.hide_inputs, + hide_outputs=self.hide_outputs, + root_conversation=root, ) if mapped != attributes: _replace_attributes(span, mapped) @@ -326,6 +360,11 @@ def on_end(self, span: ReadableSpan) -> None: except Exception: # pragma: no cover - never break the SDK logger.debug("traceai-ag2-classic: span mapping failed", exc_info=True) + @property + def capture_content(self) -> bool: + """True when any content side is still exported.""" + return not (self.hide_inputs and self.hide_outputs) + def shutdown(self) -> None: self._disabled = True try: diff --git a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py index 817731faf..fd6925d05 100644 --- a/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py +++ b/python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py @@ -6,7 +6,7 @@ 2. Prepends :class:`AG2ClassicSpanProcessor` to the provider you pass, ahead of the exporting processor ``fi_instrumentation.register()`` installed. 3. Calls upstream ``instrument_llm_wrapper`` (global, patches - ``OpenAIWrapper.create``) with ``capture_messages=capture_content``. + ``OpenAIWrapper.create``) with ``capture_messages`` set when any content side is shown. 4. Calls upstream ``instrument_agent`` / ``instrument_pattern`` for the agents and patterns you pass. @@ -23,7 +23,9 @@ from opentelemetry import trace as trace_api from ._guard import check_autogen_classic -from ._processor import AG2ClassicSpanProcessor +from fi_instrumentation import TraceConfig + +from ._processor import AG2ClassicSpanProcessor, resolve_hidden logger = logging.getLogger(__name__) @@ -60,7 +62,9 @@ def _warn_if_concurrent(active: Any) -> None: ) -def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanProcessor: +def _install_processor( + provider: Any, hide_inputs: bool, hide_outputs: bool +) -> AG2ClassicSpanProcessor: """Prepend our processor so it runs before the exporting processor. ``provider.add_span_processor`` is deliberately not used: Future AGI's @@ -84,11 +88,12 @@ def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanPr None, ) if processor is not None: - if processor.capture_content != bool(capture_content): + if (processor.hide_inputs, processor.hide_outputs) != (hide_inputs, hide_outputs): logger.warning( "traceai-ag2-classic is already installed on this provider with " - "capture_content=%s; keeping that setting.", - processor.capture_content, + "hide_inputs=%s hide_outputs=%s; keeping that setting.", + processor.hide_inputs, + processor.hide_outputs, ) if processor not in tuple(active._span_processors): _prepend(active, processor) @@ -99,7 +104,7 @@ def _install_processor(provider: Any, capture_content: bool) -> AG2ClassicSpanPr # filtering in on_end, so reusing it as is would export content. processor._reopen() else: - processor = AG2ClassicSpanProcessor(capture_content=capture_content) + processor = AG2ClassicSpanProcessor(hide_inputs=hide_inputs, hide_outputs=hide_outputs) _prepend(active, processor) _guard_add_span_processor(provider, processor) return processor @@ -286,12 +291,25 @@ def uninstrument(self) -> None: self._traced_create = None +def _hidden_from_config( + config: Optional[TraceConfig], capture_content: Optional[bool] +) -> "tuple[bool, bool]": + if capture_content is not None: + return resolve_hidden(capture_content) + cfg = config if config is not None else TraceConfig() + return ( + bool(cfg.hide_inputs or cfg.hide_input_messages or cfg.hide_input_text), + bool(cfg.hide_outputs or cfg.hide_output_messages or cfg.hide_output_text), + ) + + def setup( tracer_provider: Optional[Any] = None, *, agents: Iterable[Any] = (), patterns: Iterable[Any] = (), - capture_content: bool = False, + config: Optional[TraceConfig] = None, + capture_content: Optional[bool] = None, instrument_llm: bool = True, ) -> AG2ClassicTracing: """Enable AG2 Classic's native tracing on a Future AGI tracer provider. @@ -301,8 +319,15 @@ def setup( Defaults to the global provider, which must be an SDK provider. agents: Agents to pass to upstream ``instrument_agent``. patterns: Group patterns to pass to upstream ``instrument_pattern``. - capture_content: Keep message bodies, tool arguments/results, human - input and code output on spans. Off by default. + config: A ``fi_instrumentation.TraceConfig``. Defaults to + ``TraceConfig()``, which reads ``FI_HIDE_INPUTS`` / + ``FI_HIDE_OUTPUTS`` / ``FI_HIDE_INPUT_MESSAGES`` / + ``FI_HIDE_OUTPUT_MESSAGES`` / ``FI_HIDE_INPUT_TEXT`` / + ``FI_HIDE_OUTPUT_TEXT``. Content (message bodies, tool + arguments/results, human input, code output) is captured unless + hidden, like every traceAI instrumentor. + capture_content: Optional override of ``config``. ``False`` hides all + content, ``True`` keeps all of it. instrument_llm: Call upstream ``instrument_llm_wrapper`` (LLM spans). Returns: @@ -318,12 +343,13 @@ def setup( """ version = check_autogen_classic() provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() - processor = _install_processor(provider, capture_content) + hide_inputs, hide_outputs = _hidden_from_config(config, capture_content) + processor = _install_processor(provider, hide_inputs, hide_outputs) handle = AG2ClassicTracing( tracer_provider=provider, processor=processor, autogen_version=version, - capture_content=bool(capture_content), + capture_content=processor.capture_content, ) if instrument_llm: handle._instrument_llm_wrapper()