From 1b3072bb63d3a8c51b3a296b4135b44cd9d1701c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:16:50 -0800 Subject: [PATCH 01/16] feat(python): share one attachment manager across the auto module traceai.init passes ONE attachment manager to every instrumentor it enables, as attachments=. It is the provider's manager, or one the application lends with attachment_manager= (never shut down by traceai; it only gains the hide and PII flags of trace_config). With attachments OFF, or killed by FI_ATTACHMENTS_ENABLED=false, instrumentors receive exactly the keyword arguments they received before. New public helpers: get_attachment_manager(), report() (attachment state and per-integration media support: provider_adapter, session_binding or unqualified, version aware), with_attachment_capture_policy() and with_attachment_ownership(). The shared provider-media helper reads the two context keys by name: a per-call CapturePolicy can only add restrictions, and an owner mark turns capture off for that call. The fi-instrumentation-otel floor is unchanged; attachments stay import-guarded. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 53 +++- .../attachments_providers/test_auto_init.py | 174 +++++++++++- python/traceai/README.md | 24 ++ python/traceai/__init__.py | 247 +++++++++++++++++- python/traceai/tests/conftest.py | 1 + python/traceai/tests/test_attachments.py | 166 ++++++++++++ 6 files changed, 650 insertions(+), 15 deletions(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index a2fd637c4..28fd3a94a 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,6 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- A per-call ``CapturePolicy`` set in the OpenTelemetry context under + ``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only + add restrictions. A context marked under ``OWNER_KEY`` + (``traceai.with_attachment_ownership``) belongs to an adapter that captures + the media itself, so nothing is captured here for that call. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -26,6 +31,8 @@ from collections import OrderedDict from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple +from opentelemetry import context as otel_context + from .policy import CapturePolicy from .stream import StreamOptions from .types import Bytes, CorrelationHandle @@ -50,6 +57,13 @@ # mulaw have no F01 MIME, so they are never captured. SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", "wav": "audio/wav"} +# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the +# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. +CAPTURE_POLICY_KEY = "traceai.auto.attachment_capture_policy.v1" +OWNER_KEY = "traceai.auto.attachment_owner.v1" +# CapturePolicy facts that grant capture; every other fact only restricts. +_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) +_DENY_ALL = CapturePolicy(capture_content=False, requested=False) class Occurrence(NamedTuple): @@ -93,6 +107,40 @@ def _restricted(config, name: str) -> bool: return True +def merge_capture_policies(*policies: Any) -> CapturePolicy: + """Deny-wins intersection: a grant needs every policy, a restriction needs any one. + + ``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies. + """ + present = [policy for policy in policies if policy is not None] + if any(not isinstance(policy, CapturePolicy) for policy in present): + return _DENY_ALL + if not present: + return CapturePolicy() + values = {} + for item in dataclasses.fields(CapturePolicy): + facts = [getattr(policy, item.name) for policy in present] + values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts) + return CapturePolicy(**values) + + +def context_capture_policy(context=None) -> Any: + """The per-call policy in the (current) OpenTelemetry context; None when unset.""" + try: + return otel_context.get_value(CAPTURE_POLICY_KEY, context) + except Exception: + # An unreadable restriction is a restriction, never permission. + return _DENY_ALL + + +def owned_elsewhere(context=None) -> bool: + """True when another adapter marked this call's media as its own.""" + try: + return otel_context.get_value(OWNER_KEY, context) is not None + except Exception: + return True + + def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, base: Optional[CapturePolicy] = None) -> CapturePolicy: """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. @@ -103,6 +151,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal ancestry, so a tool screenshot is still treated as model input. """ base = base if isinstance(base, CapturePolicy) else CapturePolicy() + base = merge_capture_policies(base, context_capture_policy()) names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") if image and direction == "input" and _restricted(trace_config, "hide_input_images"): @@ -116,6 +165,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal changes["model_input"] = True if error_detail: changes["error_detail"] = True + if owned_elsewhere(): + changes["requested"] = False return dataclasses.replace(base, **changes) @@ -262,7 +313,7 @@ class Observation: def __init__(self, manager, span, namespace: str): self.manager, self.span, self.namespace = manager, span, namespace - self.key = span_key(span) if active(manager) else None + self.key = span_key(span) if active(manager) and not owned_elsewhere() else None self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() self._visited = 0 diff --git a/python/tests/attachments_providers/test_auto_init.py b/python/tests/attachments_providers/test_auto_init.py index a9ba1d425..8fbebe6fb 100644 --- a/python/tests/attachments_providers/test_auto_init.py +++ b/python/tests/attachments_providers/test_auto_init.py @@ -6,8 +6,10 @@ openai = pytest.importorskip("openai") pytest.importorskip("traceai_openai") -from conftest import b64, flush, png # noqa: E402 -from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from conftest import b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy # noqa: E402 +from opentelemetry import context as otel_context # noqa: E402 from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 @@ -18,8 +20,10 @@ def auto_state(monkeypatch): monkeypatch.setattr(traceai, "_init_options", None) monkeypatch.setattr(traceai, "_instrumented", set()) monkeypatch.setattr(traceai, "_adapter_status", {}) + monkeypatch.setattr(traceai, "_attachment_manager_borrowed", False) yield - for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_anthropic", "AnthropicInstrumentor")): + for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_anthropic", "AnthropicInstrumentor"), + ("traceai_groq", "GroqInstrumentor")): try: instrumentor = getattr(__import__(module), name)() if instrumentor.is_instrumented_by_opentelemetry: @@ -66,3 +70,167 @@ def test_auto_init_with_attachments_off_is_unchanged(provider, auto_state): assert OpenAIInstrumentor().is_instrumented_by_opentelemetry finally: tracer_provider.shutdown() + + +def _enabled_init(f01, monkeypatch, libraries, **options): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + if "attachment_manager" not in options: + options["attachment_config"] = AttachmentConfig(enabled=True, endpoint=f01.endpoint) + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=libraries, + verbose=False, set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + return tracer_provider, exporter + + +def _chat_image(provider, image): + openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0).chat.completions.create( + model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}]}]) + + +def _shot(data): + return {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": b64(data)}} + + +def test_auto_init_captures_openai_image_and_anthropic_computer_screenshot(f01, provider, auto_state, monkeypatch): + """A01 end to end: ONE manager from traceai.init reaches both instrumentors and the loopback F01.""" + anthropic = pytest.importorskip("anthropic") + pytest.importorskip("traceai_anthropic") + tracer_provider, exporter = _enabled_init(f01, monkeypatch, ["openai", "anthropic"]) + manager = traceai.get_attachment_manager() + try: + assert manager is tracer_provider.attachment_manager + assert manager.ready(5), manager.status() + image, screenshot = png(seed=51), png(seed=52) + _chat_image(provider, image) + anthropic.Anthropic(api_key="sk-synthetic", base_url=provider.url, max_retries=0).messages.create( + model="claude-test", max_tokens=16, + tools=[{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, + "display_height_px": 768}], + messages=[{"role": "user", "content": [{"type": "text", "text": "open the page"}]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_9", "name": "computer", "input": {"action": "screenshot"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_9", "content": [_shot(screenshot)]}]}]) + flush(manager) + assert sorted(f01.payloads) == sorted([image, screenshot]) + uploads = {u["purpose"]: u for u in f01.uploads} + assert uploads["computer_screenshot"].get("tool_call_id") == "toolu_9" + assert uploads["model_input"]["direction"] == "input" + span_refs = [ref for span in exporter.get_finished_spans() for ref in refs(span)] + assert sorted(ref["purpose"] for ref in span_refs) == ["computer_screenshot", "model_input"] + shot = [ref for ref in span_refs if ref["purpose"] == "computer_screenshot"][0] + assert (shot["message_index"], shot["part_index"], shot["role"]) == (2, 0, "tool") + exported = str([dict(span.attributes) for span in exporter.get_finished_spans()]) + assert b64(image) not in exported and b64(screenshot) not in exported + report = traceai.report() + assert report["attachments"]["state"] == "ready" + rows = {row["name"]: row for row in report["integrations"]} + for name in ("openai", "anthropic"): + assert (rows[name]["status"], rows[name]["media"]) == ("registered", "provider_adapter") + assert "tool_result.input" in rows["anthropic"]["media_operations"] + assert "SYNTHETIC" not in str(report) and str(f01.server.server_port) not in str(report) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_report_rows_for_installed_media_adapters(auto_state): + pytest.importorskip("traceai_groq") + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert {name: rows[name]["media"] for name in ("openai", "anthropic", "groq")} == dict.fromkeys( + ("openai", "anthropic", "groq"), "provider_adapter") + assert rows["litellm"]["media"] == "unqualified" + + +def test_auto_borrowed_manager_is_shared_and_tightened(f01, provider, auto_state, monkeypatch): + from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider + + borrowed = f01.manager() + sdk = SdkTracerProvider() + try: + with sdk.get_tracer("synthetic").start_as_current_span("probe") as span: + ticket = borrowed.preflight(span, mime_type="image/png", direction="input", max_bytes=16) + assert ticket.admitted, ticket.reason + ticket.release() + tracer_provider, _ = _enabled_init(f01, monkeypatch, ["openai"], attachment_manager=borrowed, + trace_config=TraceConfig(pii_redaction=True)) + assert traceai.get_attachment_manager() is borrowed is tracer_provider.attachment_manager + assert openai.OpenAI.request._self_wrapper._attachments is borrowed + # Auto's PII flag now restricts the application's manager itself, for every caller. + with sdk.get_tracer("synthetic").start_as_current_span("probe") as span: + assert borrowed.preflight(span, mime_type="image/png", direction="input", + max_bytes=16).reason == "policy_denied" + _chat_image(provider, png(seed=53)) + flush(borrowed) + assert f01.payloads == [] + tracer_provider.shutdown() + # Auto never shuts a borrowed manager down. + assert borrowed.status().state == "ready" + finally: + borrowed.shutdown(5) + sdk.shutdown() + + +def test_auto_kill_switch_after_init_denies_every_admission(f01, provider, auto_state, monkeypatch): + tracer_provider, _ = _enabled_init(f01, monkeypatch, ["openai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5) + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + _chat_image(provider, png(seed=54)) + flush(manager) + assert f01.payloads == [] + assert traceai.report()["attachments"]["state"] == "disabled" + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED") + image = png(seed=55) + _chat_image(provider, image) + flush(manager) + assert f01.payloads == [image] + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_kill_switch_at_init_keeps_the_old_code_path(f01, provider, auto_state, monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + tracer_provider, _ = _enabled_init(f01, monkeypatch, ["openai"]) + try: + assert tracer_provider.attachment_manager.status().state == "disabled" + assert openai.OpenAI.request._self_wrapper._attachments is None + _chat_image(provider, png(seed=56)) + assert f01.state["requests"] == [] # not even a session bootstrap + finally: + tracer_provider.shutdown() + + +def test_auto_per_call_policy_and_ownership(f01, provider, auto_state, monkeypatch): + tracer_provider, exporter = _enabled_init(f01, monkeypatch, ["openai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5) + for context in (traceai.with_attachment_capture_policy(CapturePolicy(capture_content=False)), + traceai.with_attachment_capture_policy(CapturePolicy(muted=True)), + traceai.with_attachment_ownership()): + token = otel_context.attach(context) + try: + _chat_image(provider, png(seed=57)) + finally: + otel_context.detach(token) + flush(manager) + assert f01.payloads == [] + assert not any(refs(span) for span in exporter.get_finished_spans()) + image = png(seed=58) + token = otel_context.attach(traceai.with_attachment_capture_policy(CapturePolicy())) + try: + _chat_image(provider, image) + finally: + otel_context.detach(token) + flush(manager) + assert f01.payloads == [image] + finally: + manager.shutdown(5) + tracer_provider.shutdown() diff --git a/python/traceai/README.md b/python/traceai/README.md index 500b77469..c58997fd7 100644 --- a/python/traceai/README.md +++ b/python/traceai/README.md @@ -39,6 +39,30 @@ The helper does not install instrumentors or supply provider credentials. See [OpenAI setup](https://docs.futureagi.com/docs/integrations/traceai/openai) for a provider-specific example. +## Attachments (images, files, audio) + +Attachments are **off** by default; upgrading captures no new media. They need a +`fi-instrumentation-otel` release that includes `fi_instrumentation.attachments`. + +| Setting | Behavior | +|---|---| +| `attachment_config=AttachmentConfig(enabled=True)` or `FI_ATTACHMENTS_ENABLED=true` | The provider creates one attachment manager | +| `FI_ATTACHMENTS_ENABLED=false` | Kill switch: wins over code and denies every capture, even for a manager that is already running | +| `attachment_manager=` | Borrow your own manager instead. `traceai` never shuts it down and only adds the hide and PII flags of `trace_config` to it | +| `trace_config=TraceConfig(...)` | Hide flags and PII redaction also turn media capture off; nothing can relax them | + +`traceai` passes the same manager to every instrumentor it enables (as `attachments=`), +and `traceai.get_attachment_manager()` returns it. Use `with_attachment_capture_policy(policy)` +to add a `CapturePolicy` restriction for some calls, and `with_attachment_ownership()` to +mark calls whose media your own code captures, so instrumentors do not upload it again. +Both return an OpenTelemetry context to attach around those calls. + +`traceai.report()` shows the attachment state and, per integration, `media`: +`provider_adapter` when the installed instrumentor captures media (today `traceAI-openai`, +`traceAI-anthropic` and `traceAI-groq`), `session_binding` for voice integrations that +capture only through an explicit `bind_*` call, otherwise `unqualified`. It contains no +endpoint or credential and is not a playback claim. + ## License Apache-2.0. diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 4888b539d..2cf77de90 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -11,7 +11,9 @@ response = client.chat.completions.create(...) """ +import dataclasses import importlib +import importlib.util import threading from typing import Any, Dict, List, Optional @@ -29,15 +31,37 @@ # core, importing traceai and plain init() keep working; asking for attachments # raises a clear error instead. try: - from fi_instrumentation.attachments import AttachmentConfig + from fi_instrumentation.attachments import AttachmentConfig, DisabledAttachmentManager + from fi_instrumentation.attachments.policy import env_disabled as _env_disabled _ATTACHMENTS_SUPPORTED = True except ImportError: # pragma: no cover - exercised in a subprocess test AttachmentConfig = None + DisabledAttachmentManager = None _ATTACHMENTS_SUPPORTED = False -_CORE_TOO_OLD = ("attachment_config and trace_config need a fi-instrumentation-otel release that " - "includes fi_instrumentation.attachments; upgrade fi-instrumentation-otel") + def _env_disabled() -> bool: + return True + +# Context keys read by the shared provider-media helper (and any adapter) BY NAME, so +# setting them never needs the attachments package. They match +# fi_instrumentation.attachments.media_capture.CAPTURE_POLICY_KEY / OWNER_KEY. +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_capture_policy.v1" +_OWNER_KEY = "traceai.auto.attachment_owner.v1" + +_CORE_TOO_OLD = ("attachment_config, attachment_manager and trace_config need a fi-instrumentation-otel " + "release that includes fi_instrumentation.attachments; upgrade fi-instrumentation-otel") + +# Instrumentors that capture media through the shared manager they receive as +# ``attachments``: (module, descriptor). A descriptor is only present in an installed +# instrumentor version that has the adapter, so the report is version aware. +_MEDIA_ADAPTERS = { + "traceai_openai": ("traceai_openai", "openai_attachment_capabilities"), + "traceai_anthropic": ("traceai_anthropic", "anthropic_attachment_capabilities"), + "traceai_groq": ("traceai_groq._audio", "groq_attachment_capabilities"), +} +# Voice integrations capture only through an explicit session binding (bind_* below). +_SESSION_BINDINGS = {"traceai_pipecat": "bind_audio_processor", "traceai_livekit": "bind_session"} # Registry of available instrumentors _INSTRUMENTOR_REGISTRY: Dict[str, str] = { @@ -97,6 +121,9 @@ _init_lock = threading.RLock() _instrumented = set() _adapter_status = {} +# Whether the application lent the provider's attachment manager (auto never shuts a +# borrowed manager down). +_attachment_manager_borrowed = False def init( @@ -112,6 +139,7 @@ def init( set_global_tracer_provider: bool = True, attachment_config: Optional["AttachmentConfig"] = None, trace_config: Optional[TraceConfig] = None, + attachment_manager: Any = None, ) -> TracerProvider: """ Initialize TraceAI with a simple one-liner. @@ -129,6 +157,15 @@ def init( batch: Use batch span processor instead of simple (better for production) verbose: Print configuration details set_global_tracer_provider: Set as global OpenTelemetry tracer provider + attachment_config: Attachment options for the manager the provider creates. + Attachments are OFF unless enabled here or with FI_ATTACHMENTS_ENABLED=true. + trace_config: Hide and PII flags; they also restrict attachment capture. + attachment_manager: Borrow the application's AttachmentManager instead. Auto + never shuts it down and only adds trace_config's hide/PII restrictions to it. + Cannot be combined with attachment_config. + + Every instrumentor auto enables receives the same manager (``attachments=``); + ``get_attachment_manager()`` returns it and ``report()`` describes it. Returns: TracerProvider: The configured tracer provider @@ -161,7 +198,7 @@ def init( verbose=False, ) """ - global _tracer_provider, _init_options + global _tracer_provider, _init_options, _attachment_manager_borrowed # Convert string enums to enum types project_type_enum = ( @@ -171,13 +208,21 @@ def init( semantic_convention_enum = SemanticConvention(semantic_convention.lower()) transport_enum = Transport(transport.lower()) - if not _ATTACHMENTS_SUPPORTED and (attachment_config is not None or trace_config is not None): + if not _ATTACHMENTS_SUPPORTED and (attachment_config is not None or trace_config is not None + or attachment_manager is not None): raise RuntimeError(_CORE_TOO_OLD) + if attachment_manager is not None and attachment_config is not None: + raise ValueError("pass either attachment_config or attachment_manager, not both") # An older core's register() accepts neither keyword: pass them only when supported. attachment_kwargs = (dict(attachment_config=attachment_config, trace_config=trace_config) if _ATTACHMENTS_SUPPORTED else {}) + if attachment_manager is not None: + # The provider must not create a second manager (FI_ATTACHMENTS_ENABLED=true + # would otherwise start one with its own threads). + attachment_kwargs["attachment_config"] = AttachmentConfig(enabled=False) options = (project_name, project_type, project_version_name, semantic_convention, - transport, batch, set_global_tracer_provider, attachment_config, trace_config) + transport, batch, set_global_tracer_provider, attachment_config, trace_config, + id(attachment_manager) if attachment_manager is not None else None) with _init_lock: if _tracer_provider is not None and _init_options != options: raise ValueError("traceai is already initialized with different configuration") @@ -195,8 +240,14 @@ def init( semantic_convention=semantic_convention_enum, **attachment_kwargs, ) + if attachment_manager is not None: + _restrict_borrowed(attachment_manager, getattr(provider, "trace_config", None)) + # The provider holds the ONE manager, so get_attachment_manager(), the + # instrumentors and the bind_* helpers all use the borrowed one. + provider.attachment_manager = attachment_manager _tracer_provider = provider _init_options = options + _attachment_manager_borrowed = attachment_manager is not None _instrumented.clear() _adapter_status.clear() @@ -245,8 +296,14 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: provider = _tracer_provider kwargs = dict(tracer_provider=provider, config=getattr(provider, "trace_config", None)) if _ATTACHMENTS_SUPPORTED: - manager = getattr(provider, "attachment_manager", None) - kwargs.update(attachment_manager=manager, attachment_policy=getattr(manager, "policy", None)) + manager = get_attachment_manager() + if _capturing(manager): + # The ONE shared manager, as ``attachments`` (``attachment_manager`` is the + # alias the Python instrumentors also accept). + kwargs.update(attachments=manager, attachment_manager=manager) + else: + # Attachments OFF: exactly the keyword arguments auto passed before. + kwargs.update(attachment_manager=manager, attachment_policy=getattr(manager, "policy", None)) instrumentor.instrument(**kwargs) _instrumented.add(instrumentor_module) _adapter_status[lib_name_lower] = "registered" @@ -273,6 +330,73 @@ def get_tracer_provider() -> Optional[TracerProvider]: return _tracer_provider +def get_attachment_manager(): + """The one attachment manager auto passes to instrumentors, or None before init(). + + It is the borrowed ``attachment_manager``, else the one the provider created + (the disabled null object while attachments are OFF). Call ``ready()`` on it + to wait for the attachment session at startup. + """ + return getattr(_tracer_provider, "attachment_manager", None) + + +def _capturing(manager) -> bool: + """A manager that can admit media now: enabled, and not killed by FI_ATTACHMENTS_ENABLED.""" + return (manager is not None and DisabledAttachmentManager is not None + and not isinstance(manager, DisabledAttachmentManager) + and callable(getattr(manager, "preflight", None)) and not _env_disabled()) + + +_RESTRICTIONS = ("hide_inputs", "hide_outputs", "hide_input_messages", "hide_output_messages", + "hide_input_images", "hide_input_text", "hide_output_text", "pii_redaction") + + +def _restrict_borrowed(manager, trace_config) -> None: + """Add auto's hide/PII flags to a borrowed manager's policy. Never removes one.""" + policy = getattr(manager, "policy", None) + current = getattr(policy, "trace_config", None) + if policy is None or current is None or trace_config is None: + return + added = {name: True for name in _RESTRICTIONS + if getattr(trace_config, name, False) and not getattr(current, name, False)} + if getattr(trace_config, "hide_error_details", False) and not getattr(current, "hide_error_details", False): + added["hide_error_details"] = True + if added: + # TraceConfig is frozen: replace it whole; every flag already set stays set. + policy.trace_config = dataclasses.replace(current, **added) + + +def with_attachment_capture_policy(policy, context=None): + """Return a context whose calls capture media under ``policy`` as well. + + ``policy`` is a ``fi_instrumentation.attachments.CapturePolicy``. It can only + restrict: it is intersected with any policy already in the context and with every + other layer (server, credential, hide/PII flags, modality). Anything that is not a + ``CapturePolicy`` denies capture. Attach the result with + ``opentelemetry.context.attach`` around the calls it covers. + """ + from opentelemetry import context as otel_context + + try: + from fi_instrumentation.attachments.media_capture import merge_capture_policies + except ImportError: + merged = policy + else: + merged = merge_capture_policies(otel_context.get_value(_CAPTURE_POLICY_KEY, context), policy) + return otel_context.set_value(_CAPTURE_POLICY_KEY, merged, context) + + +def with_attachment_ownership(context=None): + """Return a context marking its calls' media as captured by your own adapter. + + Instrumentors called inside it capture no media for those calls, so the same + media is not uploaded twice. Mark only the downstream call, not your own capture. + """ + from opentelemetry import context as otel_context + + return otel_context.set_value(_OWNER_KEY, True, context) + + def instrument(*libraries: str, verbose: bool = True) -> None: """ Instrument additional libraries after init(). @@ -290,13 +414,110 @@ def instrument(*libraries: str, verbose: bool = True) -> None: _instrument_library(lib_name, verbose=verbose) +_AUTO_SOURCES = ( + "inline base64 and data-URI images, PDFs and files in OpenAI chat and Responses requests, " + "including computer_call_output screenshots (traceai_openai)", + "OpenAI images.generate b64_json and completed Responses image_generation_call output (traceai_openai)", + "OpenAI and Groq speech the SDK buffered, and OpenAI speech the application consumed", + "OpenAI and Groq transcription/translation audio the SDK holds as bytes", + "Anthropic Messages base64 images and PDFs, and computer-use tool_result screenshots (traceai_anthropic)", + "Pipecat and LiveKit recordings, only through an explicit bind_* call", +) +_AUTO_UNSUPPORTED = ( + "integrations whose media row is 'unqualified'", + "remote URLs and provider file ids (never fetched)", + "open file objects and file paths the SDK does not read itself", + "partial or streamed images, OpenAI images.edit and variations", + "Anthropic beta.messages and generated output media", + "HTTP wire capture (the Python auto module has no wire layer)", + "new recordings without an explicit voice binding", + "product playback qualification", +) + + +def _attachment_report() -> dict: + manager = get_attachment_manager() + if not _ATTACHMENTS_SUPPORTED: + state = "unsupported_version" + elif manager is None: + state = "not_initialized" + elif _env_disabled(): + # The kill switch denies at every admission even for an enabled manager, + # whose own status() does not consult it. + state = "disabled" + else: + try: + status = manager.status() + state = status.state + except Exception: + status, state = None, "unavailable" + report = {"state": state, "transport": "shared_manager", "borrowed": _attachment_manager_borrowed, + "sources": list(_AUTO_SOURCES), "unsupported": list(_AUTO_UNSUPPORTED)} + if state not in ("unsupported_version", "not_initialized", "disabled", "unavailable"): + report["status"] = {name: getattr(status, name) for name in + ("queued", "active", "tickets", "streams", "recordings", "owned_bytes", "dropped") + if isinstance(getattr(status, name, None), int)} + return report + + +def _installed(module: str) -> bool: + try: + return importlib.util.find_spec(module) is not None + except (ImportError, ValueError): + return False + + +def _media_support(module: str) -> Dict[str, Any]: + """Whether the INSTALLED instrumentor version captures media; imports, never instruments.""" + if not _ATTACHMENTS_SUPPORTED or not _installed(module): + return {"media": "unqualified"} + try: + if module in _MEDIA_ADAPTERS: + path, name = _MEDIA_ADAPTERS[module] + descriptor = getattr(importlib.import_module(path), name) + return {"media": "provider_adapter", "media_operations": list(descriptor["operations"])} + if module in _SESSION_BINDINGS and callable(getattr(importlib.import_module(module), + _SESSION_BINDINGS[module], None)): + return {"media": "session_binding"} + except Exception: + # An older instrumentor or core: no descriptor means no media capture. + pass + return {"media": "unqualified"} + + +def report() -> dict: + """Content-free setup report: attachment state and per-integration media support. + + ``integrations[].media`` is ``provider_adapter`` when the installed instrumentor + captures media through the shared manager, ``session_binding`` for voice + integrations that capture only through an explicit ``bind_*`` call, otherwise + ``unqualified``. It describes capability, not that any media was captured, and + never a playback claim. No endpoint, header or credential is included. + """ + names = {} + for name, module in _INSTRUMENTOR_REGISTRY.items(): + names.setdefault(module, name) + integrations = [] + for module, name in names.items(): + aliases = [alias for alias, target in _INSTRUMENTOR_REGISTRY.items() if target == module] + status = next((_adapter_status[a] for a in aliases if a in _adapter_status), None) + if status is None: + status = "not_enabled" if _installed(module) else "not_installed" + integrations.append({"name": name, "package": module, "status": status, **_media_support(module)}) + return {"initialized": _tracer_provider is not None, "attachments": _attachment_report(), + "integrations": integrations} + + def attachment_status() -> dict: """Content-free setup status, never an integrated recording-ready claim.""" if not _ATTACHMENTS_SUPPORTED: return {"runtime": "unsupported_version", "adapters": dict(_adapter_status)} - manager = getattr(_tracer_provider, "attachment_manager", None) - return {"runtime": manager.status().state if manager is not None else "uninitialized", - "adapters": dict(_adapter_status)} + manager = get_attachment_manager() + if manager is None: + runtime = "uninitialized" + else: + runtime = "disabled" if _env_disabled() else manager.status().state + return {"runtime": runtime, "adapters": dict(_adapter_status)} def _bind(library: str, entry_point: str, *args, **kwargs): @@ -340,6 +561,10 @@ def bind_livekit_session(session, job_context, *, owned_recording_files=False, "init", "AttachmentConfig", "attachment_status", + "get_attachment_manager", + "report", + "with_attachment_capture_policy", + "with_attachment_ownership", "bind_pipecat_audio_processor", "bind_livekit_session", "instrument", diff --git a/python/traceai/tests/conftest.py b/python/traceai/tests/conftest.py index b53fd44e1..93d76b235 100644 --- a/python/traceai/tests/conftest.py +++ b/python/traceai/tests/conftest.py @@ -9,3 +9,4 @@ def isolated_auto_state(monkeypatch): monkeypatch.setattr(traceai, "_init_options", None) monkeypatch.setattr(traceai, "_instrumented", set()) monkeypatch.setattr(traceai, "_adapter_status", {}) + monkeypatch.setattr(traceai, "_attachment_manager_borrowed", False) diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index b4e18e306..c4fc56195 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -94,7 +94,19 @@ def old_register(*, project_name=None, project_type=None, project_version_name=N assert calls == ["synthetic"], calls assert traceai.bind_livekit_session(object(), object()) is None assert traceai.attachment_status()["adapters"]["livekit"] == "unsupported_version" +assert traceai.get_attachment_manager() is None +assert traceai.report()["attachments"]["state"] == "unsupported_version" +assert all(row["media"] == "unqualified" for row in traceai.report()["integrations"]) +from opentelemetry import context as otel_context +marked = traceai.with_attachment_ownership() +assert otel_context.get_value("traceai.auto.attachment_owner.v1", marked) is True traceai._tracer_provider = None +try: + traceai.init(project_name="other", verbose=False, attachment_manager=object()) +except RuntimeError as error: + assert "fi-instrumentation-otel" in str(error) +else: + raise AssertionError("attachment_manager accepted on a core without attachments") try: traceai.init(project_name="other", verbose=False, trace_config=object()) except RuntimeError as error: @@ -117,3 +129,157 @@ def test_import_and_init_on_core_without_attachments(): timeout=120, env=env) assert result.returncode == 0, result.stderr[-2000:] assert "OLD_CORE_OK" in result.stdout + + +# ----------------------------------------------------------- shared manager (A01) +import dataclasses # noqa: E402 + +import pytest # noqa: E402 +from fi_instrumentation.attachments import AttachmentManager, CapturePolicy, DisabledAttachmentManager # noqa: E402 +from fi_instrumentation.attachments import media_capture # noqa: E402 +from opentelemetry import context as otel_context # noqa: E402 + + +def _enabled_like_manager(): + """Stands in for an enabled manager: not the null object, and it can preflight.""" + return SimpleNamespace(policy=object(), preflight=Mock(), status=lambda: SimpleNamespace(state="ready")) + + +def _install(monkeypatch, manager, config=None): + provider = SimpleNamespace(attachment_manager=manager, trace_config=config or TraceConfig()) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + constructor = Mock() + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(OpenAIInstrumentor=constructor))) + return provider, constructor + + +def test_enabled_manager_reaches_every_instrumentor_as_attachments(monkeypatch): + manager = _enabled_like_manager() + provider, constructor = _install(monkeypatch, manager) + assert traceai.get_attachment_manager() is None + traceai.init(project_name="synthetic", auto_instrument=["openai"], verbose=False) + assert traceai.get_attachment_manager() is manager + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachments=manager, attachment_manager=manager) + + +def test_kill_switch_keeps_the_off_path_and_reports_disabled(monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = _enabled_like_manager() + provider, constructor = _install(monkeypatch, manager) + traceai.init(project_name="synthetic", auto_instrument=["openai"], verbose=False) + # Exactly the keyword arguments auto passed before this change, with attachments OFF. + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachment_manager=manager, + attachment_policy=manager.policy) + assert traceai.report()["attachments"]["state"] == "disabled" + assert "status" not in traceai.report()["attachments"] + assert traceai.attachment_status()["runtime"] == "disabled" + + +def test_borrowed_manager_gains_only_restrictions(monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "true") + app_config = TraceConfig(hide_inputs=True, pii_redaction=False) + borrowed = AttachmentManager(config=AttachmentConfig(enabled=True), trace_config=app_config) # no scope: no thread + register = Mock(return_value=SimpleNamespace(attachment_manager=DisabledAttachmentManager(), + trace_config=TraceConfig(hide_inputs=False, pii_redaction=True))) + monkeypatch.setattr(traceai, "register", register) + try: + traceai.init(project_name="synthetic", verbose=False, attachment_manager=borrowed, + trace_config=TraceConfig(hide_inputs=False, pii_redaction=True)) + # The provider does not build a second (env-enabled) manager. + assert register.call_args.kwargs["attachment_config"].enabled is False + assert traceai.get_attachment_manager() is borrowed + assert traceai.report()["attachments"]["borrowed"] is True + restricted = borrowed.policy.trace_config + assert restricted.pii_redaction is True # auto's restriction reached the borrowed manager + assert restricted.hide_inputs is True # the application's own flag survived auto's False + assert app_config.pii_redaction is False # the application's object is not mutated + finally: + borrowed.shutdown(0) + + +def test_borrowed_manager_and_attachment_config_are_exclusive(monkeypatch): + monkeypatch.setattr(traceai, "register", Mock(side_effect=AssertionError("registered"))) + with pytest.raises(ValueError): + traceai.init(project_name="synthetic", verbose=False, attachment_manager=_enabled_like_manager(), + attachment_config=AttachmentConfig(enabled=False)) + + +def test_context_keys_match_the_shared_helper(): + assert traceai._CAPTURE_POLICY_KEY == media_capture.CAPTURE_POLICY_KEY + assert traceai._OWNER_KEY == media_capture.OWNER_KEY + + +def test_capture_policy_helper_only_restricts(): + denied = traceai.with_attachment_capture_policy(CapturePolicy(capture_content=False, muted=True)) + # A later, permissive policy in the same context cannot relax the earlier denial. + relaxed = traceai.with_attachment_capture_policy(CapturePolicy(), denied) + token = otel_context.attach(relaxed) + try: + folded = media_capture.local_capture_policy(TraceConfig(), direction="input") + assert folded.capture_content is False and folded.muted is True + finally: + otel_context.detach(token) + assert media_capture.local_capture_policy(TraceConfig(), direction="input").capture_content is True + token = otel_context.attach(traceai.with_attachment_capture_policy("not a policy")) + try: + folded = media_capture.local_capture_policy(TraceConfig(), direction="output") + assert folded.capture_content is False and folded.requested is False + finally: + otel_context.detach(token) + + +def test_ownership_helper_turns_the_shared_helper_off(): + from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider + + manager = AttachmentManager(config=AttachmentConfig(enabled=True)) + sdk = SdkTracerProvider() + try: + with sdk.get_tracer("synthetic").start_as_current_span("call") as span: + assert media_capture.Observation(manager, span, "openai").usable is True + token = otel_context.attach(traceai.with_attachment_ownership()) + try: + assert media_capture.local_capture_policy(None, direction="input").requested is False + assert media_capture.Observation(manager, span, "openai").usable is False + finally: + otel_context.detach(token) + assert media_capture.local_capture_policy(None, direction="input").requested is True + assert media_capture.Observation(manager, span, "openai").usable is True + finally: + manager.shutdown(0) + sdk.shutdown() + + +def test_merge_is_deny_wins_for_every_fact(): + for item in dataclasses.fields(CapturePolicy): + grant = item.name in ("capture_content", "record_inputs", "record_outputs", "requested") + tight = CapturePolicy(**{item.name: not grant}) + assert getattr(media_capture.merge_capture_policies(CapturePolicy(), tight), item.name) is (not grant) + assert getattr(media_capture.merge_capture_policies(tight, CapturePolicy()), item.name) is (not grant) + + +def test_report_media_rows_are_version_aware_and_content_free(monkeypatch): + descriptor = {"operations": ("messages.create.input", "tool_result.input")} + modules = {"traceai_anthropic": SimpleNamespace(anthropic_attachment_capabilities=descriptor), + "traceai_openai": SimpleNamespace(), # an older instrumentor without the adapter + "traceai_livekit": SimpleNamespace(bind_session=lambda *a, **k: None)} + + def import_module(name): + if name in modules: + return modules[name] + raise ImportError(name) + + monkeypatch.setattr(traceai, "_installed", lambda module: module in modules) + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows["anthropic"]["media"] == "provider_adapter" + assert rows["anthropic"]["media_operations"] == ["messages.create.input", "tool_result.input"] + assert rows["openai"]["media"] == "unqualified" + assert rows["livekit"]["media"] == "session_binding" + assert rows["groq"] == {"name": "groq", "package": "traceai_groq", "status": "not_installed", + "media": "unqualified"} + assert rows["anthropic"]["status"] == "not_enabled" + assert "mistral" not in rows # aliases share one row + assert traceai.report()["attachments"]["state"] == "not_initialized" From dc69e510fc87a4604d48d842ccfdb95e30c2f883 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:35:05 -0800 Subject: [PATCH 02/16] test(python): expect the F01 PII denial reason for a tightened borrowed manager Co-Authored-By: Claude Opus 5.5 --- python/tests/attachments_providers/test_auto_init.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/tests/attachments_providers/test_auto_init.py b/python/tests/attachments_providers/test_auto_init.py index 8fbebe6fb..cd70f8880 100644 --- a/python/tests/attachments_providers/test_auto_init.py +++ b/python/tests/attachments_providers/test_auto_init.py @@ -160,10 +160,11 @@ def test_auto_borrowed_manager_is_shared_and_tightened(f01, provider, auto_state trace_config=TraceConfig(pii_redaction=True)) assert traceai.get_attachment_manager() is borrowed is tracer_provider.attachment_manager assert openai.OpenAI.request._self_wrapper._attachments is borrowed - # Auto's PII flag now restricts the application's manager itself, for every caller. + # Auto's PII flag now restricts the application's manager itself, for every caller: + # raw media under PII redaction needs a sanitizer, and none exists. with sdk.get_tracer("synthetic").start_as_current_span("probe") as span: assert borrowed.preflight(span, mime_type="image/png", direction="input", - max_bytes=16).reason == "policy_denied" + max_bytes=16).reason == "redaction_unavailable" _chat_image(provider, png(seed=53)) flush(borrowed) assert f01.payloads == [] From 7c6ef0d9812c8628ea70d77b6edb3763211a67be Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:05:47 -0800 Subject: [PATCH 03/16] feat(python): capture MCP, agent SDK, smolagents and Firecrawl tool media Shared helper fi_instrumentation.attachments.tool_media on the public manager API and media_capture: tool media is tool output and model input, so both directional hide flags and PII redaction deny; preflight before any read or decode; bounded budgets; per-manager dedup; owner-key honoured. - traceai_mcp: wrap_mcp_client() / capture_mcp_result() for typed image, audio and inline resource blobs (never resource URIs or links). - traceai_mcp_use: FutureAGICallback captures typed tool blocks. - traceai_openai_agents: computer screenshots and typed tool images/files on the function span, projected to fi-attachment URIs. - traceai_claude_agent_sdk: tool_result image/PDF blocks before truncation, on the tool span (live or its exact ended context). - traceai_smolagents: in-memory model-input and tool images. - traceai_firecrawl: inline data-URI screenshots and media only. Co-Authored-By: Claude Opus 5.5 --- .../attachments/tool_media.py | 243 ++++++++++++++++++ .../traceai_claude_agent_sdk/_attachments.py | 143 +++++++++++ .../_client_wrapper.py | 35 ++- .../traceai_claude_agent_sdk/_hooks.py | 10 +- .../traceai_claude_agent_sdk/_instrumentor.py | 26 ++ .../firecrawl/traceai_firecrawl/__init__.py | 28 +- .../traceai_firecrawl/_attachments.py | 84 ++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 12 +- .../mcp-use/traceai_mcp_use/_attachments.py | 68 +++++ .../mcp-use/traceai_mcp_use/_callback.py | 31 +++ python/frameworks/mcp/traceai_mcp/__init__.py | 7 + .../mcp/traceai_mcp/_attachments.py | 199 ++++++++++++++ .../traceai_openai_agents/__init__.py | 30 ++- .../traceai_openai_agents/_attachments.py | 110 ++++++++ .../traceai_openai_agents/_processor.py | 46 +++- .../smolagents/traceai_smolagents/__init__.py | 26 +- .../traceai_smolagents/_attachments.py | 136 ++++++++++ .../traceai_smolagents/_wrappers.py | 37 ++- .../_fake_claude_cli.py | 97 +++++++ .../tests/attachments_mcp_agents/_harness.py | 44 ++++ .../_mcp_media_server.py | 116 +++++++++ .../tests/attachments_mcp_agents/conftest.py | 21 ++ .../test_claude_agent_sdk_attachments.py | 135 ++++++++++ .../test_firecrawl_attachments.py | 114 ++++++++ .../test_mcp_attachments.py | 225 ++++++++++++++++ .../test_mcp_use_attachments.py | 102 ++++++++ .../test_openai_agents_attachments.py | 213 +++++++++++++++ .../test_smolagents_attachments.py | 164 ++++++++++++ .../attachments_mcp_agents/test_tool_media.py | 108 ++++++++ 29 files changed, 2587 insertions(+), 23 deletions(-) create mode 100644 python/fi_instrumentation/attachments/tool_media.py create mode 100644 python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py create mode 100644 python/frameworks/firecrawl/traceai_firecrawl/_attachments.py create mode 100644 python/frameworks/mcp-use/traceai_mcp_use/_attachments.py create mode 100644 python/frameworks/mcp/traceai_mcp/_attachments.py create mode 100644 python/frameworks/openai-agents/traceai_openai_agents/_attachments.py create mode 100644 python/frameworks/smolagents/traceai_smolagents/_attachments.py create mode 100644 python/tests/attachments_mcp_agents/_fake_claude_cli.py create mode 100644 python/tests/attachments_mcp_agents/_harness.py create mode 100644 python/tests/attachments_mcp_agents/_mcp_media_server.py create mode 100644 python/tests/attachments_mcp_agents/conftest.py create mode 100644 python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py create mode 100644 python/tests/attachments_mcp_agents/test_firecrawl_attachments.py create mode 100644 python/tests/attachments_mcp_agents/test_mcp_attachments.py create mode 100644 python/tests/attachments_mcp_agents/test_mcp_use_attachments.py create mode 100644 python/tests/attachments_mcp_agents/test_openai_agents_attachments.py create mode 100644 python/tests/attachments_mcp_agents/test_smolagents_attachments.py create mode 100644 python/tests/attachments_mcp_agents/test_tool_media.py diff --git a/python/fi_instrumentation/attachments/tool_media.py b/python/fi_instrumentation/attachments/tool_media.py new file mode 100644 index 000000000..600c4c471 --- /dev/null +++ b/python/fi_instrumentation/attachments/tool_media.py @@ -0,0 +1,243 @@ +"""Shared tool-media helpers for MCP and agent adapters, built on ``media_capture``. + +MCP clients, ``mcp-use``, the OpenAI Agents SDK, the Claude Agent SDK, +smolagents and Firecrawl all observe media that a TOOL returned. These helpers +give them one set of rules (the TypeScript ``@traceai/mcp`` and +``@traceai/claude-agent-sdk`` semantics): + +- Tool media is tool output AND potential model input, so input and output + hide flags, ``hide_input_images`` for images and ``pii_redaction`` all deny + (deny wins; the core cannot relax a local restriction). +- Only typed inline values are read: base64 fields, data URIs, and in-memory + images. Nothing here fetches a URL, dereferences a resource URI or path, + opens a file, or reads a stream. A URL or file-backed value is skipped. +- Every value is preflighted before it is read or decoded. Each observation + has a bounded decode budget and part count; parts beyond them are skipped. +- A context marked by another adapter as the owner of this call's media + (``OWNER_KEY``, shared by name with ``traceai``) captures nothing here. + +Not part of ``fi_instrumentation.attachments.__all__``. Every function is +non-throwing and returns ``None``/empty on denial. +""" + +import dataclasses +import io +import sys +from typing import Any, Callable, Mapping, Optional + +from opentelemetry import context as otel_context + +from . import media_capture as mc +from .policy import CapturePolicy +from .types import MIME_MODALITIES, Bytes + +KiB = 1024 +MiB = 1024 * KiB +#: Decoded bytes one tool result may contribute (TypeScript MCP_MEDIA_DECODE_BUDGET). +TOOL_RESULT_BUDGET = 256 * KiB +#: Typed parts inspected per tool result. +MAX_TOOL_PARTS = 64 +#: In-memory images are PNG-encoded only up to this many pixels (no huge synchronous encodes). +MAX_IMAGE_PIXELS = 2048 * 2048 +OWNER_KEY = "traceai.auto.attachment_owner.v1" +URI_PREFIX = "fi-attachment:" +Occurrence = mc.Occurrence + + +def owned_elsewhere() -> bool: + """True when another adapter marked this call's media as its own (by context key name).""" + try: + return otel_context.get_value(OWNER_KEY) is not None + except Exception: + return True + + +def with_ownership(context=None): + """A context that tells downstream observers this call's media is captured already.""" + return otel_context.set_value(OWNER_KEY, True, context) + + +def uri(occurrence: Occurrence) -> str: + return URI_PREFIX + occurrence.occurrence_id + + +def tool_capture_policy(trace_config=None, capture: Optional[CapturePolicy] = None, *, image: bool = False, + error: bool = False) -> CapturePolicy: + """Deny-only policy for media a tool returned: input AND output restrictions both apply.""" + base = capture if isinstance(capture, CapturePolicy) else CapturePolicy() + seen_input = mc.local_capture_policy(trace_config, direction="input", image=image, error_detail=error, + base=base) + seen_output = mc.local_capture_policy(trace_config, direction="output", image=image, error_detail=error, + base=base) + return dataclasses.replace( + seen_input, + capture_content=seen_input.capture_content and seen_output.capture_content, + requested=seen_input.requested and seen_output.requested and not owned_elsewhere(), + contains_output=True, + model_input=True, + ) + + +def mime_kind(mime: Any) -> Optional[str]: + """The F01 modality of an exact MIME; None when the MIME is not in the F01 enum.""" + return MIME_MODALITIES.get(mime) if isinstance(mime, str) else None + + +def _pil_image(value: Any) -> bool: + module = sys.modules.get("PIL.Image") + image_type = getattr(module, "Image", None) if module is not None else None + return isinstance(image_type, type) and isinstance(value, image_type) + + +def model_input_policy(trace_config=None, capture: Optional[CapturePolicy] = None, *, + image: bool = False) -> CapturePolicy: + """Deny-only policy for media an agent framework hands to a model as input.""" + policy = mc.local_capture_policy(trace_config, direction="input", image=image, base=capture) + return dataclasses.replace(policy, requested=policy.requested and not owned_elsewhere()) + + +class ToolMedia: + """One adapter's observation of one span, with a decode budget and registry dedup. + + ``namespace`` keeps positions of different adapters apart in the shared + per-manager registry; a repeated observation of the same span and + position returns the first occurrence and uploads nothing. + ``direction`` is ``"output"`` for media a tool returned (tool policy) or + ``"input"`` for media a framework sends to a model (model-input policy). + """ + + def __init__(self, manager, span, namespace: str, *, trace_config=None, + capture: Optional[CapturePolicy] = None, budget: int = TOOL_RESULT_BUDGET, + max_parts: int = MAX_TOOL_PARTS, direction: str = "output"): + self.manager, self.span = manager, span + self.direction = "input" if direction == "input" else "output" + self._trace_config, self._capture = trace_config, capture + self.remaining = budget + self._parts = 0 + self._max_parts = max_parts + self._observation = mc.Observation(manager, span, namespace) + if owned_elsewhere(): + self._observation.key = None + + def _policy(self, image: bool, error: bool) -> CapturePolicy: + if self.direction == "input": + return model_input_policy(self._trace_config, self._capture, image=image) + return tool_capture_policy(self._trace_config, self._capture, image=image, error=error) + + @property + def usable(self) -> bool: + return self._observation.usable and self.remaining > 0 + + @property + def captured(self) -> Mapping[str, Occurrence]: + return dict(self._observation.captured) + + def _visit(self, position: str, capture: Callable[[], Optional[Occurrence]]) -> Optional[Occurrence]: + if not self.usable or self._parts >= self._max_parts: + return None + self._parts += 1 + self._observation.visit(position, capture) + return self._observation.captured.get(position) + + def base64(self, position: str, read: Callable[[], Any], mime: Any, *, purpose: str, + expect: Optional[str] = None, role: str = "tool", tool_call_id: Optional[str] = None, + message_index: Optional[int] = None, part_index: Optional[int] = None, + error: bool = False) -> Optional[Occurrence]: + """A raw base64 value with a declared MIME (MCP ImageContent/AudioContent/blob, Anthropic source). + + ``expect`` ("image", "audio", ...) rejects a MIME of another modality, so a + misleading MIME cannot relabel an image to escape ``hide_input_images``. + ``read`` is called at most once, and only after a preflight admitted the part. + """ + kind = mime_kind(mime) + if kind is None or (expect is not None and kind != expect): + return None + return self._inline(position, read, (mime,), True, purpose, kind == "image", role, tool_call_id, + message_index, part_index, error) + + def data_uri(self, position: str, read: Callable[[], Any], mimes, *, purpose: str, role: str = "tool", + tool_call_id: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, error: bool = False) -> Optional[Occurrence]: + """A ``data:;base64,`` string whose MIME must be one of ``mimes`` (first is preflighted).""" + mimes = tuple(mime for mime in mimes if mime_kind(mime)) + if not mimes: + return None + image = any(mime_kind(mime) == "image" for mime in mimes) + return self._inline(position, read, mimes, False, purpose, image, role, tool_call_id, message_index, + part_index, error) + + def _inline(self, position, read, mimes, raw, purpose, image, role, tool_call_id, message_index, part_index, + error) -> Optional[Occurrence]: + seen = {} + + def once(): + if "value" not in seen: + seen["value"] = read() + return seen["value"] + + def capture(): + policy = self._policy(image, error) + return mc.attach_inline(self.manager, self.span, once, mimes, raw=raw, direction=self.direction, + purpose=purpose, capture=policy, message_index=message_index, + part_index=part_index, role=role, tool_call_id=tool_call_id, + max_bytes=self.remaining) + + before = self._observation.captured.get(position) + occurrence = self._visit(position, capture) + if occurrence is not None and before is None and "value" in seen: + value = seen["value"] + if not raw and isinstance(value, str): + value = value[value.find(",", 0, 128) + 1:] + self.remaining -= mc.encoded_length(value, self.remaining) or self.remaining + return occurrence + + def image(self, position: str, image: Any, *, purpose: str, role: str = "tool", + tool_call_id: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, error: bool = False) -> Optional[Occurrence]: + """An in-memory ``PIL.Image.Image``, PNG-encoded only after a pixel-bounded preflight. + + A file-backed image that is not loaded yet is skipped: encoding it would + read the file. PIL is never imported here; a value is an image only when + the application already imported PIL. + """ + if not _pil_image(image): + return None + + def capture(): + try: + fp = getattr(image, "fp", None) + if fp is not None and not isinstance(fp, io.BytesIO): + return None + width, height = image.size + bands = len(image.getbands()) + if not (0 < width and 0 < height and width * height <= MAX_IMAGE_PIXELS): + return None + # PNG never exceeds the raw pixel bytes plus per-row filter bytes and chunk framing. + bound = min(width * height * max(bands, 1) + height + 4 * KiB, self.remaining) + policy = self._policy(True, error) + options = dict(mime_type="image/png", direction=self.direction, purpose=purpose, + capture_policy=policy) + ticket = self.manager.preflight(self.span, max_bytes=bound, **options) + if not ticket.admitted: + return None + except Exception: + return None + try: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + data = buffer.getvalue() + except Exception: + data = b"" + try: + if not data or len(data) > ticket.max_bytes: + ticket.release() + return None + handle = ticket.attach(Bytes(data), **mc._fields(message_index, part_index, role, tool_call_id)) + if handle.admission != "queued": + return None + self.remaining -= len(data) + return Occurrence(handle.occurrence_id, "image/png") + except Exception: + return None + + return self._visit(position, capture) diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py new file mode 100644 index 000000000..a7458fd87 --- /dev/null +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py @@ -0,0 +1,143 @@ +"""Tool-result media of a Claude Agent SDK conversation, through the shared attachment manager. + +The SDK parses each CLI ``user`` message into a ``UserMessage`` whose +``ToolResultBlock.content`` keeps the original Anthropic content blocks. The +base64 ``image`` and PDF ``document`` blocks there are uploaded BEFORE any text +flattening or truncation, at the tool's span (with ``tool_call_id``): + +- while the tool span is still open, on that span (and projected on it); +- after the PostToolUse hook ended it, on its exact span context (the + occurrence keeps the causal parent; the ended span is never mutated); +- for a result with no traced tool, on the conversation span. + +URL and file sources, paths and transcripts are never read. The SDK drops +top-level user image blocks during parsing, so prompt images are not +observable here (``source_unavailable``). Model calls run inside the Claude +Code process, so no in-process provider adapter sees the same bytes. +""" + +from collections import OrderedDict +from types import MappingProxyType +from typing import Any, Mapping, Optional + +__all__ = ("capture_tool_results", "claude_agent_attachment_capabilities", "remember_ended_tool", + "media_safe") + +claude_agent_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tool_result.output",), + "sources": ("inline_base64",), + "content_types": ("image", "document"), + "unsupported": ("url_source", "file_source", "prompt_images", "transcript", "local_path"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_MEDIA_TYPES = ("image", "document", "audio") +_MAX_ENDED = 256 +_ended: "OrderedDict[str, Any]" = OrderedDict() + + +def remember_ended_tool(tool_use_id: Any, span: Any) -> None: + """Keep the exact span context of a tool span the PostToolUse hook ended (bounded).""" + try: + if not isinstance(tool_use_id, str): + return + _ended[tool_use_id] = span.get_span_context() + _ended.move_to_end(tool_use_id) + while len(_ended) > _MAX_ENDED: + _ended.popitem(last=False) + except Exception: + return + + +def forget_ended_tools() -> None: + _ended.clear() + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def capture_tool_results(manager, message: Any, conversation_span: Any, active_spans: Mapping[str, Any], *, + message_index: int, trace_config=None, capture=None) -> int: + """Capture the media blocks of every ``ToolResultBlock`` in one ``UserMessage``. Never raises. + + ``active_spans`` maps a tool_use_id to its still-open span (the hooks' table). + Returns the number of admitted blocks. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return 0 + admitted = 0 + try: + if not active(manager): + return 0 + content = _get(message, "content") + if not isinstance(content, list): + return 0 + for block in content[:MAX_TOOL_PARTS]: + if type(block).__name__ != "ToolResultBlock": + continue + blocks = _get(block, "content") + if not isinstance(blocks, list): + continue + tool_use_id = _get(block, "tool_use_id") + open_entry = active_spans.get(tool_use_id) if isinstance(tool_use_id, str) else None + if open_entry is not None: + target = open_entry[0] + elif isinstance(tool_use_id, str) and tool_use_id in _ended: + target = _ended[tool_use_id] + else: + target = conversation_span + error = _get(block, "is_error") is True + media = ToolMedia(manager, target, "claude_agent_sdk", trace_config=trace_config, capture=capture) + for part_index, part in enumerate(blocks[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(part, "type") + if kind not in ("image", "document"): + continue + source = _get(part, "source") + if _get(source, "type") != "base64": + continue # url / file sources are never fetched + mime = _get(source, "media_type") + expect = "image" if kind == "image" else "document" + found = media.base64("tool_result/%s/%d" % (tool_use_id, part_index), + lambda s=source: _get(s, "data"), mime, purpose="tool_result", + expect=expect, role="tool", tool_call_id=tool_use_id, + message_index=min(message_index, 65535), part_index=part_index, + error=error) + admitted += found is not None + except Exception: + return admitted + return admitted + + +def media_safe(value: Any, depth: int = 0) -> Any: + """A telemetry copy without inline media bytes: base64 fields of media blocks become ``"[omitted]"``. + + Used before tool responses are JSON-serialized and truncated, so bytes never + reach span attributes as text. Non-media values are returned unchanged. + """ + if depth > 6: + return value + if isinstance(value, list): + return [media_safe(item, depth + 1) for item in value] + if not isinstance(value, Mapping): + return value + media = value.get("type") in _MEDIA_TYPES + result = {} + for key, item in value.items(): + if media and key in ("data", "base64", "blob") and isinstance(item, str): + result[key] = "[omitted]" + elif media and key in ("source", "file") and isinstance(item, Mapping): + result[key] = {k: ("[omitted]" if k in ("data", "base64") and isinstance(v, str) else v) + for k, v in item.items()} + else: + result[key] = media_safe(item, depth + 1) + return result diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py index 5cdb1cc89..ed40afadf 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py @@ -14,8 +14,10 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from ._attachments import capture_tool_results, forget_ended_tools from ._attributes import ClaudeAgentAttributes, ClaudeAgentSpanKind from ._hooks import ( + _active_tool_spans, create_pre_tool_use_hook, create_subagent_start_hook, create_post_tool_use_hook, @@ -213,13 +215,20 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: def wrap_claude_sdk_client( - original_class: Any, tracer: trace_api.Tracer + original_class: Any, + tracer: trace_api.Tracer, + attachments: Any = None, + trace_config: Any = None, + attachment_capture: Any = None, ) -> Any: """Wrap ClaudeSDKClient with tracing. Args: original_class: The original ClaudeSDKClient class tracer: OpenTelemetry tracer + attachments: the shared attachment manager for tool-result media (None: off) + trace_config: TraceConfig whose hide flags and PII redaction deny media capture + attachment_capture: CapturePolicy that can only add restrictions Returns: Wrapped class with tracing @@ -277,10 +286,12 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: collected_messages: List[Dict[str, Any]] = [] result_usage: Optional[Dict[str, Any]] = None + message_index = 0 try: async for msg in messages: msg_type = type(msg).__name__ + message_index += 1 if msg_type == "AssistantMessage": # Start new turn @@ -295,6 +306,18 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: }) elif msg_type == "UserMessage": + # Tool-result media first, from the original typed blocks, + # before any text flattening or truncation. + if attachments is not None: + capture_tool_results( + attachments, + msg, + conversation_span, + _active_tool_spans, + message_index=message_index, + trace_config=trace_config, + capture=attachment_capture, + ) # End current turn, mark next start turn_tracker.end_current_turn() turn_tracker.mark_next_start() @@ -350,6 +373,7 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: turn_tracker.end_current_turn() clear_parent_span() clear_active_tool_spans() + forget_ended_tools() def _build_span_attributes(self) -> Dict[str, Any]: """Build span attributes from options and prompt.""" @@ -414,6 +438,15 @@ def _extract_content(self, msg: Any) -> Optional[str]: parts.append(block.text) elif type(block).__name__ == "ToolResultBlock": result = getattr(block, "content", "") + if isinstance(result, list): + # Only text blocks: media bytes never become span text. + result = " ".join( + item.get("text", "") + for item in result + if isinstance(item, dict) and isinstance(item.get("text"), str) + ) + elif not isinstance(result, str): + result = "" parts.append(f"[Tool Result: {result[:100]}...]") return " ".join(parts) if parts else None diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py index fa2c876b1..4d2f29285 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py @@ -13,6 +13,7 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from ._attachments import media_safe, remember_ended_tool from ._attributes import ( ClaudeAgentAttributes, ClaudeAgentSpanKind, @@ -304,10 +305,11 @@ async def post_tool_use_hook( span, start_time, _ = span_info duration_ms = (time.time() - start_time) * 1000 - # Set output and duration + # Set output and duration. Inline media bytes never go through text serialization + # or truncation; the attachment path uploads them from the typed tool result. span.set_attribute( ClaudeAgentAttributes.TOOL_OUTPUT, - safe_json_serialize(tool_response), + safe_json_serialize(media_safe(tool_response)), ) span.set_attribute(ClaudeAgentAttributes.TOOL_DURATION_MS, duration_ms) @@ -341,8 +343,10 @@ async def post_tool_use_hook( }, ) - # End the span + # End the span. Its exact context stays available (bounded) so the tool's + # result media, which the SDK streams after this hook, keep this causal parent. span.end() + remember_ended_tool(tool_use_id, span) # Clean up subagent span if this launched a subagent if tool_name in SUBAGENT_TOOLS: diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py index 8322da2b7..f2154cbfa 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py @@ -20,6 +20,23 @@ _instruments = ("claude-agent-sdk >= 0.1.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class ClaudeAgentInstrumentor(BaseInstrumentor): """OpenTelemetry Instrumentor for Claude Agent SDK. @@ -97,6 +114,12 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider, ) + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + self._attachments = _attachment_manager(kwargs) + self._trace_config = kwargs.get("config") + self._attachment_capture = kwargs.get("attachment_capture") + # Try to import and patch Claude Agent SDK try: self._patch_claude_agent_sdk() @@ -129,6 +152,9 @@ def _patch_claude_agent_sdk(self) -> None: wrapped_class = wrap_claude_sdk_client( self._original_client_class, self._tracer, + attachments=getattr(self, "_attachments", None), + trace_config=getattr(self, "_trace_config", None), + attachment_capture=getattr(self, "_attachment_capture", None), ) # Replace in module diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py index 389ff8edf..748c1a1b4 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -56,8 +56,30 @@ } +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class FirecrawlInstrumentor(BaseInstrumentor): - """Instrument Firecrawl v2 scraping, extraction, agent and browser operations.""" + """Instrument Firecrawl v2 scraping, extraction, agent and browser operations. + + With an enabled shared attachment manager (``attachments=``), inline + ``data:`` screenshots and media of returned Documents are uploaded. + Hosted URLs are never fetched. + """ __slots__ = ("_original_methods",) @@ -73,7 +95,9 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() tracer = FITracer(trace_api.get_tracer(__name__, __version__, tracer_provider), config=config) self._original_methods: Dict[Tuple[str, str, str], Any] = {} - self._state = _State() + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + self._state = _State(_attachment_manager(kwargs), kwargs.get("attachment_capture")) for module_name, class_name, is_async in ( (_SYNC_MODULE, _SYNC_CLASS, False), diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py b/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py new file mode 100644 index 000000000..8925ffd3a --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py @@ -0,0 +1,84 @@ +"""Firecrawl returned screenshots and media, through the shared attachment manager. + +A firecrawl-py v2 ``Document`` carries ``screenshot``, ``audio`` and ``video`` +strings, and browser ``actions`` may carry ``screenshots``. Only values that +are inline ``data:;base64,`` URIs are uploaded, at the operation span, +as tool results. Firecrawl Cloud returns hosted URLs for these: they are never +fetched (the SDK core has no approved remote retrieval yet), so they are +``source_unavailable``. Nothing here starts a crawl, a browser or a request. +""" + +from types import MappingProxyType +from typing import Any, Mapping + +__all__ = ("capture_firecrawl_media", "firecrawl_attachment_capabilities") + +#: Operations whose results carry Documents. +MEDIA_METHODS = frozenset(( + "scrape", "crawl", "batch_scrape", "wait_batch_scrape", "get_crawl_status", "get_batch_scrape_status", +)) +MAX_DOCUMENTS = 16 +MAX_RESULT_BYTES = 8 * 1024 * 1024 +_IMAGE = ("image/png", "image/jpeg", "image/webp") +_AUDIO = ("audio/mpeg", "audio/wav", "audio/ogg", "audio/webm", "audio/mp4", "audio/flac") +_VIDEO = ("video/mp4", "video/webm") +_EMPTY: Mapping[str, Any] = MappingProxyType({}) + +firecrawl_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": tuple(sorted(MEDIA_METHODS)), + "sources": ("inline_data_uri",), + "fields": ("screenshot", "actions.screenshots", "audio", "video"), + "unsupported": ("hosted_url", "remote_url", "images_links", "pdf_url"), + "max_documents": MAX_DOCUMENTS, + "max_decoded_bytes_per_result": MAX_RESULT_BYTES, +}) + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def _documents(method: str, result: Any) -> list: + if method == "scrape": + return [result] + data = _get(result, "data") + return list(data[:MAX_DOCUMENTS]) if isinstance(data, (list, tuple)) else [] + + +def capture_firecrawl_media(manager, span, method: str, result: Any, *, trace_config=None, + capture=None) -> Mapping[str, Any]: + """Capture inline media of one operation's returned Documents. Never raises.""" + if method not in MEDIA_METHODS: + return _EMPTY + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import ToolMedia + except ImportError: + return _EMPTY + try: + if not active(manager): + return _EMPTY + media = ToolMedia(manager, span, "firecrawl", trace_config=trace_config, capture=capture, + budget=MAX_RESULT_BYTES) + for index, document in enumerate(_documents(method, result)[:MAX_DOCUMENTS]): + values = [("screenshot", _get(document, "screenshot"), _IMAGE), + ("audio", _get(document, "audio"), _AUDIO), + ("video", _get(document, "video"), _VIDEO)] + shots = _get(_get(document, "actions"), "screenshots") + if isinstance(shots, (list, tuple)): + values.extend(("actions.screenshots.%d" % n, shot, _IMAGE) for n, shot in enumerate(shots[:16])) + for field, value, mimes in values: + if not media.usable: + break + if not isinstance(value, str) or not value.startswith("data:"): + continue # hosted URLs are never fetched + media.data_uri("document/%d/%s" % (index, field), lambda v=value: v, mimes, + purpose="tool_result", role="tool", message_index=index) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 4e4f6b75e..34e132a2e 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -107,8 +107,11 @@ class _State: instrumentor is removed. """ - def __init__(self) -> None: + def __init__(self, attachments: Any = None, attachment_capture: Any = None) -> None: self.enabled = True + # The shared attachment manager (None unless an enabled one was passed). + self.attachments = attachments + self.attachment_capture = attachment_capture def _span_kind(method_name: str) -> str: @@ -406,6 +409,13 @@ def _finish_ok( ) -> None: try: self._record_result(span, instance, result, kwargs) + if self._state.attachments is not None: + from traceai_firecrawl._attachments import capture_firecrawl_media + + capture_firecrawl_media( + self._state.attachments, span, self._method_name, result, + trace_config=self._config, capture=self._state.attachment_capture, + ) except Exception: logger.debug("Could not read %s result", self._span_name, exc_info=True) finally: diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py new file mode 100644 index 000000000..7f06da44b --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py @@ -0,0 +1,68 @@ +"""Tool media of an mcp-use agent run, through the shared attachment manager. + +mcp-use 1.7 converts an MCP ``CallToolResult`` into LangChain content blocks +(``agents/adapters/langchain_adapter.py``): ``ImageContent`` becomes +``{"type": "image", "source_type": "base64", "data", "mime_type"}``, +``AudioContent`` an ``"audio"`` block and an ``EmbeddedResource`` blob a +``"file"`` block. Only those typed inline blocks are observed. Text, URLs, +resource URIs and resource links are never fetched or scraped. Each block is +preflighted before its base64 is read; at most 64 blocks and 256 KiB of +decoded bytes per tool result. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Optional + +__all__ = ("capture_tool_blocks", "mcp_use_attachment_capabilities") + +mcp_use_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tool.output",), + "sources": ("inline_base64",), + "content_types": ("image", "audio", "file"), + "unsupported": ("url", "resource_uri", "resource_link", "local_path"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_EXPECT = {"image": "image", "audio": "audio", "file": None} + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else None + except Exception: + return None + + +def capture_tool_blocks(manager, span, content: Any, *, trace_config=None, capture=None, + tool_call_id: Optional[str] = None, error: bool = False) -> Mapping[str, Any]: + """Capture the typed base64 blocks of one tool result. Never raises; returns admitted positions.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return _EMPTY + try: + if not active(manager) or not isinstance(content, (list, tuple)): + return _EMPTY + media = ToolMedia(manager, span, "mcp_use", trace_config=trace_config, capture=capture) + for index, block in enumerate(content[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(block, "type") + if kind not in _EXPECT: + continue + # LangChain's v0 shape (source_type + data) and its v1 standard shape (base64). + if _get(block, "source_type") == "base64": + field = "data" + elif _get(block, "base64") is not None: + field = "base64" + else: + continue # url / id sources are never fetched + media.base64("tool/%d" % index, lambda b=block, f=field: _get(b, f), _get(block, "mime_type"), + purpose="tool_result", expect=_EXPECT[kind], role="tool", tool_call_id=tool_call_id, + part_index=index, error=error) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index 08b7e6348..a603ba26a 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -140,6 +140,8 @@ class _Run: chunks: int = 0 # A tool run's name as LangChain reported it (not capped or redacted). tool_name: Optional[str] = None + # The tool call id LangChain reported, when it is a safe token. + tool_call_id: Optional[str] = None def _describe(error: BaseException) -> str: @@ -329,6 +331,13 @@ class FutureAGICallback(BaseCallbackHandler): redact: secret strings to remove from everything written (for example an LLM key or an MCP server header the environment does not hold). + attachments: the shared attachment manager for tool media (images, + audio and inline resource blobs an MCP tool returned). Defaults + to ``tracer_provider.attachment_manager``. Capture is off unless + that manager is enabled; ``config`` hide flags and PII redaction + deny it. + attachment_capture: a ``fi_instrumentation.attachments.CapturePolicy`` + that can only add restrictions. """ raise_error = False @@ -345,6 +354,8 @@ def __init__( tracer_provider: Optional[trace_api.TracerProvider] = None, config: Optional[TraceConfig] = None, redact: Iterable[str] = (), + attachments: Any = None, + attachment_capture: Any = None, ) -> None: super().__init__() if config is None: @@ -373,6 +384,11 @@ def __init__( self._hide_outputs = bool(config.hide_outputs) self._pii = bool(config.pii_redaction) self._secrets = Secrets(secrets) + self._config = config + self._attachments = ( + attachments if attachments is not None else getattr(provider, "attachment_manager", None) + ) + self._attachment_capture = attachment_capture self._lock = threading.RLock() self._runs: Dict[UUID, _Run] = {} self._roots: "OrderedDict[UUID, _Root]" = OrderedDict() @@ -1042,6 +1058,9 @@ def on_tool_start( return if isinstance(raw_name, str): run.tool_name = raw_name + raw_call_id = kwargs.get("tool_call_id") + if isinstance(raw_call_id, str): + run.tool_call_id = raw_call_id if run.span is None: return arguments = _json(inputs) if isinstance(inputs, Mapping) else input_str @@ -1092,6 +1111,8 @@ def on_tool_end( ) except Exception: logger.debug("Could not classify the tool result", exc_info=True) + if run.span is not None and self._attachments is not None: + self._capture_tool_media(run, output, failure is not None) text = _tool_output_text(output) self._remember(root, [text], output=True) if failure is not None and root is not None: @@ -1115,6 +1136,16 @@ def on_tool_end( except Exception: logger.debug("mcp-use callback on_tool_end failed", exc_info=True) + def _capture_tool_media(self, run: _Run, output: Any, failed: bool) -> None: + """Upload the typed inline media blocks a tool returned (never URLs or paths).""" + try: + from traceai_mcp_use._attachments import capture_tool_blocks + except ImportError: + return + content = getattr(output, "content", output) + capture_tool_blocks(self._attachments, run.span, content, trace_config=self._config, + capture=self._attachment_capture, tool_call_id=run.tool_call_id, error=failed) + def on_tool_error( self, error: BaseException, diff --git a/python/frameworks/mcp/traceai_mcp/__init__.py b/python/frameworks/mcp/traceai_mcp/__init__.py index 1a8e685e3..d2b3bf458 100644 --- a/python/frameworks/mcp/traceai_mcp/__init__.py +++ b/python/frameworks/mcp/traceai_mcp/__init__.py @@ -8,12 +8,19 @@ from opentelemetry import context, propagate from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from traceai_mcp.package import _instruments +from traceai_mcp._attachments import capture_mcp_result, mcp_attachment_capabilities, wrap_mcp_client from wrapt import ObjectProxy, register_post_import_hook, wrap_function_wrapper +__all__ = ["MCPInstrumentor", "capture_mcp_result", "mcp_attachment_capabilities", "wrap_mcp_client"] + + class MCPInstrumentor(BaseInstrumentor): # type: ignore """ An instrumenter for MCP. + + It propagates trace context over MCP transports only and never captures + media. Use ``wrap_mcp_client(session)`` for tool and resource media. """ def instrumentation_dependencies(self) -> Collection[str]: diff --git a/python/frameworks/mcp/traceai_mcp/_attachments.py b/python/frameworks/mcp/traceai_mcp/_attachments.py new file mode 100644 index 000000000..1851e2fee --- /dev/null +++ b/python/frameworks/mcp/traceai_mcp/_attachments.py @@ -0,0 +1,199 @@ +"""MCP tool and resource media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/mcp`` (``wrapMCPClient`` / ``captureMCPResult``). Only typed +inline bytes are observed: ``ImageContent.data``, ``AudioContent.data`` and an +``EmbeddedResource`` (or ``resources/read``) ``BlobResourceContents.blob``. +Resource URIs, ``resource_link`` blocks, URLs, local paths and +``structuredContent`` are never fetched, opened or scraped. Each part is +preflighted before its base64 is read or decoded; at most 64 parts and +256 KiB of decoded bytes per result. + +This module imports ``fi_instrumentation.attachments`` lazily, so the +transport-only ``MCPInstrumentor`` keeps working on a core without it. +""" + +import functools +import inspect +from types import MappingProxyType +from typing import Any, Mapping, Optional + +from opentelemetry import context as otel_context +from opentelemetry import trace as trace_api +from opentelemetry.trace import SpanKind, Status, StatusCode +from wrapt import ObjectProxy + +from traceai_mcp.version import __version__ + +__all__ = ("capture_mcp_result", "wrap_mcp_client", "mcp_attachment_capabilities") + +#: Declarative handoff for auto and reconciliation. +mcp_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tools/call.output", "resources/read.output"), + "sources": ("inline_base64",), + "content_types": ("image", "audio", "resource.blob"), + "unsupported": ("resource_link", "resource_uri", "remote_url", "local_path", "structured_content", + "transport_only_instrumentor"), + "max_parts": 64, + "max_decoded_bytes_per_result": 256 * 1024, +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_OPERATIONS = ("tools/call", "resources/read") + + +def _get(value: Any, *names: str) -> Any: + """A field of an MCP model (1.x camelCase or 2.x snake_case) or a plain dict; never raises.""" + for name in names: + try: + found = value.get(name) if isinstance(value, Mapping) else getattr(value, name, None) + except Exception: + found = None + if found is not None: + return found + return None + + +def _active(manager) -> bool: + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return False + return active(manager) + + +def capture_mcp_result(manager, span, result: Any, operation: str = "tools/call", capture=None, + trace_config=None) -> Mapping[str, Any]: + """Capture typed inline media of one MCP ``tools/call`` or ``resources/read`` result. + + ``span`` is the span that represents this call. Tool media is tool output + and potential model input, so input and output hide flags both apply. + Returns the admitted positions (``"/"`` -> occurrence); + empty when no manager, disabled, unsampled or denied. Never raises and + never changes ``result``. + """ + if operation not in _OPERATIONS or not _active(manager): + return _EMPTY + try: + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return _EMPTY + try: + media = ToolMedia(manager, span, "mcp", trace_config=trace_config, capture=capture) + if not media.usable: + return _EMPTY + error = _get(result, "isError", "is_error") is True + parts = _get(result, "content" if operation == "tools/call" else "contents") + if not isinstance(parts, (list, tuple)): + return _EMPTY + for index, part in enumerate(parts[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(part, "type") + if operation == "resources/read": + holder, field, expect = part, "blob", None + elif kind in ("image", "audio"): + holder, field, expect = part, "data", kind + elif kind == "resource": + holder, field, expect = _get(part, "resource"), "blob", None + else: + continue # text, resource_link and unknown blocks are never dereferenced + if holder is None: + continue + mime = _get(holder, "mimeType", "mime_type") + media.base64("%s/%d" % (operation, index), lambda h=holder, f=field: _get(h, f), mime, + purpose="tool_result", expect=expect, role="tool", part_index=index, error=error) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY # Malformed or untrusted results must not change the tool response. + + +def _provider_manager(tracer_provider) -> Any: + owner = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + manager = getattr(owner, "attachment_manager", None) + if manager is None: + delegate = getattr(owner, "_real_tracer_provider", None) or getattr(owner, "_tracer_provider", None) + manager = getattr(delegate, "attachment_manager", None) + return manager + + +class _TracedSession(ObjectProxy): # type: ignore[misc] + """Proxies an MCP ``ClientSession``; only ``call_tool`` and ``read_resource`` gain a span.""" + + def __init__(self, wrapped, config): + super().__init__(wrapped) + self._self_config = config + + def _self_traced(self, operation: str, method, args, kwargs): + tracer_provider, attachments, trace_config, capture = self._self_config + + @functools.wraps(method) + async def run(): + span = None + manager = None + try: + manager = attachments if attachments is not None else _provider_manager(tracer_provider) + provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + tracer = provider.get_tracer("traceai_mcp", __version__) + span = tracer.start_span("mcp.%s" % operation, kind=SpanKind.CLIENT, + attributes={"gen_ai.span.kind": "TOOL", "mcp.method.name": operation}) + except Exception: + span = None # Tracing never prevents the SDK call. + if span is None: + return await method(*args, **kwargs) + token = otel_context.attach(trace_api.set_span_in_context(span)) + try: + result = await method(*args, **kwargs) + except BaseException: + otel_context.detach(token) + try: + span.set_status(Status(StatusCode.ERROR)) + finally: + span.end() + raise + otel_context.detach(token) + try: + if manager is not None: + capture_mcp_result(manager, span, result, operation, capture, trace_config) + failed = _get(result, "isError", "is_error") is True + span.set_status(Status(StatusCode.ERROR if failed else StatusCode.OK)) + except Exception: + pass # Never log SDK results or exceptions. + finally: + span.end() + return result + + return run() + + def call_tool(self, *args, **kwargs): + method = self.__wrapped__.call_tool + if not inspect.iscoroutinefunction(method): + return method(*args, **kwargs) + return self._self_traced("tools/call", method, args, kwargs) + + def read_resource(self, *args, **kwargs): + method = self.__wrapped__.read_resource + if not inspect.iscoroutinefunction(method): + return method(*args, **kwargs) + return self._self_traced("resources/read", method, args, kwargs) + + +def wrap_mcp_client(session, *, tracer_provider=None, attachments=None, trace_config=None, + attachment_capture=None): + """Trace a ``mcp.ClientSession``'s ``call_tool`` and ``read_resource`` and capture their media. + + ``attachments`` defaults to the provider's shared manager + (``tracer_provider.attachment_manager``, as ``fi_instrumentation.register`` + and ``traceai.init`` set it); no uploader is created here. ``trace_config`` + and ``attachment_capture`` can only add restrictions. The SDK result, + exception and cancellation are returned unchanged. Wrapping a traced session + again returns it unchanged, so spans never nest and media is never uploaded + twice. + """ + # A traced session is returned as is, so spans never nest and media is never captured twice. + # (ObjectProxy delegates ==/hash to the session, so identity caches would see them as equal.) + if isinstance(session, _TracedSession): + return session + proxy = _TracedSession(session, (tracer_provider, attachments, trace_config, attachment_capture)) + return proxy diff --git a/python/frameworks/openai-agents/traceai_openai_agents/__init__.py b/python/frameworks/openai-agents/traceai_openai_agents/__init__.py index fa114bfe7..b5f5e0edc 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/__init__.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/__init__.py @@ -12,6 +12,23 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class OpenAIAgentsInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for openai-agents @@ -36,15 +53,22 @@ def _instrument(self, **kwargs: Any) -> None: from traceai_openai_agents._processor import FiTracingProcessor - + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + processor = FiTracingProcessor( + cast(Tracer, tracer), + attachments=_attachment_manager(kwargs), + trace_config=config, + attachment_capture=kwargs.get("attachment_capture"), + ) if exclusive_processor: from agents import set_trace_processors - set_trace_processors([FiTracingProcessor(cast(Tracer, tracer))]) + set_trace_processors([processor]) else: from agents import add_trace_processor - add_trace_processor(FiTracingProcessor(cast(Tracer, tracer))) + add_trace_processor(processor) def _uninstrument(self, **kwargs: Any) -> None: # TODO : OpenAI Agents does not support uninstrumentation currently diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py b/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py new file mode 100644 index 000000000..6b318edb6 --- /dev/null +++ b/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py @@ -0,0 +1,110 @@ +"""Tool media of OpenAI Agents SDK function spans, through the shared attachment manager. + +The SDK records what a tool returned on its ``FunctionSpanData.output`` (only +when ``RunConfig.trace_include_sensitive_data`` is on). These typed outputs +are observed, at the tool span, before ``OUTPUT_VALUE`` is written: + +- the computer tool's screenshot: a ``data:image/png;base64,`` string on the + ``computer`` function span (SDK 0.10.0 and later record it there); +- ``ToolOutputImage`` / ``{"type": "image", "image_url": }`` + (SDK 0.4.0+; MCP image results become these from 0.8.0); +- ``ToolOutputFileContent`` / ``{"type": "file", "file_data": }``; +- a list of those. + +Remote URLs, ``file_id`` values and plain strings on other tools are never +fetched or scraped. Model-call input media (``ResponseSpanData.input``) is +left to the OpenAI provider adapter, which owns that span. Admitted values +become ``fi-attachment:`` in the span's ``OUTPUT_VALUE``; the +SDK's own objects are never changed. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Tuple + +__all__ = ("capture_function_output", "openai_agents_attachment_capabilities") + +COMPUTER_TOOL_NAMES = ("computer", "computer_use_preview") +_IMAGE_MIMES = ("image/png", "image/jpeg", "image/webp") +_FILE_MIMES = ("application/pdf", "application/octet-stream") +_EMPTY: Mapping[str, Any] = MappingProxyType({}) + +openai_agents_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("function.output", "computer.screenshot"), + "sources": ("inline_data_uri",), + "minimum_sdk": {"computer.screenshot": "0.10.0", "tool_output_image": "0.4.0", "mcp_image": "0.8.0"}, + "unsupported": ("remote_url", "file_id", "response_input", "trace_include_sensitive_data_off"), +}) + + +def _field(value: Any, key: str) -> Any: + try: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + except Exception: + return None + + +def _typed(value: Any) -> bool: + return isinstance(value, Mapping) or type(value).__name__ in ("ToolOutputImage", "ToolOutputFileContent") + + +def capture_function_output(manager, span, name: Any, output: Any, *, trace_config=None, + capture=None) -> Tuple[Mapping[str, Any], Any]: + """Capture one function span's typed media; returns (admitted positions, telemetry output). + + The telemetry output is ``output`` itself when nothing was admitted, else a + copy where only the admitted values are ``fi-attachment:`` URIs. Never raises. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia, uri + except ImportError: + return _EMPTY, output + try: + if not active(manager) or output is None: + return _EMPTY, output + media = ToolMedia(manager, span, "openai_agents", trace_config=trace_config, capture=capture) + if not media.usable: + return _EMPTY, output + + def item(value, index): + position = "function/%d" % index + if isinstance(value, str): + if name in COMPUTER_TOOL_NAMES and value.startswith("data:image/"): + found = media.data_uri(position, lambda: value, _IMAGE_MIMES, purpose="computer_screenshot", + role="tool", part_index=index) + return uri(found) if found else value + return value + if not _typed(value): + return value + kind = _field(value, "type") + if kind == "image": + key, mimes = "image_url", _IMAGE_MIMES + elif kind == "file": + key, mimes = "file_data", _FILE_MIMES + else: + return value + encoded = _field(value, key) + if not isinstance(encoded, str) or not encoded.startswith("data:"): + return value # URLs and file ids are never fetched + found = media.data_uri(position, lambda: encoded, mimes, purpose="tool_result", role="tool", + part_index=index) + if not found: + return value + plain = dict(value) if isinstance(value, Mapping) else value.model_dump(exclude_none=True) + plain[key] = uri(found) + return plain + + if isinstance(output, list): + projected = [item(value, index) if index < MAX_TOOL_PARTS else value + for index, value in enumerate(output)] + else: + projected = item(output, 0) + captured = media.captured + if not captured: + return _EMPTY, output + return MappingProxyType(captured), projected + except Exception: + return _EMPTY, output diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py index 48684260b..ea23b690c 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py @@ -67,8 +67,19 @@ class FiTracingProcessor(TracingProcessor): _MAX_HANDOFFS_IN_FLIGHT = 1000 - def __init__(self, tracer: Tracer) -> None: + def __init__( + self, + tracer: Tracer, + *, + attachments: Any = None, + trace_config: Any = None, + attachment_capture: Any = None, + ) -> None: self._tracer = tracer + # The shared attachment manager (None unless an enabled one was passed). + self._attachments = attachments + self._trace_config = trace_config + self._attachment_capture = attachment_capture self._root_spans: dict[str, OtelSpan] = {} self._otel_spans: dict[str, OtelSpan] = {} self._tokens: dict[str, object] = {} @@ -227,7 +238,19 @@ def _set_span_attributes(self, span: Span[Any], otel_span: OtelSpan) -> None: for k, v in _get_attributes_from_generation_span_data(data): otel_span.set_attribute(k, v) elif isinstance(data, FunctionSpanData): - for k, v in _get_attributes_from_function_span_data(data): + output = data.output + if self._attachments is not None and output is not None: + from traceai_openai_agents._attachments import capture_function_output + + _, output = capture_function_output( + self._attachments, + otel_span, + data.name, + output, + trace_config=self._trace_config, + capture=self._attachment_capture, + ) + for k, v in _get_attributes_from_function_span_data(data, output): otel_span.set_attribute(k, v) elif isinstance(data, MCPListToolsSpanData): for k, v in _get_attributes_from_mcp_list_tool_span_data(data): @@ -560,20 +583,27 @@ def _convert_to_primitive(value: Any) -> Union[bool, str, bytes, int, float]: return str(value) +_UNSET = object() + + def _get_attributes_from_function_span_data( obj: FunctionSpanData, + output: Any = _UNSET, ) -> Iterator[tuple[str, AttributeValue]]: + # ``output`` is a telemetry copy in which admitted media are fi-attachment URIs. + if output is _UNSET: + output = obj.output yield GEN_AI_TOOL_NAME, obj.name if obj.input: yield INPUT_VALUE, obj.input yield INPUT_MIME_TYPE, JSON - if obj.output is not None: - yield OUTPUT_VALUE, _convert_to_primitive(obj.output) + if output is not None: + yield OUTPUT_VALUE, _convert_to_primitive(output) if ( - isinstance(obj.output, str) - and len(obj.output) > 1 - and obj.output[0] == "{" - and obj.output[-1] == "}" + isinstance(output, str) + and len(output) > 1 + and output[0] == "{" + and output[-1] == "}" ): yield OUTPUT_MIME_TYPE, JSON diff --git a/python/frameworks/smolagents/traceai_smolagents/__init__.py b/python/frameworks/smolagents/traceai_smolagents/__init__.py index e6aac0cdd..642e55863 100644 --- a/python/frameworks/smolagents/traceai_smolagents/__init__.py +++ b/python/frameworks/smolagents/traceai_smolagents/__init__.py @@ -4,6 +4,7 @@ from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from traceai_smolagents._wrappers import ( + _Media, _ModelWrapper, _RunWrapper, _StepWrapper, @@ -16,6 +17,23 @@ _instruments = ("smolagents >= 1.2.2",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class SmolagentsInstrumentor(BaseInstrumentor): __slots__ = ( "_original_run_method", @@ -43,6 +61,9 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + media = _Media(_attachment_manager(kwargs), config, kwargs.get("attachment_capture")) run_wrapper = _RunWrapper(tracer=self._tracer) self._original_run_method = getattr(MultiStepAgent, "run", None) @@ -72,9 +93,10 @@ def _instrument(self, **kwargs: Any) -> None: if method not in vars(model_subclass): continue self._original_model_call_methods[(model_subclass, method)] = vars(model_subclass)[method] - wrap_function_wrapper(model_subclass, method, _ModelWrapper(tracer=self._tracer, method_name=method)) + wrap_function_wrapper(model_subclass, method, + _ModelWrapper(tracer=self._tracer, method_name=method, media=media)) - tool_call_wrapper = _ToolCallWrapper(tracer=self._tracer) + tool_call_wrapper = _ToolCallWrapper(tracer=self._tracer, media=media) self._original_tool_call_method = getattr(Tool, "__call__", None) wrap_function_wrapper( module="smolagents", diff --git a/python/frameworks/smolagents/traceai_smolagents/_attachments.py b/python/frameworks/smolagents/traceai_smolagents/_attachments.py new file mode 100644 index 000000000..20ebe166b --- /dev/null +++ b/python/frameworks/smolagents/traceai_smolagents/_attachments.py @@ -0,0 +1,136 @@ +"""smolagents typed media observations, through the shared attachment manager. + +Observed, and nothing else: + +- images smolagents hands to a model: ``{"type": "image", "image": }`` + content blocks (task images and ``ActionStep.observations_images``, such as + a browser screenshot a step callback stored), on the model span as model + input, before the model converts them; +- a tool's returned image: an in-memory ``PIL.Image.Image`` or an + ``AgentImage`` whose image is already in memory, on the tool span. + +Images are PNG-encoded only after a pixel-bounded preflight admitted them. +A file-backed ``AgentImage`` or PIL image that is not loaded yet, image URLs, +``image_url`` strings and every ``AgentAudio`` (a path or a tensor) are not +read (``source_unavailable``). smolagents replays memory images on every +model call; the same image object is uploaded once per manager. +""" + +import threading +import weakref +from collections import OrderedDict +from types import MappingProxyType +from typing import Any, Mapping + +__all__ = ("capture_model_images", "capture_tool_image", "smolagents_attachment_capabilities") + +smolagents_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.input.image", "tool.output.image"), + "sources": ("in_memory_pil_image",), + "unsupported": ("agent_audio", "file_backed_image", "image_url", "tensor"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_MAX_SEEN = 1024 +_MAX_MESSAGES = 256 + + +class _Seen: + """Image objects already uploaded, per manager (identity only; PIL images are unhashable).""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() + + def __contains__(self, item) -> bool: + manager, image = item + with self._lock: + entry = self._managers.get(manager, {}).get(id(image)) + return entry is not None and entry() is image + + def add(self, manager, image) -> None: + with self._lock: + seen = self._managers.get(manager) + if seen is None: + seen = self._managers[manager] = OrderedDict() + seen[id(image)] = weakref.ref(image) + while len(seen) > _MAX_SEEN: + seen.popitem(last=False) + + +_SEEN = _Seen() + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def _in_memory(value: Any) -> Any: + """The PIL image an ``AgentImage`` already holds in memory, else the value itself.""" + if type(value).__name__ == "AgentImage": + raw = getattr(value, "_raw", None) + return raw # None for path- or tensor-backed values: never loaded here + return value + + +def capture_model_images(manager, span, messages: Any, *, trace_config=None, capture=None) -> int: + """Capture the in-memory images of one model call's input messages. Returns the admitted count.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return 0 + admitted = 0 + try: + if not active(manager) or not isinstance(messages, (list, tuple)): + return 0 + media = ToolMedia(manager, span, "smolagents", trace_config=trace_config, capture=capture, + budget=8 * 1024 * 1024, direction="input") + for message_index, message in enumerate(messages[:_MAX_MESSAGES]): + content = _get(message, "content") + if not isinstance(content, list): + continue + for part_index, part in enumerate(content[:MAX_TOOL_PARTS]): + if not media.usable: + return admitted + if _get(part, "type") != "image": + continue + image = _in_memory(_get(part, "image")) + if image is None or (manager, image) in _SEEN: + continue + found = media.image("model/%d/%d" % (message_index, part_index), image, purpose="model_input", + role="user", message_index=message_index, part_index=part_index) + if found is not None: + _SEEN.add(manager, image) + admitted += 1 + except Exception: + return admitted + return admitted + + +def capture_tool_image(manager, span, response: Any, *, trace_config=None, capture=None) -> int: + """Capture a tool's returned in-memory image on its span. Returns 1 when admitted.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import ToolMedia + except ImportError: + return 0 + try: + if not active(manager): + return 0 + image = _in_memory(response) + if image is None or (manager, image) in _SEEN: + return 0 + media = ToolMedia(manager, span, "smolagents", trace_config=trace_config, capture=capture, + budget=8 * 1024 * 1024) + found = media.image("tool/0", image, purpose="tool_result", role="tool", part_index=0) + if found is None: + return 0 + _SEEN.add(manager, image) + return 1 + except Exception: + return 0 diff --git a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py index 0c3cd0cb1..49b37074f 100644 --- a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py +++ b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py @@ -418,10 +418,23 @@ def _input_value_and_mime_type( yield INPUT_VALUE, safe_json_dumps(arguments) +_OWNER_KEY = "traceai.auto.attachment_owner.v1" + + +class _Media: + """The shared attachment manager and its local restrictions (None manager: capture off).""" + + __slots__ = ("manager", "trace_config", "capture") + + def __init__(self, manager: Any = None, trace_config: Any = None, capture: Any = None) -> None: + self.manager, self.trace_config, self.capture = manager, trace_config, capture + + class _ModelWrapper: - def __init__(self, tracer: trace_api.Tracer, method_name: str = "__call__") -> None: + def __init__(self, tracer: trace_api.Tracer, method_name: str = "__call__", media: Any = None) -> None: self._tracer = tracer self._method_name = method_name + self._media = media if media is not None else _Media() def __call__( self, @@ -446,7 +459,19 @@ def __call__( **dict(_get_raw_input(args, **kwargs)), }, ) as span: - output_message = wrapped(*args, **kwargs) + owner_token = None + if self._media.manager is not None: + from traceai_smolagents._attachments import capture_model_images + + if capture_model_images(self._media.manager, span, arguments.get("messages"), + trace_config=self._media.trace_config, capture=self._media.capture): + # These images are captured here; a provider adapter below must not upload them again. + owner_token = context_api.attach(context_api.set_value(_OWNER_KEY, True)) + try: + output_message = wrapped(*args, **kwargs) + finally: + if owner_token is not None: + context_api.detach(owner_token) span.set_status(trace_api.StatusCode.OK) span.set_attribute(OUTPUT_VALUE, _get_raw_output(output_message)) # Modern SDKs return usage on ChatMessage; historical releases @@ -473,8 +498,9 @@ def __call__( class _ToolCallWrapper: - def __init__(self, tracer: trace_api.Tracer) -> None: + def __init__(self, tracer: trace_api.Tracer, media: Any = None) -> None: self._tracer = tracer + self._media = media if media is not None else _Media() def __call__( self, @@ -501,6 +527,11 @@ def __call__( }, ) as span: response = wrapped(*args, **kwargs) + if self._media.manager is not None: + from traceai_smolagents._attachments import capture_tool_image + + capture_tool_image(self._media.manager, span, response, + trace_config=self._media.trace_config, capture=self._media.capture) span.set_attribute(OUTPUT_VALUE, _get_raw_output(response)) span.set_status(trace_api.StatusCode.OK) span.set_attributes( diff --git a/python/tests/attachments_mcp_agents/_fake_claude_cli.py b/python/tests/attachments_mcp_agents/_fake_claude_cli.py new file mode 100644 index 000000000..29a56adba --- /dev/null +++ b/python/tests/attachments_mcp_agents/_fake_claude_cli.py @@ -0,0 +1,97 @@ +"""A stand-in for the Claude Code CLI that speaks the SDK's stream-json control protocol. + +The real ``claude_agent_sdk`` spawns this through ``ClaudeAgentOptions(cli_path=...)``. +For every user message it replays one tool call: the assistant ``tool_use``, the +PreToolUse and PostToolUse hook callbacks the SDK registered, the ``tool_result`` +carrying the media in ``FAKE_CLAUDE_RESULT`` (a JSON file), then a final answer. +``FAKE_CLAUDE_ORDER=result-first`` streams the tool result before PostToolUse. +No model, network or file other than the result file is touched. +""" + +import json +import os +import sys + +if "-v" in sys.argv[1:] or "--version" in sys.argv[1:]: + print("2.1.999 (Claude Code)") + sys.exit(0) + +with open(os.environ["FAKE_CLAUDE_RESULT"]) as handle: + TOOL_RESULT = json.load(handle) +ORDER = os.environ.get("FAKE_CLAUDE_ORDER", "hooks-first") +TOOL = "mcp__screen__capture" +hooks = {} +counter = [0] + + +def send(message): + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +def wait_response(request_id): + for line in sys.stdin: + message = json.loads(line) + if message.get("type") == "control_response": + if message.get("response", {}).get("request_id") == request_id: + return message + else: + handle(message) + + +def hook(event, tool_use_id, extra): + for callback_id in hooks.get(event, []): + counter[0] += 1 + request_id = "cli_%d" % counter[0] + send({"type": "control_request", "request_id": request_id, "request": { + "subtype": "hook_callback", "callback_id": callback_id, "tool_use_id": tool_use_id, + "input": dict({"hook_event_name": event, "session_id": "s1", "tool_name": TOOL, "tool_input": {}, + "cwd": "/", "transcript_path": ""}, **extra)}}) + wait_response(request_id) + + +def assistant(content): + send({"type": "assistant", "parent_tool_use_id": None, "session_id": "s1", "message": { + "id": "msg_%d" % counter[0], "type": "message", "role": "assistant", "model": "claude-test", + "content": content, "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 1}}}) + + +def turn(): + tool_use_id = "toolu_screen_1" + assistant([{"type": "tool_use", "id": tool_use_id, "name": TOOL, "input": {}}]) + hook("PreToolUse", tool_use_id, {}) + result = {"type": "user", "parent_tool_use_id": None, "session_id": "s1", "message": { + "role": "user", "content": [dict(TOOL_RESULT, type="tool_result", tool_use_id=tool_use_id)]}} + if ORDER == "result-first": + send(result) + hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) + else: + hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) + send(result) + assistant([{"type": "text", "text": "done"}]) + send({"type": "result", "subtype": "success", "duration_ms": 1, "duration_api_ms": 1, "is_error": False, + "num_turns": 1, "session_id": "s1", "total_cost_usd": 0.0, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}) + + +def handle(message): + kind = message.get("type") + if kind == "control_request": + request = message.get("request", {}) + if request.get("subtype") == "initialize": + for event, matchers in (request.get("hooks") or {}).items(): + hooks[event] = [cid for matcher in matchers for cid in matcher.get("hookCallbackIds", [])] + send({"type": "control_response", "response": { + "subtype": "success", "request_id": message.get("request_id"), + "response": {"commands": [], "models": [], "account": {}} if request.get("subtype") == "initialize" + else {}}}) + elif kind == "user": + turn() + + +send({"type": "system", "subtype": "init", "session_id": "s1", "tools": [], "mcp_servers": [], "model": "claude-test", + "cwd": "/", "permissionMode": "default", "apiKeySource": "none"}) +for line in sys.stdin: + line = line.strip() + if line: + handle(json.loads(line)) diff --git a/python/tests/attachments_mcp_agents/_harness.py b/python/tests/attachments_mcp_agents/_harness.py new file mode 100644 index 000000000..b7e2d2a53 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_harness.py @@ -0,0 +1,44 @@ +"""Shared helpers: the provider-attachments harness, loaded by path (one F01 loopback for every lane).""" + +import importlib.util +import json +import sys +from pathlib import Path + +_NAME = "_attachments_providers_harness" +if _NAME in sys.modules: + harness = sys.modules[_NAME] +else: + _path = Path(__file__).resolve().parent.parent / "attachments_providers" / "conftest.py" + _spec = importlib.util.spec_from_file_location(_NAME, _path) + harness = importlib.util.module_from_spec(_spec) + sys.modules[_NAME] = harness + _spec.loader.exec_module(harness) + +png, wav, b64, refs, wait_uploads, flush, PDF = (harness.png, harness.wav, harness.b64, harness.refs, + harness.wait_uploads, harness.flush, harness.PDF) + + +def data_uri(mime, data): + return "data:%s;base64,%s" % (mime, b64(data)) + + +def uploaded(f01): + """(mime_type, direction, purpose, role, tool_call_id) of each initialize body, sorted.""" + return sorted((u.get("mime_type"), u.get("direction"), u.get("purpose"), u.get("role"), + (u.get("correlation") or {}).get("tool_call_id") or u.get("tool_call_id")) + for u in f01.uploads) + + +def span_named(exporter, name): + found = [s for s in exporter.get_finished_spans() if s.name == name] + assert found, [s.name for s in exporter.get_finished_spans()] + return found[-1] + + +def no_bytes(exporter, data): + """No exported span attribute carries the media's base64 text.""" + text = b64(data)[:40] + for span in exporter.get_finished_spans(): + for value in (span.attributes or {}).values(): + assert text not in json.dumps(value, default=str), span.name diff --git a/python/tests/attachments_mcp_agents/_mcp_media_server.py b/python/tests/attachments_mcp_agents/_mcp_media_server.py new file mode 100644 index 000000000..726dde217 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_mcp_media_server.py @@ -0,0 +1,116 @@ +"""A real MCP server over stdio whose tools return synthetic typed media. + +Used by the MCP (``wrap_mcp_client``) and mcp-use tests. It talks to nothing +else. ``reads`` reports how many times the client read each resource, so a +test can prove a ``resource_link`` URI was never fetched. +""" + +import base64 +import json +import struct +import zlib +from importlib.metadata import version + +from mcp import types + +if int(version("mcp").split(".")[0]) >= 2: + from mcp.server.mcpserver import MCPServer as Server +else: + from mcp.server.fastmcp import FastMCP as Server + + +def png(seed=0, width=2, height=2) -> bytes: + raw = b"".join(b"\x00" + bytes((seed + x + y) % 256 for x in range(width * 3)) for y in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +WAV = b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00@\x1f\x00\x00\x80>\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00" +PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n" +SECRET_FILE = "file:///etc/hostname" +READS = {"shot": 0, "secret": 0} + + +def b64(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def block(kind, **fields): + """A content block from its wire (camelCase) shape, valid on MCP 1.x and 2.x.""" + model = {"image": types.ImageContent, "audio": types.AudioContent, "text": types.TextContent, + "resource": types.EmbeddedResource, "resource_link": types.ResourceLink}[kind] + return model.model_validate({"type": kind, **fields}) + + +server = Server("media-fixture") + + +@server.tool() +def screenshot(): + """A screenshot and a caption.""" + return [block("text", text="CAPTION-MARKER"), block("image", data=b64(png(1)), mimeType="image/png")] + + +@server.tool() +def voice(): + """A short WAV clip.""" + return [block("audio", data=b64(WAV), mimeType="audio/wav")] + + +@server.tool() +def report(): + """An embedded PDF resource blob plus a resource link that must never be fetched.""" + return [ + block("resource", resource={"uri": "file:///report.pdf", "mimeType": "application/pdf", "blob": b64(PDF)}), + block("resource_link", uri=SECRET_FILE, name="hostname", mimeType="image/png"), + ] + + +@server.tool() +def relabeled(): + """An image block whose MIME claims another modality, and one outside the F01 enum.""" + return [block("image", data=b64(png(2)), mimeType="audio/wav"), + block("image", data=b64(png(3)), mimeType="image/gif")] + + +@server.tool() +def malformed(): + """Not valid base64.""" + return [block("image", data="!!!not-base64!!!", mimeType="image/png")] + + +@server.tool() +def many(count: int): + """``count`` distinct images.""" + return [block("image", data=b64(png(10 + n)), mimeType="image/png") for n in range(count)] + + +@server.tool() +def big(): + """An image larger than the 256 KiB per-result budget.""" + return [block("image", data=b64(png(4) + b"\x00" * (300 * 1024)), mimeType="image/png")] + + +@server.tool() +def reads() -> str: + return json.dumps(READS) + + +@server.resource("file:///shot.png", mime_type="image/png") +def shot() -> bytes: + READS["shot"] += 1 + return png(5) + + +@server.resource(SECRET_FILE, mime_type="image/png") +def secret() -> bytes: + READS["secret"] += 1 + return png(6) + + +if __name__ == "__main__": + server.run(transport="stdio") diff --git a/python/tests/attachments_mcp_agents/conftest.py b/python/tests/attachments_mcp_agents/conftest.py new file mode 100644 index 000000000..536f9cd48 --- /dev/null +++ b/python/tests/attachments_mcp_agents/conftest.py @@ -0,0 +1,21 @@ +"""Loopback harness for MCP and agent tool-media tests: synthetic data only. + +Reuses the provider-attachments harness (``tests/attachments_providers``): +``f01`` is a TLS loopback F01 attachment server and ``spans`` an in-memory +OpenTelemetry provider. Nothing here reaches a network beyond 127.0.0.1. +""" + +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("fi_instrumentation.attachments.tool_media") +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _harness import harness # noqa: E402 + +# Fixtures, registered by being attributes of this conftest. +clean_env = harness.clean_env +f01 = harness.f01 +spans = harness.spans diff --git a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py new file mode 100644 index 000000000..12f832a7e --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py @@ -0,0 +1,135 @@ +"""Real claude_agent_sdk ClaudeSDKClient over its subprocess transport -> loopback F01. + +The CLI is ``_fake_claude_cli.py`` (the SDK's real stream-json control protocol: +initialize, hook callbacks, user/assistant/result messages); no model or key. +""" + +import asyncio +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +claude_agent_sdk = pytest.importorskip("claude_agent_sdk") +pytest.importorskip("traceai_claude_agent_sdk") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_claude_agent_sdk import ClaudeAgentInstrumentor # noqa: E402 + +from _harness import PDF, b64, flush, png, refs # noqa: E402 + +CLI = Path(__file__).resolve().parent / "_fake_claude_cli.py" +SHOT = png(seed=51) + + +def image(data, mime="image/png"): + return {"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64(data)}} + + +@pytest.fixture +def cli(tmp_path, monkeypatch): + def make(content, order="hooks-first", is_error=False): + result = tmp_path / "result.json" + result.write_text(json.dumps({"content": content, "is_error": is_error})) + launcher = tmp_path / "claude" + launcher.write_text("#!%s\nimport runpy\nrunpy.run_path(%r, run_name='__main__')\n" % (sys.executable, str(CLI))) + launcher.chmod(launcher.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("FAKE_CLAUDE_RESULT", str(result)) + monkeypatch.setenv("FAKE_CLAUDE_ORDER", order) + monkeypatch.setenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "1") + return str(launcher) + + return make + + +@pytest.fixture +def instrument(spans): + provider, exporter = spans + state = {} + + def install(manager, config=None): + instrumentor = ClaudeAgentInstrumentor() + instrumentor.instrument(tracer_provider=provider, attachments=manager, config=config or TraceConfig(), + skip_dep_check=True) + state["instrumentor"] = instrumentor + return exporter + + yield install + if "instrumentor" in state: + state["instrumentor"].uninstrument() + + +def converse(cli_path): + async def go(): + options = claude_agent_sdk.ClaudeAgentOptions(cli_path=cli_path) + messages = [] + async with claude_agent_sdk.ClaudeSDKClient(options=options) as client: + await client.query("take a screenshot") + async for message in client.receive_response(): + messages.append(message) + return messages + + return asyncio.run(go()) + + +def tool_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.startswith("tool.")] + + +@pytest.mark.parametrize("order", ["hooks-first", "result-first"]) +def test_tool_result_image_and_pdf_upload_before_truncation_on_the_tool_span(f01, instrument, cli, order): + manager = f01.manager() + exporter = instrument(manager) + messages = converse(cli([{"type": "text", "text": "captured"}, image(SHOT), + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", + "data": b64(PDF)}}], order=order)) + flush(manager) + assert type(messages[-1]).__name__ == "ResultMessage" + assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ + ("application/pdf", "output", "tool_result", "toolu_screen_1"), + ("image/png", "output", "tool_result", "toolu_screen_1")] + assert sorted(f01.payloads) == sorted([SHOT, PDF]) + (tool,) = tool_spans(exporter) + parents = {u["association"]["parent"]["span_id"] for u in f01.uploads} + assert parents == {format(tool.context.span_id, "016x")} + if order == "result-first": + assert len(refs(tool)) == 2 # still open: projected on the span + # Bytes never reach a text attribute (tool output, conversation output). + for span in exporter.get_finished_spans(): + for value in span.attributes.values(): + assert b64(SHOT)[:40] not in str(value) + manager.shutdown(5) + + +def test_url_and_file_sources_never_fetch(f01, instrument, cli): + manager = f01.manager() + instrument(manager) + converse(cli([{"type": "image", "source": {"type": "url", "url": "http://127.0.0.1:9/x.png"}}, + {"type": "image", "source": {"type": "file", "file_id": "file_1"}}, + {"type": "image", "source": {"type": "base64", "media_type": "text/plain", "data": b64(SHOT)}}])) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny(f01, instrument, cli, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + converse(cli([image(SHOT)])) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing_and_the_conversation_is_unchanged(f01, instrument, cli): + instrument(create_attachment_manager(config=AttachmentConfig())) + messages = converse(cli([image(SHOT)])) + assert f01.uploads == [] + blocks = [m for m in messages if type(m).__name__ == "UserMessage"][0].content + assert blocks[0].content[0]["source"]["data"] == b64(SHOT) diff --git a/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py new file mode 100644 index 000000000..de6847722 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py @@ -0,0 +1,114 @@ +"""Real firecrawl-py v2 clients against a loopback Firecrawl API -> loopback F01.""" + +import asyncio +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("firecrawl") +pytest.importorskip("traceai_firecrawl") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from firecrawl import AsyncFirecrawl, Firecrawl # noqa: E402 +from traceai_firecrawl import FirecrawlInstrumentor # noqa: E402 + +from _harness import data_uri, flush, png, refs, span_named # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "firecrawl" / "tests")) +from _firecrawl_fake import FakeFirecrawl # noqa: E402 + +SHOT = png(seed=61) +ACTION_SHOT = png(seed=62) +KEY = "fc-SYNTHETIC-PLACEHOLDER" + + +@pytest.fixture +def fake(): + server = FakeFirecrawl() + yield server + server.close() + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + FirecrawlInstrumentor().uninstrument() + + +def page(screenshot, actions=None): + data = {"markdown": "body", "screenshot": screenshot, "metadata": {"sourceURL": "https://example.com"}} + if actions is not None: + data["actions"] = {"screenshots": actions} + return {"success": True, "data": data} + + +def test_inline_screenshots_upload_and_hosted_urls_are_never_fetched(f01, instrument, fake): + manager = f01.manager() + exporter = instrument(manager) + hosted = fake.origin + "/hosted/screenshot.png" + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT), [data_uri("image/jpeg", ACTION_SHOT), hosted])) + document = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert document.screenshot == data_uri("image/png", SHOT) # the SDK result is unchanged + assert sorted((u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads) == [ + ("image/jpeg", "output", "tool_result"), ("image/png", "output", "tool_result")] + assert sorted(f01.payloads) == sorted([SHOT, ACTION_SHOT]) + assert fake.calls_to("GET", "/hosted/screenshot.png") == 0 + assert len(refs(span_named(exporter, "firecrawl.scrape"))) == 2 + manager.shutdown(5) + + +def test_hosted_only_screenshot_uploads_nothing(f01, instrument, fake): + manager = f01.manager() + instrument(manager) + fake.routes[("POST", "/v2/scrape")] = (200, page(fake.origin + "/hosted/x.png")) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert f01.uploads == [] and fake.calls_to("GET", "/hosted/x.png") == 0 + manager.shutdown(5) + + +def test_async_client_and_crawl_documents(f01, instrument, fake): + manager = f01.manager() + instrument(manager) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + + async def go(): + return await AsyncFirecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape( + "https://example.com", formats=["screenshot"]) + + asyncio.run(go()) + flush(manager) + assert f01.payloads == [SHOT] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_inputs", "pii_redaction"]) +def test_local_restrictions_deny(f01, instrument, fake, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, instrument, fake): + instrument(create_attachment_manager(config=AttachmentConfig())) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_mcp_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_attachments.py new file mode 100644 index 000000000..fd5a3edc3 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_mcp_attachments.py @@ -0,0 +1,225 @@ +"""Real ``mcp`` ClientSession over stdio -> wrap_mcp_client -> TLS loopback F01 server.""" + +import asyncio +import json +import sys +import threading +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") +traceai_mcp = pytest.importorskip("traceai_mcp") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from mcp import ClientSession, StdioServerParameters # noqa: E402 +from mcp.client.stdio import stdio_client # noqa: E402 + +from _harness import PDF, flush, no_bytes, refs, span_named, uploaded # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _mcp_media_server as server # noqa: E402 + +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" + + +def run(spans, manager, calls, **wrap): + """Open a real stdio session, wrap it, and run ``calls(session)``.""" + provider, _ = spans + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + traced = traceai_mcp.wrap_mcp_client(session, tracer_provider=provider, attachments=manager, **wrap) + return await calls(traced) + + return asyncio.run(go()) + + +def test_tool_image_audio_and_embedded_blob_upload_with_tool_identity(f01, spans): + manager = f01.manager() + _, exporter = spans + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + result = await session.call_tool("report", {}) + reads = await session.call_tool("reads", {}) + return result, reads + + result, reads = run(spans, manager, calls) + flush(manager) + assert sorted(u["mime_type"] for u in f01.uploads) == ["application/pdf", "audio/wav", "image/png"] + assert {(u["direction"], u["purpose"]) for u in f01.uploads} == {("output", "tool_result")} + assert server.png(1) in f01.payloads and server.WAV in f01.payloads and PDF in f01.payloads + # The resource_link URI was never dereferenced. + assert json.loads(reads.content[0].text)["secret"] == 0 + # The SDK result is unchanged. + assert result.content[0].resource.blob == server.b64(PDF) + shot = [s for s in exporter.get_finished_spans() if s.name == "mcp.tools/call"] + assert len(shot) == 4 + assert sum(len(refs(s)) for s in shot) == 3 + assert {r["role"] for s in shot for r in refs(s)} == {"tool"} + no_bytes(exporter, server.png(1)) + manager.shutdown(5) + + +def test_resources_read_blob_uploads_and_text_is_ignored(f01, spans): + manager = f01.manager() + + async def calls(session): + return await session.read_resource("file:///shot.png") + + result = run(spans, manager, calls) + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["image/png"] + assert f01.payloads == [server.png(5)] + assert result.contents[0].blob == server.b64(server.png(5)) + manager.shutdown(5) + + +def test_relabeled_unknown_malformed_and_oversized_media_upload_nothing(f01, spans): + manager = f01.manager() + + async def calls(session): + for name in ("relabeled", "malformed", "big"): + await session.call_tool(name, {}) + + run(spans, manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_part_budget_caps_uploads_per_result(f01, spans): + manager = f01.manager() + + async def calls(session): + await session.call_tool("many", {"count": 70}) + + run(spans, manager, calls) + flush(manager) + assert len(f01.uploads) == 64 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_messages", "hide_output_text", + "pii_redaction"]) +def test_local_hide_flags_and_pii_deny_every_tool_media(f01, spans, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + + run(spans, manager, calls, trace_config=TraceConfig(**{flag: True})) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_hide_input_images_denies_images_but_not_audio(f01, spans): + manager = f01.manager() + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + + run(spans, manager, calls, trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["audio/wav"] + manager.shutdown(5) + + +def test_env_hide_flag_denies(f01, spans, monkeypatch): + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + manager = f01.manager() + + async def calls(session): + await session.call_tool("screenshot", {}) + + run(spans, manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_disabled_manager_and_unsampled_span_upload_nothing_and_start_no_threads(f01, spans): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + + before = {t.name for t in threading.enumerate()} + disabled = create_attachment_manager(config=AttachmentConfig()) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + result = run(spans, disabled, calls) + assert result.content[1].data == server.b64(server.png(1)) + assert not {t.name for t in threading.enumerate()} - before + manager = f01.manager() + unsampled = TracerProvider(sampler=ALWAYS_OFF) + run((unsampled, None), manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_provider_manager_is_the_default_and_rewrapping_is_idempotent(f01, spans): + manager = f01.manager() + provider, exporter = spans + provider.attachment_manager = manager + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + traced = traceai_mcp.wrap_mcp_client(session, tracer_provider=provider) + assert traceai_mcp.wrap_mcp_client(traced) is traced + await traced.call_tool("screenshot", {}) + + asyncio.run(go()) + flush(manager) + assert len(f01.uploads) == 1 + assert len([s for s in exporter.get_finished_spans() if s.name == "mcp.tools/call"]) == 1 + manager.shutdown(5) + + +def test_tool_errors_keep_the_sdk_exception_and_end_the_span(f01, spans): + manager = f01.manager() + _, exporter = spans + + async def calls(session): + with pytest.raises(Exception): + await session.read_resource("file:///missing.png") + + run(spans, manager, calls) + span = span_named(exporter, "mcp.resources/read") + assert span.status.status_code.name == "ERROR" + manager.shutdown(5) + + +def test_capture_mcp_result_is_preflight_first_and_never_raises(f01, spans): + manager = f01.manager() + provider, _ = spans + reads = [] + + class Part(dict): + def get(self, key, default=None): + if key == "data": + reads.append(key) + return super().get(key, default) + + tracer = provider.get_tracer("t") + with tracer.start_as_current_span("denied") as span: + traceai_mcp.capture_mcp_result(manager, span, {"content": [Part(type="image", data="AAAA", mimeType="image/png")]}, + trace_config=TraceConfig(hide_outputs=True)) + assert reads == [] + assert traceai_mcp.capture_mcp_result(manager, span, object()) == {} + assert traceai_mcp.capture_mcp_result(None, span, {"content": []}) == {} + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py new file mode 100644 index 000000000..b862ad56e --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py @@ -0,0 +1,102 @@ +"""Real mcp-use MCPAgent (scripted LangChain chat model) + stdio MCP media server -> loopback F01.""" + +import asyncio +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("mcp_use") +pytest.importorskip("traceai_mcp_use") +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "mcp-use" / "tests")) +import _mcp_use_support as support # noqa: E402 (also turns mcp-use telemetry off) + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_mcp_use import FutureAGICallback # noqa: E402 + +from _harness import flush, refs # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _mcp_media_server as server # noqa: E402 + +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" + + +def run(callback, tool, args=None): + from mcp_use import MCPAgent, MCPClient + + async def go(): + client = MCPClient.from_dict({"mcpServers": {"media": {"command": sys.executable, "args": [str(SERVER)]}}}) + try: + agent = MCPAgent(llm=support.ChatFake(script=[support.tool_call(tool, args or {}, "call_media_1"), + support.answer()]), + client=client, callbacks=[callback]) + return await agent.run(support.PROMPT) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +def tool_span(exporter, name): + return [s for s in exporter.get_finished_spans() if s.name == "execute_tool " + name][-1] + + +def test_tool_image_audio_and_blob_upload_on_the_tool_span(f01, spans): + manager = f01.manager() + provider, exporter = spans + callback = FutureAGICallback(tracer_provider=provider, attachments=manager) + assert run(callback, "screenshot") == support.ANSWER + run(callback, "voice") + run(callback, "report") + flush(manager) + assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ + ("application/pdf", "output", "tool_result", "call_media_1"), + ("audio/wav", "output", "tool_result", "call_media_1"), + ("image/png", "output", "tool_result", "call_media_1")] + assert server.png(1) in f01.payloads + span = tool_span(exporter, "screenshot") + assert len(refs(span)) == 1 and refs(span)[0]["role"] == "tool" + assert server.b64(server.png(1))[:40] not in str(span.attributes) + manager.shutdown(5) + + +def test_provider_manager_is_the_default(f01, spans): + manager = f01.manager() + provider, _ = spans + provider.attachment_manager = manager + run(FutureAGICallback(tracer_provider=provider), "screenshot") + flush(manager) + assert len(f01.uploads) == 1 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "pii_redaction"]) +def test_local_restrictions_deny(f01, spans, flag): + manager = f01.manager() + provider, _ = spans + before = len(f01.after_bootstrap()) + run(FutureAGICallback(tracer_provider=provider, config=TraceConfig(**{flag: True}), attachments=manager), + "screenshot") + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_malformed_and_relabeled_media_upload_nothing(f01, spans): + manager = f01.manager() + provider, _ = spans + callback = FutureAGICallback(tracer_provider=provider, attachments=manager) + run(callback, "relabeled") + run(callback, "malformed") + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, spans): + provider, _ = spans + run(FutureAGICallback(tracer_provider=provider, attachments=create_attachment_manager(config=AttachmentConfig())), + "screenshot") + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py new file mode 100644 index 000000000..b3520be7e --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py @@ -0,0 +1,213 @@ +"""Real OpenAI Agents SDK Runner (scripted local Model, real ComputerTool and function tools) -> loopback F01.""" + +import asyncio +import json +from importlib.metadata import version + +import pytest + +agents = pytest.importorskip("agents") +pytest.importorskip("traceai_openai_agents") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from packaging.version import Version # noqa: E402 +from traceai_openai_agents import OpenAIAgentsInstrumentor # noqa: E402 + +from _harness import data_uri, flush, png, refs # noqa: E402 + +SDK = Version(version("openai-agents")) +COMPUTER_SPAN = SDK >= Version("0.10.0") # the SDK records computer calls on a function span from 0.10.0 +TYPED_IMAGE = SDK >= Version("0.4.0") # ToolOutputImage exists from 0.4.0 +SHOT = png(seed=41) +TOOL_IMAGE = png(seed=42) + + +class ScriptedModel(agents.Model): + """Replays model outputs; no network.""" + + def __init__(self, turns): + self.turns = list(turns) + self.inputs = [] + + async def get_response(self, *args, **kwargs): + from agents.usage import Usage + + self.inputs.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + output = self.turns.pop(0) + fields = dict(output=output, usage=Usage(), response_id=None) + try: + return agents.ModelResponse(**fields) + except TypeError: + return agents.ModelResponse(output=output, usage=Usage(), response_id=None, request_id=None) + + def stream_response(self, *args, **kwargs): + raise NotImplementedError + + +def message(text="done"): + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + return ResponseOutputMessage(id="m1", type="message", role="assistant", status="completed", + content=[ResponseOutputText(type="output_text", text=text, annotations=[])]) + + +def computer_call(call_id="call_shot"): + from openai.types.responses import ResponseComputerToolCall + + return ResponseComputerToolCall.model_validate({ + "id": "cc1", "type": "computer_call", "call_id": call_id, "status": "completed", + "pending_safety_checks": [], "action": {"type": "screenshot"}}) + + +def function_call(name, call_id="call_fn"): + from openai.types.responses import ResponseFunctionToolCall + + return ResponseFunctionToolCall(id="fc1", type="function_call", call_id=call_id, name=name, arguments="{}") + + +class Screen(agents.AsyncComputer): + shots = 0 + + @property + def environment(self): + return "browser" + + @property + def dimensions(self): + return (1024, 768) + + async def screenshot(self): + Screen.shots += 1 + return data_uri("image/png", SHOT).split(",", 1)[1] + + async def click(self, x, y, button): + return None + + async def double_click(self, x, y): + return None + + async def scroll(self, x, y, scroll_x, scroll_y): + return None + + async def type(self, text): + return None + + async def wait(self): + return None + + async def move(self, x, y): + return None + + async def keypress(self, keys): + return None + + async def drag(self, path): + return None + + +@pytest.fixture +def traced(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + state = {} + + def install(manager, config=None): + OpenAIAgentsInstrumentor().instrument(tracer_provider=provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + agents.set_trace_processors([]) + provider.shutdown() + + +def run_computer(manager, install, config=None): + exporter = install(manager, config) + model = ScriptedModel([[computer_call()], [message()]]) + agent = agents.Agent(name="operator", model=model, tools=[agents.ComputerTool(computer=Screen())]) + result = asyncio.run(agents.Runner.run(agent, "take a screenshot")) + return exporter, model, result + + +def function_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "TOOL"] + + +def test_computer_screenshot_uploads_on_the_computer_tool_span(f01, traced): + manager = f01.manager() + exporter, model, result = run_computer(manager, traced) + flush(manager) + assert result.final_output == "done" + if not COMPUTER_SPAN: + assert f01.uploads == [] # no function span records the computer call before 0.10.0 + return + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "computer_screenshot")] + assert f01.payloads == [SHOT] + tool = [s for s in function_spans(exporter) if refs(s)] + assert len(tool) == 1 + output = tool[0].attributes["output.value"] + assert output.startswith("fi-attachment:") and output[len("fi-attachment:"):] == refs(tool[0])[0]["occurrence_id"] + # The SDK still sent the original screenshot to the model. + sent = json.dumps(model.inputs[-1], default=str) + assert data_uri("image/png", SHOT) in sent + manager.shutdown(5) + + +@pytest.mark.skipif(not TYPED_IMAGE, reason="ToolOutputImage needs openai-agents 0.4.0") +def test_function_tool_typed_image_uploads_and_url_or_file_id_never_fetch(f01, traced): + manager = f01.manager() + exporter = traced(manager) + + @agents.function_tool + def chart() -> list: + """Draw a chart.""" + return [agents.ToolOutputImage(image_url=data_uri("image/png", TOOL_IMAGE)), + agents.ToolOutputImage(image_url="https://example.invalid/chart.png"), + agents.ToolOutputImage(file_id="file-123")] + + model = ScriptedModel([[function_call("chart")], [message()]]) + agent = agents.Agent(name="analyst", model=model, tools=[chart]) + asyncio.run(agents.Runner.run(agent, "chart it")) + flush(manager) + assert [(u["mime_type"], u["purpose"]) for u in f01.uploads] == [("image/png", "tool_result")] + assert f01.payloads == [TOOL_IMAGE] + span = [s for s in function_spans(exporter) if refs(s)][0] + assert "https://example.invalid/chart.png" in span.attributes["output.value"] + assert data_uri("image/png", TOOL_IMAGE) not in span.attributes["output.value"] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny_the_screenshot(f01, traced, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + run_computer(manager, traced, TraceConfig(**{flag: True})) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_keeps_the_old_attributes(f01, traced): + exporter, _, result = run_computer(create_attachment_manager(config=AttachmentConfig()), traced) + assert result.final_output == "done" and f01.uploads == [] + if COMPUTER_SPAN: + outputs = [s.attributes.get("output.value") for s in function_spans(exporter)] + assert data_uri("image/png", SHOT) in outputs + + +def test_sensitive_data_off_records_and_uploads_nothing(f01, traced): + manager = f01.manager() + traced(manager) + model = ScriptedModel([[computer_call()], [message()]]) + agent = agents.Agent(name="operator", model=model, tools=[agents.ComputerTool(computer=Screen())]) + asyncio.run(agents.Runner.run(agent, "shot", run_config=agents.RunConfig(trace_include_sensitive_data=False))) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_smolagents_attachments.py b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py new file mode 100644 index 000000000..f2fc93f5a --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py @@ -0,0 +1,164 @@ +"""Real smolagents (OpenAIServerModel on a loopback OpenAI double, real Tool and agent loop) -> loopback F01.""" + +import io + +import pytest + +smolagents = pytest.importorskip("smolagents") +PIL_Image = pytest.importorskip("PIL.Image") +pytest.importorskip("openai") +pytest.importorskip("traceai_smolagents") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_smolagents import SmolagentsInstrumentor # noqa: E402 + +from _harness import flush, harness # noqa: E402 + +provider = harness.provider # the loopback OpenAI-compatible API (fixture) + + +def picture(seed=0, size=(8, 6)): + image = PIL_Image.new("RGB", size, (seed % 256, 10, 20)) + return image + + +def png_of(image): + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + SmolagentsInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + SmolagentsInstrumentor().uninstrument() + + +def model(provider): + return smolagents.OpenAIServerModel(model_id="m", api_base=provider.url + "/v1", api_key="SYNTHETIC_KEY") + + +def user_message(*images): + content = [{"type": "text", "text": "look"}] + [{"type": "image", "image": image} for image in images] + if not hasattr(smolagents.OpenAIServerModel, "generate"): + return {"role": "user", "content": content} # releases before generate() take plain dicts + from smolagents.models import ChatMessage, MessageRole + + return ChatMessage(role=MessageRole.USER, content=content) + + +def call(m, messages): + method = getattr(m, "generate", None) or m.__call__ + return method(messages) + + +def test_model_input_images_upload_once_on_the_model_span(f01, instrument, provider): + manager = f01.manager() + exporter = instrument(manager) + shot = picture(1) + m = model(provider) + call(m, [user_message(shot)]) + call(m, [user_message(shot)]) # memory replay of the same observation image + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "input", "model_input")] + assert f01.payloads == [png_of(shot)] + # The model still received the image. + assert any(b"data:image/png;base64," in raw for _, _, raw in provider.requests) + llm = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "LLM"] + assert llm and harness.refs(llm[0]) + manager.shutdown(5) + + +def test_tool_returned_images_upload_on_the_tool_span(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + raw, wrapped_image = picture(2), picture(3) + + class Snap(smolagents.Tool): + name = "snap" + description = "Take a picture." + inputs = {"which": {"type": "string", "description": "which"}} + output_type = "image" + + def forward(self, which): + if which == "raw": + return raw + return smolagents.AgentImage(wrapped_image) + + tool = Snap() + tool(which="raw", sanitize_inputs_outputs=False) + tool(which="agent", sanitize_inputs_outputs=False) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] * 2 + assert sorted(f01.payloads) == sorted([png_of(raw), png_of(wrapped_image)]) + tools = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "TOOL"] + assert sum(len(harness.refs(s)) for s in tools) == 2 + manager.shutdown(5) + + +def test_file_backed_and_audio_values_are_never_read(f01, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + picture(4).save(path) + + class Files(smolagents.Tool): + name = "files" + description = "Return files." + inputs = {"which": {"type": "string", "description": "which"}} + output_type = "image" + + def forward(self, which): + if which == "agent-path": + return smolagents.AgentImage(str(path)) + return PIL_Image.open(path) + + tool = Files() + tool(which="agent-path", sanitize_inputs_outputs=False) + tool(which="opened", sanitize_inputs_outputs=False) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny_model_images(f01, instrument, provider, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + call(model(provider), [user_message(picture(5))]) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_agent_run_with_task_images_uploads_each_image_once(f01, instrument, provider): + manager = f01.manager() + instrument(manager) + agent = smolagents.ToolCallingAgent(tools=[], model=model(provider), max_steps=1) + try: + agent.run("describe", images=[picture(6)]) + except Exception: + pass # the double never calls a tool; the loop's own errors are not under test + flush(manager) + assert [u["purpose"] for u in f01.uploads] == ["model_input"] + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, instrument, provider): + instrument(create_attachment_manager(config=AttachmentConfig())) + call(model(provider), [user_message(picture(7))]) + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_tool_media.py b/python/tests/attachments_mcp_agents/test_tool_media.py new file mode 100644 index 000000000..866fbf82c --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_tool_media.py @@ -0,0 +1,108 @@ +"""The shared tool-media helper: deny-wins folding, preflight before read, budgets, dedup, ownership.""" + +import pytest + +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy, create_attachment_manager +from fi_instrumentation.attachments import tool_media +from opentelemetry import context as otel_context + +from _harness import b64, data_uri, flush, png, refs + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_messages", "hide_input_text", "hide_outputs", + "hide_output_messages", "hide_output_text", "pii_redaction"]) +def test_tool_policy_folds_input_and_output_restrictions(flag): + policy = tool_media.tool_capture_policy(TraceConfig(**{flag: True})) + assert policy.capture_content is False and policy.model_input and policy.contains_output + + +def test_tool_policy_images_and_errors(): + assert tool_media.tool_capture_policy(TraceConfig(hide_input_images=True), image=True).capture_content is False + assert tool_media.tool_capture_policy(TraceConfig(hide_input_images=True)).capture_content is True + assert tool_media.tool_capture_policy(TraceConfig(hide_error_details=True), error=True).capture_content is False + base = CapturePolicy(record_outputs=False) + assert tool_media.tool_capture_policy(None, base).record_outputs is False + + +def test_owner_context_disables_capture(f01, spans): + manager = f01.manager() + provider, _ = spans + token = otel_context.attach(tool_media.with_ownership()) + try: + with provider.get_tracer("t").start_as_current_span("owned") as span: + media = tool_media.ToolMedia(manager, span, "test") + assert not media.usable + assert tool_media.tool_capture_policy().requested is False + finally: + otel_context.detach(token) + manager.shutdown(5) + + +def test_value_is_read_only_after_admission_and_only_once(f01, spans): + manager = f01.manager() + provider, _ = spans + reads = [] + + def read(): + reads.append(1) + return b64(png(1)) + + with provider.get_tracer("t").start_as_current_span("tool") as span: + denied = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_outputs=True)) + assert denied.base64("a", read, "image/png", purpose="tool_result", expect="image") is None + assert reads == [] + media = tool_media.ToolMedia(manager, span, "test") + first = media.base64("b", read, "image/png", purpose="tool_result", expect="image") + again = tool_media.ToolMedia(manager, span, "test").base64("b", read, "image/png", purpose="tool_result") + assert first is not None and again == first and reads == [1] + assert media.base64("c", read, "audio/wav", purpose="tool_result", expect="image") is None + assert media.base64("d", read, "image/gif", purpose="tool_result") is None + assert reads == [1] + flush(manager) + assert len(f01.uploads) == 1 and len(refs(span)) == 1 + manager.shutdown(5) + + +def test_budget_and_part_cap(f01, spans): + manager = f01.manager() + provider, _ = spans + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", budget=200, max_parts=2) + image = png(2, width=4, height=4) + assert len(image) < 200 + assert media.base64("1", lambda: b64(image), "image/png", purpose="tool_result") is not None + assert media.remaining == 200 - len(image) + assert media.data_uri("2", lambda: data_uri("image/png", png(3, 40, 40)), ("image/png",), + purpose="tool_result") is None # over the remaining budget + assert media.base64("3", lambda: b64(png(4)), "image/png", purpose="tool_result") is None # part cap + flush(manager) + assert len(f01.uploads) == 1 + manager.shutdown(5) + + +def test_disabled_and_missing_managers_are_inert(spans): + provider, _ = spans + with provider.get_tracer("t").start_as_current_span("tool") as span: + for manager in (None, create_attachment_manager(config=AttachmentConfig())): + media = tool_media.ToolMedia(manager, span, "test") + assert not media.usable + assert media.base64("x", lambda: pytest.fail("read"), "image/png", purpose="tool_result") is None + + +def test_pil_images_are_encoded_only_when_in_memory(f01, spans, tmp_path): + Image = pytest.importorskip("PIL.Image") + manager = f01.manager() + provider, _ = spans + in_memory = Image.new("RGB", (4, 3), (1, 2, 3)) + path = tmp_path / "x.png" + in_memory.save(path) + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test") + assert media.image("file", Image.open(path), purpose="tool_result") is None + assert media.image("mem", in_memory, purpose="tool_result") is not None + assert media.image("huge", Image.new("L", (4096, 4096)), purpose="tool_result") is None + assert media.image("not-an-image", b"\x89PNG", purpose="tool_result") is None + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["image/png"] + manager.shutdown(5) From 3ed3321c804587bdc0eef04dab6b8e1be1d6ce94 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:17:10 -0800 Subject: [PATCH 04/16] test(python): MCP/agent attachment suites, CI workflow and README sections Co-Authored-By: Claude Opus 5.5 --- .../attachments-python-mcp-agents-tests.yml | 106 ++++++++++++++++++ python/frameworks/claude-agent-sdk/README.md | 25 +++++ python/frameworks/firecrawl/README.md | 23 ++++ python/frameworks/mcp-use/README.md | 24 ++++ .../mcp-use/traceai_mcp_use/_attachments.py | 3 +- python/frameworks/mcp/README.md | 38 +++++++ python/frameworks/openai-agents/README.md | 30 +++++ python/frameworks/smolagents/README.md | 29 +++++ .../test_mcp_use_attachments.py | 4 +- .../attachments_mcp_agents/test_tool_media.py | 8 +- 10 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/attachments-python-mcp-agents-tests.yml diff --git a/.github/workflows/attachments-python-mcp-agents-tests.yml b/.github/workflows/attachments-python-mcp-agents-tests.yml new file mode 100644 index 000000000..2ef4e204d --- /dev/null +++ b/.github/workflows/attachments-python-mcp-agents-tests.yml @@ -0,0 +1,106 @@ +name: Python MCP and agent attachments + +# Real MCP, mcp-use, OpenAI Agents SDK, Claude Agent SDK, smolagents and +# firecrawl-py releases (declared floor and latest) through the Python +# instrumentors to a TLS loopback F01 attachment server. Outbound network is +# disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/mcp/**' + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/openai-agents/**' + - 'python/frameworks/claude-agent-sdk/**' + - 'python/frameworks/smolagents/**' + - 'python/frameworks/firecrawl/**' + - 'python/tests/attachments_mcp_agents/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-mcp-agents-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/mcp/**' + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/openai-agents/**' + - 'python/frameworks/claude-agent-sdk/**' + - 'python/frameworks/smolagents/**' + - 'python/frameworks/firecrawl/**' + - 'python/tests/attachments_mcp_agents/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-mcp-agents-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +env: + MCP_USE_ANONYMIZED_TELEMETRY: 'false' + MCP_USE_LANGFUSE: 'false' + MCP_USE_LAMINAR: 'false' + +jobs: + attachments: + name: ${{ matrix.package }} attachments (${{ matrix.tier }}) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - {package: core, tier: latest, python: '3.12', path: '', suite: test_tool_media.py, sdks: 'pillow'} + - {package: mcp, tier: floor, python: '3.11', path: mcp, suite: test_mcp_attachments.py, sdks: 'mcp==1.30.0'} + - {package: mcp, tier: latest, python: '3.12', path: mcp, suite: test_mcp_attachments.py, sdks: 'mcp==2.3.0'} + - {package: mcp-use, tier: floor, python: '3.11', path: mcp-use, suite: test_mcp_use_attachments.py, + sdks: 'mcp-use==1.7.1 langchain-core==1.6.12'} + - {package: mcp-use, tier: latest, python: '3.12', path: mcp-use, suite: test_mcp_use_attachments.py, + sdks: 'mcp-use==1.7.1'} + # 0.2.6 is the declared floor; computer calls get a function span from 0.10.0. + - {package: openai-agents, tier: floor, python: '3.11', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.2.6'} + - {package: openai-agents, tier: computer-span, python: '3.12', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.10.0'} + - {package: openai-agents, tier: latest, python: '3.12', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.24.0'} + - {package: claude-agent-sdk, tier: floor, python: '3.11', path: claude-agent-sdk, + suite: test_claude_agent_sdk_attachments.py, sdks: 'claude-agent-sdk==0.1.0'} + - {package: claude-agent-sdk, tier: latest, python: '3.12', path: claude-agent-sdk, + suite: test_claude_agent_sdk_attachments.py, sdks: 'claude-agent-sdk==0.2.180'} + - {package: smolagents, tier: floor, python: '3.11', path: smolagents, suite: test_smolagents_attachments.py, + sdks: 'smolagents==1.12.0 openai'} + - {package: smolagents, tier: latest, python: '3.12', path: smolagents, suite: test_smolagents_attachments.py, + sdks: 'smolagents==1.26.0 openai'} + - {package: firecrawl, tier: floor, python: '3.11', path: firecrawl, suite: test_firecrawl_attachments.py, + sdks: 'firecrawl-py==4.46.2'} + - {package: firecrawl, tier: latest, python: '3.12', path: firecrawl, suite: test_firecrawl_attachments.py, + sdks: 'firecrawl-py==4.55.0'} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install the source integration and the selected SDK + env: + SDKS: ${{ matrix.sdks }} + PKG_PATH: ${{ matrix.path }} + run: | + python -m venv .venv + extra="" + if [ -n "$PKG_PATH" ]; then extra="-e python/frameworks/$PKG_PATH"; fi + # shellcheck disable=SC2086 + .venv/bin/pip install -e python $extra opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' packaging $SDKS + .venv/bin/pip freeze | grep -iE '^(mcp|mcp-use|openai-agents|openai|claude-agent-sdk|smolagents|firecrawl-py|pillow|langchain-core)==' || true + - name: Run with outbound network disabled + env: + SUITE: ${{ matrix.suite }} + run: | + sudo --preserve-env=MCP_USE_ANONYMIZED_TELEMETRY,MCP_USE_LANGFUSE,MCP_USE_LAMINAR unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + PYTHONPATH=. "$1" -m pytest "tests/attachments_mcp_agents/$3" -q -p no:cacheprovider -o addopts="" \ + -o faulthandler_timeout=120 + ' _ "$PWD/.venv/bin/python" "$PWD" "$SUITE" diff --git a/python/frameworks/claude-agent-sdk/README.md b/python/frameworks/claude-agent-sdk/README.md index 33a9fc14f..5f74be457 100644 --- a/python/frameworks/claude-agent-sdk/README.md +++ b/python/frameworks/claude-agent-sdk/README.md @@ -183,3 +183,28 @@ pytest tests/ -v ## License MIT + +## Tool-result media (attachments) + +Pass the shared manager: `ClaudeAgentInstrumentor().instrument(tracer_provider=..., attachments=manager)`. +The base64 `image` and PDF `document` blocks of a `ToolResultBlock` are +uploaded from the original SDK message, before any text flattening or +truncation, as `tool_result` occurrences with the `tool_use_id`. They belong +to the tool's span: while it is open they are projected on it; once the +PostToolUse hook ended it, they keep its exact span context as their parent. +Media bytes never reach `tool.output` or the conversation output text. The +SDK drops top-level user image blocks while parsing, so prompt images are not +captured. URL and file sources are never read. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index 03d710941..f68184784 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -114,3 +114,26 @@ Direct `FirecrawlClient` and `AsyncFirecrawlClient` instances look methods up on ## Watchers `Firecrawl.watcher()` polls `get_crawl_status` from its own thread. Each poll produces a separate root span; a swallowed poll error still appears as ERROR. `AsyncFirecrawl.watcher()` polls in the iterating task, so its spans inherit that task's current parent. + +## Screenshots and media (attachments) + +Pass the shared manager: `FirecrawlInstrumentor().instrument(tracer_provider=..., attachments=manager)`. +For `scrape`, `crawl`, `batch_scrape` and their status calls, a returned +Document's `screenshot`, `actions.screenshots`, `audio` and `video` are +uploaded on the operation span when they are inline `data:` URIs (up to 16 +documents and 8 MiB per result). Firecrawl Cloud returns hosted URLs for +these: they are never fetched, so nothing is uploaded for them. No extra +request, crawl or browser session is started. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 6c0c1b30a..450fede1e 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -369,3 +369,27 @@ path is exported either way. The example test runs `examples/agent_with_tools.py` as its own process, once alone and once with `traceai_mcp` instrumented. The suite turns off mcp-use's usage telemetry. + +## Tool media (attachments) + +`FutureAGICallback(attachments=manager)` (default: the tracer provider's +`attachment_manager`) uploads the typed base64 blocks mcp-use 1.7 makes from +an MCP tool result: `image` and `file` (inline resource blob), on the tool +span as `tool_result` occurrences with the tool call id (an `audio` block is +captured when it reaches the callback as a typed block; LangChain 1.x turns +audio tool content into a string first, so nothing is uploaded for it). At +most 64 blocks and 256 KiB of decoded bytes per result. +`attachment_capture=` only adds restrictions. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py index 7f06da44b..00c864136 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py @@ -20,7 +20,8 @@ "operations": ("tool.output",), "sources": ("inline_base64",), "content_types": ("image", "audio", "file"), - "unsupported": ("url", "resource_uri", "resource_link", "local_path"), + # LangChain stringifies tool content it does not know as a block (audio), so no typed block reaches callbacks. + "unsupported": ("url", "resource_uri", "resource_link", "local_path", "audio_stringified_by_langchain"), }) _EMPTY: Mapping[str, Any] = MappingProxyType({}) diff --git a/python/frameworks/mcp/README.md b/python/frameworks/mcp/README.md index 8f1a00c87..508874f25 100644 --- a/python/frameworks/mcp/README.md +++ b/python/frameworks/mcp/README.md @@ -45,3 +45,41 @@ enabled is attached only if the reader is enabled when it consumes the item. An already-running request retains the context under which it started. Native MCP SDK tracing remains independent and can continue when this instrumentor is off, particularly with MCP 2.x. + +## Tool and resource media (attachments) + +`MCPInstrumentor` propagates context only and never captures media. To upload +the images, audio and inline resource blobs a server returns, wrap the client +session: + +```python +from traceai_mcp import wrap_mcp_client + +session = wrap_mcp_client(session, tracer_provider=trace_provider) +result = await session.call_tool("screenshot", {}) # unchanged SDK result +``` + +Each `call_tool` / `read_resource` gets an `mcp.tools/call` / +`mcp.resources/read` CLIENT span. `ImageContent.data`, `AudioContent.data` +and `BlobResourceContents.blob` (in an `EmbeddedResource` or a +`resources/read` result) are uploaded as `tool_result` occurrences: at most 64 +parts and 256 KiB of decoded bytes per result. `resource_link` blocks, +resource URIs and `structuredContent` are never read. A MIME of another +modality than the block type, or outside the attachment MIME list, uploads +nothing. `attachments=` overrides the provider's manager; `trace_config=` and +`attachment_capture=` only add restrictions. Wrapping a wrapped session +returns it unchanged. `capture_mcp_result(manager, span, result, operation)` +is the same capture for an existing span. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/openai-agents/README.md b/python/frameworks/openai-agents/README.md index bba264856..2712bfe36 100644 --- a/python/frameworks/openai-agents/README.md +++ b/python/frameworks/openai-agents/README.md @@ -59,3 +59,33 @@ print(result.final_output) ``` +## Tool media and computer-use screenshots (attachments) + +Pass the shared manager: `OpenAIAgentsInstrumentor().instrument(tracer_provider=..., attachments=manager)` +(`traceai.init` passes it for you). On the tool's function span, before +`output.value` is written: + +- the computer tool's screenshot (`computer_screenshot`), which the SDK records + on the `computer` function span from openai-agents 0.10.0; +- `ToolOutputImage` / `{"type": "image", "image_url": "data:..."}` and + `ToolOutputFileContent` with `file_data` (openai-agents 0.4.0+; MCP image + results become these from 0.8.0), as `tool_result`. + +Admitted values become `fi-attachment:` in `output.value`. +Nothing is recorded or uploaded when `RunConfig.trace_include_sensitive_data` +is off. Model-call input media is left to the OpenAI instrumentor's own span: +with both enabled, a screenshot is uploaded once as the tool output and once +as the next model input. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/smolagents/README.md b/python/frameworks/smolagents/README.md index f942a29d0..720973c55 100644 --- a/python/frameworks/smolagents/README.md +++ b/python/frameworks/smolagents/README.md @@ -80,3 +80,32 @@ manager_agent.run( ) ``` +## Image observations (attachments) + +Pass the shared manager: `SmolagentsInstrumentor().instrument(tracer_provider=..., attachments=manager)`. + +- Images smolagents hands to a model (`{"type": "image", "image": }` + blocks: task images and `ActionStep.observations_images`, such as a browser + screenshot a step callback stored) are uploaded on the model span as + `model_input`, once per image object (memory replays are not re-uploaded). + When the model span captured an image, provider instrumentors below it are + told the call's media is already owned. +- A tool's returned in-memory PIL image or `AgentImage` is uploaded on the + tool span as `tool_result`. + +Images are PNG-encoded only after the manager admitted them, up to +2048 x 2048 pixels. File-backed images that are not loaded yet, image URLs +and every `AgentAudio` are never read. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py index b862ad56e..458ceff2e 100644 --- a/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py +++ b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py @@ -43,17 +43,17 @@ def tool_span(exporter, name): return [s for s in exporter.get_finished_spans() if s.name == "execute_tool " + name][-1] -def test_tool_image_audio_and_blob_upload_on_the_tool_span(f01, spans): +def test_tool_image_and_blob_upload_on_the_tool_span(f01, spans): manager = f01.manager() provider, exporter = spans callback = FutureAGICallback(tracer_provider=provider, attachments=manager) assert run(callback, "screenshot") == support.ANSWER + # LangChain turns an audio-only tool result into a string before callbacks see it: no typed block, no upload. run(callback, "voice") run(callback, "report") flush(manager) assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ ("application/pdf", "output", "tool_result", "call_media_1"), - ("audio/wav", "output", "tool_result", "call_media_1"), ("image/png", "output", "tool_result", "call_media_1")] assert server.png(1) in f01.payloads span = tool_span(exporter, "screenshot") diff --git a/python/tests/attachments_mcp_agents/test_tool_media.py b/python/tests/attachments_mcp_agents/test_tool_media.py index 866fbf82c..d3125549f 100644 --- a/python/tests/attachments_mcp_agents/test_tool_media.py +++ b/python/tests/attachments_mcp_agents/test_tool_media.py @@ -46,7 +46,7 @@ def test_value_is_read_only_after_admission_and_only_once(f01, spans): def read(): reads.append(1) - return b64(png(1)) + return b64(png(seed=1)) with provider.get_tracer("t").start_as_current_span("tool") as span: denied = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_outputs=True)) @@ -69,13 +69,13 @@ def test_budget_and_part_cap(f01, spans): provider, _ = spans with provider.get_tracer("t").start_as_current_span("tool") as span: media = tool_media.ToolMedia(manager, span, "test", budget=200, max_parts=2) - image = png(2, width=4, height=4) + image = png(width=4, height=4, seed=2) assert len(image) < 200 assert media.base64("1", lambda: b64(image), "image/png", purpose="tool_result") is not None assert media.remaining == 200 - len(image) - assert media.data_uri("2", lambda: data_uri("image/png", png(3, 40, 40)), ("image/png",), + assert media.data_uri("2", lambda: data_uri("image/png", png(width=40, height=40, seed=3)), ("image/png",), purpose="tool_result") is None # over the remaining budget - assert media.base64("3", lambda: b64(png(4)), "image/png", purpose="tool_result") is None # part cap + assert media.base64("3", lambda: b64(png(seed=4)), "image/png", purpose="tool_result") is None # part cap flush(manager) assert len(f01.uploads) == 1 manager.shutdown(5) From ecfac0c51e4c617c9cb09351b75c58f544e7ceaa Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:27:56 -0800 Subject: [PATCH 05/16] test(python): fix fixtures for real SDK floors and the root requests mock Co-Authored-By: Claude Opus 5.5 --- .../workflows/attachments-python-mcp-agents-tests.yml | 2 +- .../traceai_claude_agent_sdk/_instrumentor.py | 10 +++++++--- python/tests/attachments_mcp_agents/conftest.py | 6 ++++++ .../test_claude_agent_sdk_attachments.py | 5 ++++- .../test_openai_agents_attachments.py | 8 +++++--- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/attachments-python-mcp-agents-tests.yml b/.github/workflows/attachments-python-mcp-agents-tests.yml index 2ef4e204d..89bf05ee6 100644 --- a/.github/workflows/attachments-python-mcp-agents-tests.yml +++ b/.github/workflows/attachments-python-mcp-agents-tests.yml @@ -61,7 +61,7 @@ jobs: - {package: openai-agents, tier: floor, python: '3.11', path: openai-agents, suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.2.6'} - {package: openai-agents, tier: computer-span, python: '3.12', path: openai-agents, - suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.10.0'} + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.10.0 openai==2.19.0'} - {package: openai-agents, tier: latest, python: '3.12', path: openai-agents, suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.24.0'} - {package: claude-agent-sdk, tier: floor, python: '3.11', path: claude-agent-sdk, diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py index f2154cbfa..74b3bff25 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py @@ -149,12 +149,16 @@ def _patch_claude_agent_sdk(self) -> None: self._original_client_class = claude_agent_sdk.ClaudeSDKClient # Create wrapped class + attachments = getattr(self, "_attachments", None) + media = {} if attachments is None else dict( + attachments=attachments, + trace_config=getattr(self, "_trace_config", None), + attachment_capture=getattr(self, "_attachment_capture", None), + ) wrapped_class = wrap_claude_sdk_client( self._original_client_class, self._tracer, - attachments=getattr(self, "_attachments", None), - trace_config=getattr(self, "_trace_config", None), - attachment_capture=getattr(self, "_attachment_capture", None), + **media, ) # Replace in module diff --git a/python/tests/attachments_mcp_agents/conftest.py b/python/tests/attachments_mcp_agents/conftest.py index 536f9cd48..41d06ea86 100644 --- a/python/tests/attachments_mcp_agents/conftest.py +++ b/python/tests/attachments_mcp_agents/conftest.py @@ -19,3 +19,9 @@ clean_env = harness.clean_env f01 = harness.f01 spans = harness.spans + + +@pytest.fixture(autouse=True) +def mock_all_http_requests(): + """Overrides python/tests/conftest.py: these suites talk to real loopback servers through requests.""" + yield None diff --git a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py index 12f832a7e..2b3f4dd86 100644 --- a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py +++ b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py @@ -41,6 +41,8 @@ def make(content, order="hooks-first", is_error=False): monkeypatch.setenv("FAKE_CLAUDE_RESULT", str(result)) monkeypatch.setenv("FAKE_CLAUDE_ORDER", order) monkeypatch.setenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "1") + # Releases without ClaudeAgentOptions.cli_path find the CLI as "claude" on PATH. + monkeypatch.setenv("PATH", str(tmp_path) + os.pathsep + os.environ.get("PATH", "")) return str(launcher) return make @@ -65,7 +67,8 @@ def install(manager, config=None): def converse(cli_path): async def go(): - options = claude_agent_sdk.ClaudeAgentOptions(cli_path=cli_path) + fields = getattr(claude_agent_sdk.ClaudeAgentOptions, "__dataclass_fields__", {}) + options = claude_agent_sdk.ClaudeAgentOptions(**({"cli_path": cli_path} if "cli_path" in fields else {})) messages = [] async with claude_agent_sdk.ClaudeSDKClient(options=options) as client: await client.query("take a screenshot") diff --git a/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py index b3520be7e..946697b24 100644 --- a/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py +++ b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py @@ -123,6 +123,8 @@ def install(manager, config=None): return exporter yield install + if state.get("on"): + OpenAIAgentsInstrumentor().uninstrument() # lets the next test install a fresh processor agents.set_trace_processors([]) provider.shutdown() @@ -197,9 +199,9 @@ def test_local_restrictions_deny_the_screenshot(f01, traced, flag): def test_disabled_manager_keeps_the_old_attributes(f01, traced): exporter, _, result = run_computer(create_attachment_manager(config=AttachmentConfig()), traced) assert result.final_output == "done" and f01.uploads == [] - if COMPUTER_SPAN: - outputs = [s.attributes.get("output.value") for s in function_spans(exporter)] - assert data_uri("image/png", SHOT) in outputs + for span in function_spans(exporter): + assert not refs(span) + assert not str(span.attributes.get("output.value", "")).startswith("fi-attachment:") def test_sensitive_data_off_records_and_uploads_nothing(f01, traced): From be331c3931d6c31ee2dfe1b472d99201327bb04d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:52:07 -0800 Subject: [PATCH 06/16] test(python): cover Firecrawl crawl-status documents Co-Authored-By: Claude Opus 5.5 --- .../test_firecrawl_attachments.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py index de6847722..f710e16e0 100644 --- a/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py +++ b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py @@ -17,7 +17,7 @@ from _harness import data_uri, flush, png, refs, span_named # noqa: E402 sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "firecrawl" / "tests")) -from _firecrawl_fake import FakeFirecrawl # noqa: E402 +from _firecrawl_fake import JOB_ID, FakeFirecrawl, crawl_status # noqa: E402 SHOT = png(seed=61) ACTION_SHOT = png(seed=62) @@ -80,7 +80,7 @@ def test_hosted_only_screenshot_uploads_nothing(f01, instrument, fake): manager.shutdown(5) -def test_async_client_and_crawl_documents(f01, instrument, fake): +def test_async_client_scrape(f01, instrument, fake): manager = f01.manager() instrument(manager) fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) @@ -95,6 +95,20 @@ async def go(): manager.shutdown(5) +def test_crawl_status_documents_upload_on_the_status_span(f01, instrument, fake): + manager = f01.manager() + exporter = instrument(manager) + body = crawl_status("completed", pages=2) + body["data"] = [dict(page, screenshot=data_uri("image/png", png(seed=63 + n))) for n, page in enumerate(body["data"])] + fake.routes[("GET", "/v2/crawl/" + JOB_ID)] = (200, body) + job = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).get_crawl_status(JOB_ID) + flush(manager) + assert len(job.data) == 2 + assert sorted(f01.payloads) == sorted([png(seed=63), png(seed=64)]) + assert len(refs(span_named(exporter, "firecrawl.get_crawl_status"))) == 2 + manager.shutdown(5) + + @pytest.mark.parametrize("flag", ["hide_outputs", "hide_inputs", "pii_redaction"]) def test_local_restrictions_deny(f01, instrument, fake, flag): manager = f01.manager() From 0b2eb4b50d8f80e1999bfbbb276ea13ac1536db9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:04:33 -0800 Subject: [PATCH 07/16] fix(python): name the per-call capture-policy context key like the TS key The Python auto module and the shared media helper stored the per-call capture policy under "traceai.auto.attachment_capture_policy.v1", while the TypeScript auto module uses "traceai.auto.attachment_policy.v1". Use the TS name in both places so the two languages share one key name, as the owner key already did. Tests pin both names to the TS strings and prove a policy set by name alone restricts capture. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 2 +- python/traceai/__init__.py | 5 +++-- python/traceai/tests/test_attachments.py | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 28fd3a94a..1673fd5f9 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -59,7 +59,7 @@ "wav": "audio/wav"} # Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the # TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. -CAPTURE_POLICY_KEY = "traceai.auto.attachment_capture_policy.v1" +CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" OWNER_KEY = "traceai.auto.attachment_owner.v1" # CapturePolicy facts that grant capture; every other fact only restricts. _PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 2cf77de90..2eef5d036 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -45,8 +45,9 @@ def _env_disabled() -> bool: # Context keys read by the shared provider-media helper (and any adapter) BY NAME, so # setting them never needs the attachments package. They match -# fi_instrumentation.attachments.media_capture.CAPTURE_POLICY_KEY / OWNER_KEY. -_CAPTURE_POLICY_KEY = "traceai.auto.attachment_capture_policy.v1" +# fi_instrumentation.attachments.media_capture.CAPTURE_POLICY_KEY / OWNER_KEY and the +# TypeScript `traceai_auto` keys (`createContextKey` names) exactly. +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" _OWNER_KEY = "traceai.auto.attachment_owner.v1" _CORE_TOO_OLD = ("attachment_config, attachment_manager and trace_config need a fi-instrumentation-otel " diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index c4fc56195..4168bb04c 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -212,6 +212,28 @@ def test_context_keys_match_the_shared_helper(): assert traceai._OWNER_KEY == media_capture.OWNER_KEY +def test_context_keys_use_the_typescript_names(): + # The TypeScript auto module names its keys createContextKey("traceai.auto.attachment_policy.v1") + # and createContextKey("traceai.auto.attachment_owner.v1"); the Python names are the same strings. + assert traceai._CAPTURE_POLICY_KEY == "traceai.auto.attachment_policy.v1" + assert traceai._OWNER_KEY == "traceai.auto.attachment_owner.v1" + assert media_capture.CAPTURE_POLICY_KEY == "traceai.auto.attachment_policy.v1" + assert media_capture.OWNER_KEY == "traceai.auto.attachment_owner.v1" + + +def test_policy_set_under_the_shared_name_is_honoured(): + # An adapter that sets the per-call policy by the shared NAME (no traceai import) restricts capture. + ctx = otel_context.set_value("traceai.auto.attachment_policy.v1", CapturePolicy(capture_content=False)) + token = otel_context.attach(ctx) + try: + assert media_capture.local_capture_policy(TraceConfig(), direction="input").capture_content is False + finally: + otel_context.detach(token) + assert media_capture.local_capture_policy(TraceConfig(), direction="input").capture_content is True + assert otel_context.get_value("traceai.auto.attachment_policy.v1", + traceai.with_attachment_capture_policy(CapturePolicy(muted=True))).muted is True + + def test_capture_policy_helper_only_restricts(): denied = traceai.with_attachment_capture_policy(CapturePolicy(capture_content=False, muted=True)) # A later, permissive policy in the same context cannot relax the earlier denial. From 42724ea617a4826b585d38a7080aa41cec6119bb Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:51:34 -0800 Subject: [PATCH 08/16] feat(python): register the MCP and agent media adapters in the auto module traceai.init now enables openai_agents, claude_agent_sdk, smolagents and firecrawl by name and hands them the one shared attachment manager, with the same OFF keyword arguments as every other instrumentor when attachments are off. report() reads each installed package's capability descriptor for those four plus mcp and mcp_use, so a package version without the adapter stays unqualified. mcp_use has a LangChain callback, not an instrumentor: enabling it only records it, and traceai.mcp_use_callback() builds the callback on auto's provider, hide flags and shared manager. traceai.wrap_mcp_client(session) does the same for MCP, whose instrumentor only propagates context. Both pass the manager only while capture is on and the installed package has the adapter. Co-Authored-By: Claude Opus 5.5 --- .../attachments-python-mcp-agents-tests.yml | 11 +- .../test_auto_registry.py | 256 ++++++++++++++++++ python/traceai/README.md | 17 +- python/traceai/__init__.py | 82 +++++- python/traceai/tests/test_attachments.py | 168 ++++++++++++ 5 files changed, 526 insertions(+), 8 deletions(-) create mode 100644 python/tests/attachments_mcp_agents/test_auto_registry.py diff --git a/.github/workflows/attachments-python-mcp-agents-tests.yml b/.github/workflows/attachments-python-mcp-agents-tests.yml index 89bf05ee6..04b4741c3 100644 --- a/.github/workflows/attachments-python-mcp-agents-tests.yml +++ b/.github/workflows/attachments-python-mcp-agents-tests.yml @@ -17,6 +17,7 @@ on: - 'python/frameworks/firecrawl/**' - 'python/tests/attachments_mcp_agents/**' - 'python/tests/attachments_providers/conftest.py' + - 'python/traceai/**' - '.github/workflows/attachments-python-mcp-agents-tests.yml' push: branches: [dev] @@ -30,6 +31,7 @@ on: - 'python/frameworks/firecrawl/**' - 'python/tests/attachments_mcp_agents/**' - 'python/tests/attachments_providers/conftest.py' + - 'python/traceai/**' - '.github/workflows/attachments-python-mcp-agents-tests.yml' workflow_dispatch: @@ -76,6 +78,10 @@ jobs: sdks: 'firecrawl-py==4.46.2'} - {package: firecrawl, tier: latest, python: '3.12', path: firecrawl, suite: test_firecrawl_attachments.py, sdks: 'firecrawl-py==4.55.0'} + # traceai.init enabling all six by name (registry, report rows, capture through auto). + - {package: auto, tier: latest, python: '3.12', expect: '1', suite: test_auto_registry.py, + path: 'mcp mcp-use openai-agents claude-agent-sdk smolagents firecrawl', + sdks: 'mcp==2.3.0 mcp-use==1.7.1 openai-agents==0.24.0 claude-agent-sdk==0.2.180 smolagents==1.26.0 openai pillow firecrawl-py==4.55.0'} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -90,15 +96,16 @@ jobs: run: | python -m venv .venv extra="" - if [ -n "$PKG_PATH" ]; then extra="-e python/frameworks/$PKG_PATH"; fi + for pkg in $PKG_PATH; do extra="$extra -e python/frameworks/$pkg"; done # shellcheck disable=SC2086 .venv/bin/pip install -e python $extra opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' packaging $SDKS .venv/bin/pip freeze | grep -iE '^(mcp|mcp-use|openai-agents|openai|claude-agent-sdk|smolagents|firecrawl-py|pillow|langchain-core)==' || true - name: Run with outbound network disabled env: SUITE: ${{ matrix.suite }} + TRACEAI_EXPECT_MCP_AGENT_ADAPTERS: ${{ matrix.expect || '0' }} run: | - sudo --preserve-env=MCP_USE_ANONYMIZED_TELEMETRY,MCP_USE_LANGFUSE,MCP_USE_LAMINAR unshare --net -- bash -e -c ' + sudo --preserve-env=MCP_USE_ANONYMIZED_TELEMETRY,MCP_USE_LANGFUSE,MCP_USE_LAMINAR,TRACEAI_EXPECT_MCP_AGENT_ADAPTERS unshare --net -- bash -e -c ' ip link set lo up cd "$2/python" PYTHONPATH=. "$1" -m pytest "tests/attachments_mcp_agents/$3" -q -p no:cacheprovider -o addopts="" \ diff --git a/python/tests/attachments_mcp_agents/test_auto_registry.py b/python/tests/attachments_mcp_agents/test_auto_registry.py new file mode 100644 index 000000000..1c0abf730 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_auto_registry.py @@ -0,0 +1,256 @@ +"""The auto module with the real MCP and agent packages: registry, report rows and capture. + +``traceai.init`` borrows a loopback F01 manager (or runs with attachments OFF) and +enables each installed package by name. Every test skips the packages that are not +installed; ``TRACEAI_EXPECT_MCP_AGENT_ADAPTERS=1`` (set by CI) turns a missing +package into a failure so a broken install cannot pass silently. +""" + +import asyncio +import importlib +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +traceai = pytest.importorskip("traceai") + +from fi_instrumentation import TraceConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +from _harness import data_uri, flush, harness, png, refs # noqa: E402 + +provider = harness.provider # the loopback OpenAI-compatible API (fixture) + +# report row name -> (instrumentor package, SDK module, descriptor module, descriptor name) +PACKAGES = { + "mcp": ("traceai_mcp", "mcp", "traceai_mcp", "mcp_attachment_capabilities"), + "mcp_use": ("traceai_mcp_use", "mcp_use", "traceai_mcp_use._attachments", "mcp_use_attachment_capabilities"), + "openai_agents": ("traceai_openai_agents", "agents", "traceai_openai_agents._attachments", + "openai_agents_attachment_capabilities"), + "claude_agent_sdk": ("traceai_claude_agent_sdk", "claude_agent_sdk", "traceai_claude_agent_sdk._attachments", + "claude_agent_attachment_capabilities"), + "smolagents": ("traceai_smolagents", "smolagents", "traceai_smolagents._attachments", + "smolagents_attachment_capabilities"), + "firecrawl": ("traceai_firecrawl", "firecrawl", "traceai_firecrawl._attachments", + "firecrawl_attachment_capabilities"), +} +INSTRUMENTORS = {"openai_agents": "OpenAIAgentsInstrumentor", "claude_agent_sdk": "ClaudeAgentInstrumentor", + "smolagents": "SmolagentsInstrumentor", "firecrawl": "FirecrawlInstrumentor"} +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" +EXPECT_ALL = os.environ.get("TRACEAI_EXPECT_MCP_AGENT_ADAPTERS") == "1" + + +def require(name): + package, sdk = PACKAGES[name][:2] + for module in (package, sdk): + if importlib.util.find_spec(module) is None: + if EXPECT_ALL: + pytest.fail("%s is not installed" % module) + pytest.skip("%s is not installed" % module) + + +@pytest.fixture +def auto(monkeypatch): + """Fresh auto state; uninstruments whatever a test enabled.""" + for attr, value in (("_tracer_provider", None), ("_init_options", None), ("_instrumented", set()), + ("_adapter_status", {}), ("_attachment_manager_borrowed", False)): + monkeypatch.setattr(traceai, attr, value) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + started = [] + + def init(libraries=(), **options): + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", + auto_instrument=list(libraries), verbose=False, + set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + started.append((tracer_provider, list(libraries))) + return tracer_provider, exporter + + yield init + for tracer_provider, libraries in started: + for name in libraries: + module = PACKAGES[name][0] + if module in traceai._ENTRY_POINTS: # mcp-use has no instrumentor + instrumentor = getattr(importlib.import_module(module), traceai._ENTRY_POINTS[module])() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + tracer_provider.shutdown() + + +@pytest.mark.parametrize("name", sorted(PACKAGES)) +def test_report_row_is_the_installed_descriptor(name): + require(name) + _, _, path, attr = PACKAGES[name] + descriptor = getattr(importlib.import_module(path), attr) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows[name]["media"] == "provider_adapter" + assert rows[name]["media_operations"] == list(descriptor["operations"]) + assert rows[name]["package"] == PACKAGES[name][0] + + +@pytest.mark.parametrize("name", sorted(INSTRUMENTORS)) +def test_enabling_by_name_instruments_with_attachments_on_and_off(name, auto, f01): + require(name) + # OFF (no manager): the instrumentor accepts auto's OFF keyword arguments. + auto([name]) + assert traceai.attachment_status()["adapters"][name] == "registered" + instrumentor = getattr(importlib.import_module(PACKAGES[name][0]), INSTRUMENTORS[name])() + assert instrumentor.is_instrumented_by_opentelemetry + instrumentor.uninstrument() + traceai._tracer_provider = None + traceai._init_options = None + # ON (a borrowed, enabled manager): the same, with attachments=. + manager = f01.manager() + try: + auto([name], attachment_manager=manager) + assert traceai.attachment_status()["adapters"][name] == "registered" + assert traceai.get_attachment_manager() is manager + finally: + manager.shutdown(5) + + +def _mcp_session(calls): + from mcp import ClientSession, StdioServerParameters + from mcp.client.stdio import stdio_client + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + return await calls(traceai.wrap_mcp_client(session)) + + return asyncio.run(go()) + + +def test_wrap_mcp_client_captures_on_the_shared_manager(auto, f01): + require("mcp") + sys.path.insert(0, str(SERVER.parent)) + import _mcp_media_server as server + + manager = f01.manager() + try: + _, exporter = auto(["mcp"], attachment_manager=manager) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + _mcp_session(calls) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] + assert f01.payloads == [server.png(1)] + assert sum(len(refs(s)) for s in exporter.get_finished_spans() if s.name == "mcp.tools/call") == 1 + finally: + manager.shutdown(5) + + +def test_wrap_mcp_client_with_attachments_off_uploads_nothing(auto, f01): + require("mcp") + _, exporter = auto(["mcp"]) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + result = _mcp_session(calls) + assert result.content # the SDK result is unchanged + assert f01.uploads == [] and f01.payloads == [] + assert all(not refs(s) for s in exporter.get_finished_spans()) + + +def _mcp_use_run(callback, tool): + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "mcp-use" / "tests")) + import _mcp_use_support as support + from mcp_use import MCPAgent, MCPClient + + async def go(): + client = MCPClient.from_dict({"mcpServers": {"media": {"command": sys.executable, "args": [str(SERVER)]}}}) + try: + agent = MCPAgent(llm=support.ChatFake(script=[support.tool_call(tool, {}, "call_media_1"), + support.answer()]), + client=client, callbacks=[callback]) + return await agent.run(support.PROMPT) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +def test_mcp_use_callback_captures_on_the_shared_manager(auto, f01): + require("mcp_use") + manager = f01.manager() + try: + auto(["mcp_use"], attachment_manager=manager) + _mcp_use_run(traceai.mcp_use_callback(), "screenshot") + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] + finally: + manager.shutdown(5) + + +def test_mcp_use_callback_carries_auto_hide_flags(auto, f01): + require("mcp_use") + manager = f01.manager() + try: + auto(["mcp_use"], attachment_manager=manager, trace_config=TraceConfig(hide_outputs=True)) + _mcp_use_run(traceai.mcp_use_callback(), "screenshot") + flush(manager) + assert f01.uploads == [] + finally: + manager.shutdown(5) + + +def test_smolagents_enabled_by_name_captures_a_model_input_image(auto, f01, provider): + require("smolagents") + pil = pytest.importorskip("PIL.Image") + pytest.importorskip("openai") + import smolagents + + manager = f01.manager() + try: + auto(["smolagents"], attachment_manager=manager) + image = pil.new("RGB", (8, 6), (7, 10, 20)) + content = [{"type": "text", "text": "look"}, {"type": "image", "image": image}] + model = smolagents.OpenAIServerModel(model_id="m", api_base=provider.url + "/v1", api_key="SYNTHETIC_KEY") + if hasattr(model, "generate"): + from smolagents.models import ChatMessage, MessageRole + + model.generate([ChatMessage(role=MessageRole.USER, content=content)]) + else: + model([{"role": "user", "content": content}]) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "input", "model_input")] + finally: + manager.shutdown(5) + + +def test_firecrawl_enabled_by_name_captures_an_inline_screenshot(auto, f01): + require("firecrawl") + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "firecrawl" / "tests")) + from _firecrawl_fake import FakeFirecrawl + from firecrawl import Firecrawl + + shot = png(seed=71) + fake = FakeFirecrawl() + manager = f01.manager() + try: + _, exporter = auto(["firecrawl"], attachment_manager=manager) + fake.routes[("POST", "/v2/scrape")] = (200, {"success": True, "data": { + "markdown": "body", "screenshot": data_uri("image/png", shot), + "metadata": {"sourceURL": "https://example.com"}}}) + document = Firecrawl(api_key="fc-SYNTHETIC-PLACEHOLDER", api_url=fake.origin, max_retries=1).scrape( + "https://example.com", formats=["screenshot"]) + flush(manager) + assert document.screenshot == data_uri("image/png", shot) + assert f01.payloads == [shot] + finally: + manager.shutdown(5) + fake.close() diff --git a/python/traceai/README.md b/python/traceai/README.md index c58997fd7..8b98dfb66 100644 --- a/python/traceai/README.md +++ b/python/traceai/README.md @@ -57,11 +57,22 @@ to add a `CapturePolicy` restriction for some calls, and `with_attachment_owners mark calls whose media your own code captures, so instrumentors do not upload it again. Both return an OpenTelemetry context to attach around those calls. +MCP and agent tool media (screenshots, images, audio, files) uses the same manager: + +| Integration | How to enable it | +|---|---| +| `openai_agents`, `claude_agent_sdk`, `smolagents`, `firecrawl` | `auto_instrument=[...]` with the name | +| `mcp` | `session = traceai.wrap_mcp_client(session)` on your `mcp.ClientSession` (the `mcp` instrumentor only propagates context) | +| `mcp_use` | `MCPAgent(..., callbacks=[traceai.mcp_use_callback()])` (mcp-use has a callback, not an instrumentor) | + +Both helpers use the provider and hide flags from `traceai.init`, and pass the shared +manager only while capture is on. + `traceai.report()` shows the attachment state and, per integration, `media`: `provider_adapter` when the installed instrumentor captures media (today `traceAI-openai`, -`traceAI-anthropic` and `traceAI-groq`), `session_binding` for voice integrations that -capture only through an explicit `bind_*` call, otherwise `unqualified`. It contains no -endpoint or credential and is not a playback claim. +`traceAI-anthropic`, `traceAI-groq` and the six MCP and agent packages above), +`session_binding` for voice integrations that capture only through an explicit `bind_*` +call, otherwise `unqualified`. It contains no endpoint or credential and is not a playback claim. ## License diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 2eef5d036..441cbe043 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -60,9 +60,21 @@ def _env_disabled() -> bool: "traceai_openai": ("traceai_openai", "openai_attachment_capabilities"), "traceai_anthropic": ("traceai_anthropic", "anthropic_attachment_capabilities"), "traceai_groq": ("traceai_groq._audio", "groq_attachment_capabilities"), + # MCP and agent tool media. MCP and mcp-use capture only through their explicit entry + # points (``wrap_mcp_client`` and ``mcp_use_callback`` below): the MCP instrumentor only + # propagates context, and mcp-use has a callback, not an instrumentor. + "traceai_mcp": ("traceai_mcp", "mcp_attachment_capabilities"), + "traceai_mcp_use": ("traceai_mcp_use._attachments", "mcp_use_attachment_capabilities"), + "traceai_openai_agents": ("traceai_openai_agents._attachments", "openai_agents_attachment_capabilities"), + "traceai_claude_agent_sdk": ("traceai_claude_agent_sdk._attachments", "claude_agent_attachment_capabilities"), + "traceai_smolagents": ("traceai_smolagents._attachments", "smolagents_attachment_capabilities"), + "traceai_firecrawl": ("traceai_firecrawl._attachments", "firecrawl_attachment_capabilities"), } # Voice integrations capture only through an explicit session binding (bind_* below). _SESSION_BINDINGS = {"traceai_pipecat": "bind_audio_processor", "traceai_livekit": "bind_session"} +# Integrations with no Instrumentor: enabling one only records it, and the application +# connects it explicitly (a pipeline processor, a live session, a LangChain callback). +_EXPLICIT_ONLY = frozenset(("traceai_pipecat", "traceai_livekit", "traceai_mcp_use")) # Registry of available instrumentors _INSTRUMENTOR_REGISTRY: Dict[str, str] = { @@ -97,6 +109,15 @@ def _env_disabled() -> bool: "instructor": "traceai_instructor", "guardrails": "traceai_guardrails", "mcp": "traceai_mcp", + "mcp_use": "traceai_mcp_use", + "mcp-use": "traceai_mcp_use", + # Agent frameworks and tools + "openai_agents": "traceai_openai_agents", + "openai-agents": "traceai_openai_agents", + "claude_agent_sdk": "traceai_claude_agent_sdk", + "claude-agent-sdk": "traceai_claude_agent_sdk", + "smolagents": "traceai_smolagents", + "firecrawl": "traceai_firecrawl", "pipecat": "traceai_pipecat", "livekit": "traceai_livekit", } @@ -115,6 +136,8 @@ def _env_disabled() -> bool: "traceai_autogen": "AutogenInstrumentor", "traceai_dspy": "DSPyInstrumentor", "traceai_haystack": "HaystackInstrumentor", "traceai_instructor": "InstructorInstrumentor", "traceai_guardrails": "GuardrailsInstrumentor", "traceai_mcp": "MCPInstrumentor", + "traceai_openai_agents": "OpenAIAgentsInstrumentor", "traceai_claude_agent_sdk": "ClaudeAgentInstrumentor", + "traceai_smolagents": "SmolagentsInstrumentor", "traceai_firecrawl": "FirecrawlInstrumentor", } _tracer_provider: Optional[TracerProvider] = None @@ -284,9 +307,10 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: # Import the instrumentor package module = importlib.import_module(instrumentor_module) - if lib_name_lower in ("pipecat", "livekit"): - # These use native/session integrations, not an Instrumentor. Auto - # cannot insert a pipeline processor or guess a live job/session. + if instrumentor_module in _EXPLICIT_ONLY: + # These use native/session/callback integrations, not an Instrumentor. Auto + # cannot insert a pipeline processor, guess a live job/session or attach a + # LangChain callback (``mcp_use_callback()`` builds one on the shared manager). _adapter_status[lib_name_lower] = "registered" return True with _init_lock: @@ -545,6 +569,56 @@ def _bind(library: str, entry_point: str, *args, **kwargs): return binding +def _explicit_kwargs(library: str, kwargs: dict, provider_key: str, config_key: str) -> dict: + """Defaults for an explicit entry point: auto's provider, its trace config and, only + while capture is on and the installed package has the media adapter, the ONE manager.""" + if _tracer_provider is None: + raise RuntimeError("initialize traceai before connecting " + library) + kwargs.setdefault(provider_key, _tracer_provider) + config = getattr(_tracer_provider, "trace_config", None) + if config is not None: + # Auto's hide and PII flags reach the integration; the caller can pass its own. + kwargs.setdefault(config_key, config) + if _ATTACHMENTS_SUPPORTED: + manager = get_attachment_manager() + module = _INSTRUMENTOR_REGISTRY[library] + if _capturing(manager) and _media_support(module)["media"] == "provider_adapter": + kwargs.setdefault("attachments", manager) + return kwargs + + +def mcp_use_callback(**kwargs): + """A ``traceai_mcp_use.FutureAGICallback`` on auto's provider and shared manager. + + Pass it to the agent: ``MCPAgent(..., callbacks=[traceai.mcp_use_callback()])``. + With attachments off, or an installed ``traceAI-mcp-use`` without the media adapter, + it is the same callback as before (no ``attachments=`` argument). Keyword arguments + are passed through and win over auto's defaults. + """ + module = importlib.import_module(_INSTRUMENTOR_REGISTRY["mcp_use"]) + callback = module.FutureAGICallback(**_explicit_kwargs("mcp_use", kwargs, "tracer_provider", "config")) + _adapter_status["mcp_use"] = "registered" + return callback + + +def wrap_mcp_client(session, **kwargs): + """Trace an ``mcp.ClientSession`` with ``traceai_mcp.wrap_mcp_client`` on auto's provider + and shared manager, so tool and resource media reach Future AGI. + + The MCP instrumentor only propagates context; this is the entry point that captures + media. An installed ``traceAI-mcp`` without ``wrap_mcp_client`` returns the session + unchanged and reports ``unsupported_version``. + """ + module = importlib.import_module(_INSTRUMENTOR_REGISTRY["mcp"]) + wrap = getattr(module, "wrap_mcp_client", None) + if wrap is None: + _adapter_status["mcp"] = "unsupported_version" + return session + wrapped = wrap(session, **_explicit_kwargs("mcp", kwargs, "tracer_provider", "trace_config")) + _adapter_status.setdefault("mcp", "registered") + return wrapped + + def bind_pipecat_audio_processor(processor, session_context): """Observe an existing processor; never start a new recording implicitly.""" return _bind("pipecat", "bind_audio_processor", processor, session_context) @@ -568,6 +642,8 @@ def bind_livekit_session(session, job_context, *, owned_recording_files=False, "with_attachment_ownership", "bind_pipecat_audio_processor", "bind_livekit_session", + "mcp_use_callback", + "wrap_mcp_client", "instrument", "get_tracer_provider", "SemanticConvention", diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index 4168bb04c..ba2660091 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -305,3 +305,171 @@ def import_module(name): assert rows["anthropic"]["status"] == "not_enabled" assert "mistral" not in rows # aliases share one row assert traceai.report()["attachments"]["state"] == "not_initialized" + + +# ------------------------------------------------- MCP and agent media adapters (#693) +_AGENT_INSTRUMENTORS = { # registry name -> (module, entry point) + "openai_agents": ("traceai_openai_agents", "OpenAIAgentsInstrumentor"), + "claude_agent_sdk": ("traceai_claude_agent_sdk", "ClaudeAgentInstrumentor"), + "smolagents": ("traceai_smolagents", "SmolagentsInstrumentor"), + "firecrawl": ("traceai_firecrawl", "FirecrawlInstrumentor"), +} +_MCP_AGENT_DESCRIPTORS = { # report row name -> (descriptor module, descriptor name) + "mcp": ("traceai_mcp", "mcp_attachment_capabilities"), + "mcp_use": ("traceai_mcp_use._attachments", "mcp_use_attachment_capabilities"), + "openai_agents": ("traceai_openai_agents._attachments", "openai_agents_attachment_capabilities"), + "claude_agent_sdk": ("traceai_claude_agent_sdk._attachments", "claude_agent_attachment_capabilities"), + "smolagents": ("traceai_smolagents._attachments", "smolagents_attachment_capabilities"), + "firecrawl": ("traceai_firecrawl._attachments", "firecrawl_attachment_capabilities"), +} + + +def test_mcp_and_agent_packages_are_registered_with_explicit_entry_points(): + for name, (module, entry) in _AGENT_INSTRUMENTORS.items(): + assert traceai._INSTRUMENTOR_REGISTRY[name] == module + assert traceai._ENTRY_POINTS[module] == entry + for alias, name in (("openai-agents", "openai_agents"), ("claude-agent-sdk", "claude_agent_sdk"), + ("mcp-use", "mcp_use")): + assert traceai._INSTRUMENTOR_REGISTRY[alias] == traceai._INSTRUMENTOR_REGISTRY[name] + assert traceai._INSTRUMENTOR_REGISTRY["mcp_use"] == "traceai_mcp_use" + # mcp-use has a LangChain callback, not an instrumentor: auto must never guess a class for it. + assert "traceai_mcp_use" not in traceai._ENTRY_POINTS + assert "traceai_mcp_use" in traceai._EXPLICIT_ONLY + + +@pytest.mark.parametrize("name", sorted(_AGENT_INSTRUMENTORS)) +def test_agent_instrumentors_get_the_shared_manager_and_the_off_kwargs_when_off(monkeypatch, name): + module_name, entry = _AGENT_INSTRUMENTORS[name] + for env, expected in ((None, "on"), ("false", "off")): + if env is None: + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + else: + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", env) + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai, "_init_options", None) + manager = _enabled_like_manager() + provider = SimpleNamespace(attachment_manager=manager, trace_config=TraceConfig()) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + constructor = Mock() + imported = [] + + def import_module(requested, constructor=constructor, imported=imported): + imported.append(requested) + return SimpleNamespace(**{entry: constructor}) + + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + traceai.init(project_name="synthetic", auto_instrument=[name], verbose=False) + assert imported == [module_name] + if expected == "on": + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachments=manager, attachment_manager=manager) + else: # attachments OFF: exactly the keyword arguments every other instrumentor gets + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachment_manager=manager, + attachment_policy=manager.policy) + assert traceai.attachment_status()["adapters"][name] == "registered" + + +def test_enabling_mcp_use_records_it_and_constructs_nothing(monkeypatch): + module = SimpleNamespace(FutureAGICallback=Mock(side_effect=AssertionError("constructed a callback")), + SomethingInstrumentor=Mock(side_effect=AssertionError("guessed a class"))) + monkeypatch.setattr(traceai.importlib, "import_module", Mock(return_value=module)) + assert traceai._instrument_library("mcp-use", verbose=False) + assert traceai.attachment_status()["adapters"]["mcp-use"] == "registered" + + +def test_mcp_and_agent_report_rows_read_each_installed_descriptor(monkeypatch): + modules = {} + for name, (path, attr) in _MCP_AGENT_DESCRIPTORS.items(): + modules[path] = SimpleNamespace(**{attr: {"operations": (name + ".output",)}}) + modules.setdefault(traceai._INSTRUMENTOR_REGISTRY[name], SimpleNamespace()) + + def import_module(requested): + if requested in modules: + return modules[requested] + raise ImportError(requested) + + monkeypatch.setattr(traceai, "_installed", lambda module: module in modules) + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + for name in _MCP_AGENT_DESCRIPTORS: + assert rows[name]["media"] == "provider_adapter", name + assert rows[name]["media_operations"] == [name + ".output"], name + assert rows[name]["status"] == "not_enabled", name + for alias in ("mcp-use", "openai-agents", "claude-agent-sdk"): + assert alias not in rows # aliases share one row + # An installed package version without the media adapter (no descriptor) is unqualified. + for name, (path, _attr) in _MCP_AGENT_DESCRIPTORS.items(): + saved = modules[path] + if path == traceai._INSTRUMENTOR_REGISTRY[name]: + modules[path] = SimpleNamespace() # the package is installed, its descriptor is not + else: + del modules[path] # this package version has no _attachments module + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows[name] == {"name": name, "package": traceai._INSTRUMENTOR_REGISTRY[name], + "status": "not_enabled", "media": "unqualified"}, name + modules[path] = saved + + +def _explicit_setup(monkeypatch, manager, config=None, module=None, adapter=True): + provider = SimpleNamespace(attachment_manager=manager, trace_config=config or TraceConfig(hide_outputs=True)) + monkeypatch.setattr(traceai, "_tracer_provider", provider) + monkeypatch.setattr(traceai, "_media_support", lambda m: {"media": "provider_adapter" if adapter else "unqualified"}) + monkeypatch.setattr(traceai.importlib, "import_module", Mock(return_value=module)) + return provider + + +def test_mcp_use_callback_uses_the_shared_manager_only_while_capturing(monkeypatch): + callback_class = Mock() + module = SimpleNamespace(FutureAGICallback=callback_class) + manager = _enabled_like_manager() + provider = _explicit_setup(monkeypatch, manager, module=module) + assert traceai.mcp_use_callback() is callback_class.return_value + callback_class.assert_called_once_with(tracer_provider=provider, config=provider.trace_config, attachments=manager) + assert traceai.attachment_status()["adapters"]["mcp_use"] == "registered" + # Caller arguments win over auto's defaults. + callback_class.reset_mock() + own = TraceConfig() + traceai.mcp_use_callback(config=own, redact=["x"]) + assert callback_class.call_args.kwargs["config"] is own and callback_class.call_args.kwargs["redact"] == ["x"] + # Attachments OFF: the callback auto built before (no attachments argument at all). + for off in ("env", "disabled", "no_adapter"): + callback_class.reset_mock() + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + if off == "env": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + provider = _explicit_setup(monkeypatch, manager, module=module) + elif off == "disabled": + provider = _explicit_setup(monkeypatch, DisabledAttachmentManager(), module=module) + else: + provider = _explicit_setup(monkeypatch, manager, module=module, adapter=False) + traceai.mcp_use_callback() + callback_class.assert_called_once_with(tracer_provider=provider, config=provider.trace_config) + + +def test_wrap_mcp_client_uses_the_shared_manager_and_tolerates_old_packages(monkeypatch): + wrap = Mock() + manager = _enabled_like_manager() + provider = _explicit_setup(monkeypatch, manager, module=SimpleNamespace(wrap_mcp_client=wrap)) + session = object() + assert traceai.wrap_mcp_client(session) is wrap.return_value + wrap.assert_called_once_with(session, tracer_provider=provider, trace_config=provider.trace_config, + attachments=manager) + wrap.reset_mock() + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + traceai.wrap_mcp_client(session) + wrap.assert_called_once_with(session, tracer_provider=provider, trace_config=provider.trace_config) + # An installed traceAI-mcp without wrap_mcp_client: the session comes back untouched. + _explicit_setup(monkeypatch, manager, module=SimpleNamespace()) + assert traceai.wrap_mcp_client(session) is session + assert traceai.attachment_status()["adapters"]["mcp"] == "unsupported_version" + + +def test_explicit_entry_points_need_init(monkeypatch): + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(FutureAGICallback=Mock(), wrap_mcp_client=Mock()))) + with pytest.raises(RuntimeError): + traceai.mcp_use_callback() + with pytest.raises(RuntimeError): + traceai.wrap_mcp_client(object()) From a598b9adeb7f6eaa39d03b520b598d41015ade09 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:16:09 -0800 Subject: [PATCH 09/16] ci(python): pin mcp below 2 in the all-six auto row (mcp-use 1.7.1 needs mcp<2) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/attachments-python-mcp-agents-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/attachments-python-mcp-agents-tests.yml b/.github/workflows/attachments-python-mcp-agents-tests.yml index 04b4741c3..31a7a8119 100644 --- a/.github/workflows/attachments-python-mcp-agents-tests.yml +++ b/.github/workflows/attachments-python-mcp-agents-tests.yml @@ -81,7 +81,7 @@ jobs: # traceai.init enabling all six by name (registry, report rows, capture through auto). - {package: auto, tier: latest, python: '3.12', expect: '1', suite: test_auto_registry.py, path: 'mcp mcp-use openai-agents claude-agent-sdk smolagents firecrawl', - sdks: 'mcp==2.3.0 mcp-use==1.7.1 openai-agents==0.24.0 claude-agent-sdk==0.2.180 smolagents==1.26.0 openai pillow firecrawl-py==4.55.0'} + sdks: 'mcp>=1.30.0,<2 mcp-use==1.7.1 openai-agents==0.24.0 claude-agent-sdk==0.2.180 smolagents==1.26.0 openai pillow firecrawl-py==4.55.0'} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: From d8ee8cde9453838b031c654262a607a121c5c844 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:10:54 -0800 Subject: [PATCH 10/16] fix(python): project Claude Agent SDK tool media on a live tool span; deny mislabelled media Claude Agent SDK: the CLI usually runs PostToolUse before it streams the tool result. The hook ended the tool span first, so tool-result media was uploaded against an ended span context and no span carried its fi.attachment.refs. With attachments on, the hook now defers the span's end until its ToolResultBlock has been captured (bounded table, 30 s timer, conversation end and disconnect flush), then ends it at the time the hook recorded. Results with no traced tool go to the live conversation span. With attachments off the hook ends the span immediately, as before. tool_media: bytes whose leading signature names another modality than the declared MIME (a PNG labelled audio/wav or application/octet-stream) are denied, so a mislabel cannot escape hide_input_images. OWNER_KEY aliases the core constant when the core defines it. smolagents: every model call that carries a captured image, new or replayed, is marked as owner, so a provider adapter never re-uploads a replayed image. The seen-image table gets a fresh lock in a forked child (pid check, no fork hook). Co-Authored-By: Claude Opus 5.5 --- .../attachments/tool_media.py | 56 ++++++++++- python/frameworks/claude-agent-sdk/README.md | 8 +- .../traceai_claude_agent_sdk/_attachments.py | 75 +++++++------- .../_client_wrapper.py | 54 ++++++++--- .../traceai_claude_agent_sdk/_hooks.py | 97 +++++++++++++++++-- python/frameworks/mcp/README.md | 3 +- python/frameworks/smolagents/README.md | 5 +- .../traceai_smolagents/_attachments.py | 40 ++++++-- .../traceai_smolagents/_wrappers.py | 10 +- .../_fake_claude_cli.py | 12 ++- .../test_claude_agent_sdk_attachments.py | 39 +++++++- .../test_smolagents_attachments.py | 67 +++++++++++++ .../attachments_mcp_agents/test_tool_media.py | 33 +++++++ 13 files changed, 419 insertions(+), 80 deletions(-) diff --git a/python/fi_instrumentation/attachments/tool_media.py b/python/fi_instrumentation/attachments/tool_media.py index 600c4c471..525b8130b 100644 --- a/python/fi_instrumentation/attachments/tool_media.py +++ b/python/fi_instrumentation/attachments/tool_media.py @@ -13,6 +13,10 @@ opens a file, or reads a stream. A URL or file-backed value is skipped. - Every value is preflighted before it is read or decoded. Each observation has a bounded decode budget and part count; parts beyond them are skipped. +- A declared MIME is never trusted over the bytes: when the first bytes are a + known signature of ANOTHER modality (a PNG labelled ``audio/wav`` or + ``application/octet-stream``), the part is denied, so a mislabel cannot + escape ``hide_input_images`` or a modality restriction. Never relabelled. - A context marked by another adapter as the owner of this call's media (``OWNER_KEY``, shared by name with ``traceai``) captures nothing here. @@ -20,6 +24,8 @@ non-throwing and returns ``None``/empty on denial. """ +import base64 +import binascii import dataclasses import io import sys @@ -39,7 +45,8 @@ MAX_TOOL_PARTS = 64 #: In-memory images are PNG-encoded only up to this many pixels (no huge synchronous encodes). MAX_IMAGE_PIXELS = 2048 * 2048 -OWNER_KEY = "traceai.auto.attachment_owner.v1" +#: The cross-adapter owner mark, by name (``media_capture.OWNER_KEY`` where the core defines it). +OWNER_KEY = getattr(mc, "OWNER_KEY", "traceai.auto.attachment_owner.v1") URI_PREFIX = "fi-attachment:" Occurrence = mc.Occurrence @@ -83,6 +90,50 @@ def mime_kind(mime: Any) -> Optional[str]: return MIME_MODALITIES.get(mime) if isinstance(mime, str) else None +#: Leading-byte signatures with an unambiguous F01 modality. Containers shared by audio and +#: video (MP4 ``ftyp``, WebM/Matroska) have none: they can only be denied by their declared MIME. +_SIGNATURES = ( + (0, b"\x89PNG\r\n\x1a\n", "image"), (0, b"\xff\xd8\xff", "image"), (0, b"GIF87a", "image"), + (0, b"GIF89a", "image"), (8, b"WEBP", "image"), (8, b"WAVE", "audio"), + (0, b"OggS", "audio"), (0, b"fLaC", "audio"), (0, b"ID3", "audio"), (0, b"%PDF-", "document"), +) + + +def signature_kind(head: bytes) -> Optional[str]: + """The modality the first bytes name, or None when they name none (or are ambiguous).""" + for offset, magic, kind in _SIGNATURES: + if head[offset:offset + len(magic)] == magic: + if offset == 8 and head[:4] != b"RIFF": + continue + return kind + return None + + +def _declared_kind(value: str, mimes, raw: bool) -> Optional[str]: + if raw: + return mime_kind(mimes[0]) + end = value.find(";", 5, 128) + return mime_kind(value[5:end]) if value.startswith("data:") and end > 5 else None + + +def consistent(value: Any, mimes, raw: bool) -> bool: + """False when the base64 value's first bytes are a signature of another modality than declared.""" + if not isinstance(value, str): + return True # Not inline base64: the structural parse rejects it. + start = 0 + if not raw: + comma = value.find(",", 0, 128) + if comma < 0: + return True + start = comma + 1 + try: + head = base64.b64decode(value[start:start + 16], validate=True) + except (binascii.Error, ValueError): + return True # Malformed base64: strict decoding rejects it later. + found = signature_kind(head) + return found is None or found == _declared_kind(value, mimes, raw) + + def _pil_image(value: Any) -> bool: module = sys.modules.get("PIL.Image") image_type = getattr(module, "Image", None) if module is not None else None @@ -172,7 +223,8 @@ def _inline(self, position, read, mimes, raw, purpose, image, role, tool_call_id def once(): if "value" not in seen: - seen["value"] = read() + value = read() + seen["value"] = value if consistent(value, mimes, raw) else None return seen["value"] def capture(): diff --git a/python/frameworks/claude-agent-sdk/README.md b/python/frameworks/claude-agent-sdk/README.md index 5f74be457..7bc316869 100644 --- a/python/frameworks/claude-agent-sdk/README.md +++ b/python/frameworks/claude-agent-sdk/README.md @@ -190,8 +190,12 @@ Pass the shared manager: `ClaudeAgentInstrumentor().instrument(tracer_provider=. The base64 `image` and PDF `document` blocks of a `ToolResultBlock` are uploaded from the original SDK message, before any text flattening or truncation, as `tool_result` occurrences with the `tool_use_id`. They belong -to the tool's span: while it is open they are projected on it; once the -PostToolUse hook ended it, they keep its exact span context as their parent. +to the tool's span and are projected on it (`fi.attachment.refs`). The CLI +usually runs the PostToolUse hook before it streams the tool result, so with +attachments on the tool span stays recording until its result has been +captured (at most 30 seconds, or until the conversation ends) and then ends +at the time the hook recorded; its duration is unchanged. With attachments +off, the hook ends the span immediately, as before. Media bytes never reach `tool.output` or the conversation output text. The SDK drops top-level user image blocks while parsing, so prompt images are not captured. URL and file sources are never read. diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py index a7458fd87..5d8abe732 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py @@ -5,23 +5,25 @@ base64 ``image`` and PDF ``document`` blocks there are uploaded BEFORE any text flattening or truncation, at the tool's span (with ``tool_call_id``): -- while the tool span is still open, on that span (and projected on it); -- after the PostToolUse hook ended it, on its exact span context (the - occurrence keeps the causal parent; the ended span is never mutated); +- on the tool span, while it is still recording: before the PostToolUse hook + (result-first) or after it, because with capture on the hook defers the + span's end until its result has been captured (hooks-first, the CLI's + usual order); the span then ends at the time the hook recorded; - for a result with no traced tool, on the conversation span. +Every occurrence is projected (``fi.attachment.refs``) on the live span that +is its upload parent. + URL and file sources, paths and transcripts are never read. The SDK drops top-level user image blocks during parsing, so prompt images are not observable here (``source_unavailable``). Model calls run inside the Claude Code process, so no in-process provider adapter sees the same bytes. """ -from collections import OrderedDict from types import MappingProxyType -from typing import Any, Mapping, Optional +from typing import Any, List, Mapping -__all__ = ("capture_tool_results", "claude_agent_attachment_capabilities", "remember_ended_tool", - "media_safe") +__all__ = ("capture_tool_results", "claude_agent_attachment_capabilities", "media_safe", "tool_result_ids") claude_agent_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -33,40 +35,37 @@ _EMPTY: Mapping[str, Any] = MappingProxyType({}) _MEDIA_TYPES = ("image", "document", "audio") -_MAX_ENDED = 256 -_ended: "OrderedDict[str, Any]" = OrderedDict() +_MAX_BLOCKS = 64 -def remember_ended_tool(tool_use_id: Any, span: Any) -> None: - """Keep the exact span context of a tool span the PostToolUse hook ended (bounded).""" +def _get(value: Any, key: str) -> Any: try: - if not isinstance(tool_use_id, str): - return - _ended[tool_use_id] = span.get_span_context() - _ended.move_to_end(tool_use_id) - while len(_ended) > _MAX_ENDED: - _ended.popitem(last=False) + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) except Exception: - return - - -def forget_ended_tools() -> None: - _ended.clear() + return None -def _get(value: Any, key: str) -> Any: +def tool_result_ids(message: Any) -> List[str]: + """The tool_use_ids of the ``ToolResultBlock``s in one ``UserMessage`` (bounded). Never raises.""" try: - return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + content = _get(message, "content") + if not isinstance(content, list): + return [] + return [block.tool_use_id for block in content[:_MAX_BLOCKS] + if type(block).__name__ == "ToolResultBlock" and isinstance(_get(block, "tool_use_id"), str)] except Exception: - return None + return [] -def capture_tool_results(manager, message: Any, conversation_span: Any, active_spans: Mapping[str, Any], *, - message_index: int, trace_config=None, capture=None) -> int: +def capture_tool_results(manager, message: Any, conversation_span: Any, active_spans: Mapping[str, Any], + deferred_spans: Mapping[str, Any] = _EMPTY, *, message_index: int, trace_config=None, + capture=None) -> int: """Capture the media blocks of every ``ToolResultBlock`` in one ``UserMessage``. Never raises. - ``active_spans`` maps a tool_use_id to its still-open span (the hooks' table). - Returns the number of admitted blocks. + ``active_spans`` maps a tool_use_id to its open span (PostToolUse not run yet) and + ``deferred_spans`` to a span PostToolUse completed whose end waits for this result; + both are the hooks' tables and both spans are still recording. Any other result is + captured on the (recording) conversation span. Returns the number of admitted blocks. """ try: from fi_instrumentation.attachments.media_capture import active @@ -87,12 +86,11 @@ def capture_tool_results(manager, message: Any, conversation_span: Any, active_s if not isinstance(blocks, list): continue tool_use_id = _get(block, "tool_use_id") - open_entry = active_spans.get(tool_use_id) if isinstance(tool_use_id, str) else None - if open_entry is not None: - target = open_entry[0] - elif isinstance(tool_use_id, str) and tool_use_id in _ended: - target = _ended[tool_use_id] - else: + entry = None + if isinstance(tool_use_id, str): + entry = active_spans.get(tool_use_id) or deferred_spans.get(tool_use_id) + target = entry[0] if entry is not None else conversation_span + if not _recording(target): target = conversation_span error = _get(block, "is_error") is True media = ToolMedia(manager, target, "claude_agent_sdk", trace_config=trace_config, capture=capture) @@ -118,6 +116,13 @@ def capture_tool_results(manager, message: Any, conversation_span: Any, active_s return admitted +def _recording(span: Any) -> bool: + try: + return bool(span.is_recording()) + except Exception: + return False + + def media_safe(value: Any, depth: int = 0) -> Any: """A telemetry copy without inline media bytes: base64 fields of media blocks become ``"[omitted]"``. diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py index ed40afadf..671aade46 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py @@ -14,10 +14,11 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode -from ._attachments import capture_tool_results, forget_ended_tools +from ._attachments import capture_tool_results, tool_result_ids from ._attributes import ClaudeAgentAttributes, ClaudeAgentSpanKind from ._hooks import ( _active_tool_spans, + _deferred_tool_ends, create_pre_tool_use_hook, create_subagent_start_hook, create_post_tool_use_hook, @@ -25,6 +26,8 @@ clear_parent_span, set_mcp_servers, clear_active_tool_spans, + end_deferred_tool, + end_deferred_tool_spans, safe_json_serialize, ) @@ -168,12 +171,14 @@ def _flatten_content(self, content: Any) -> str: return str(content) -def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: +def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer, defer_tool_end: bool = False) -> None: """Inject OpenTelemetry tracing hooks into ClaudeAgentOptions. Args: options: ClaudeAgentOptions instance tracer: OpenTelemetry tracer + defer_tool_end: tool-result media capture is on, so a completed tool span stays + recording until its result is captured (it keeps the hook's end time) """ if not hasattr(options, "hooks"): return @@ -194,7 +199,8 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: from claude_agent_sdk import HookMatcher pre_hook = create_pre_tool_use_hook(tracer) - post_hook = create_post_tool_use_hook(tracer) + post_hook = (create_post_tool_use_hook(tracer, defer_end=True) if defer_tool_end + else create_post_tool_use_hook(tracer)) start_hook = create_subagent_start_hook(tracer) # Insert at beginning to ensure we trace before user hooks @@ -241,7 +247,10 @@ def __init__(self, *args: Any, **kwargs: Any): # Extract options for hook injection options = kwargs.get("options") or (args[0] if args else None) if options: - inject_tracing_hooks(options, tracer) + if attachments is not None: + inject_tracing_hooks(options, tracer, defer_tool_end=True) + else: + inject_tracing_hooks(options, tracer) # Store MCP server configs for tool attribution mcp_servers = getattr(options, "mcp_servers", None) @@ -308,16 +317,23 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: elif msg_type == "UserMessage": # Tool-result media first, from the original typed blocks, # before any text flattening or truncation. + # A tool span the PostToolUse hook completed is still recording + # (deferred end), so the media refs are projected on it. if attachments is not None: - capture_tool_results( - attachments, - msg, - conversation_span, - _active_tool_spans, - message_index=message_index, - trace_config=trace_config, - capture=attachment_capture, - ) + try: + capture_tool_results( + attachments, + msg, + conversation_span, + _active_tool_spans, + _deferred_tool_ends, + message_index=message_index, + trace_config=trace_config, + capture=attachment_capture, + ) + finally: + for tool_use_id in tool_result_ids(msg): + end_deferred_tool(tool_use_id) # End current turn, mark next start turn_tracker.end_current_turn() turn_tracker.mark_next_start() @@ -373,7 +389,6 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: turn_tracker.end_current_turn() clear_parent_span() clear_active_tool_spans() - forget_ended_tools() def _build_span_attributes(self) -> Dict[str, Any]: """Build span attributes from options and prompt.""" @@ -521,4 +536,15 @@ async def __aenter__(self) -> "TracedClaudeSDKClient": async def __aexit__(self, *args: Any) -> None: await super().__aexit__(*args) + if attachments is not None and hasattr(original_class, "disconnect"): + + async def disconnect(self: Any, *args: Any, **kwargs: Any) -> Any: + """Disconnect; completed tool spans still waiting for their result end now.""" + try: + return await super(TracedClaudeSDKClient, self).disconnect(*args, **kwargs) + finally: + end_deferred_tool_spans() + + TracedClaudeSDKClient.disconnect = disconnect + return TracedClaudeSDKClient diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py index 4d2f29285..ecae04e36 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py @@ -7,13 +7,14 @@ import json import logging import time +from collections import OrderedDict from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional, Tuple from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode -from ._attachments import media_safe, remember_ended_tool +from ._attachments import media_safe from ._attributes import ( ClaudeAgentAttributes, ClaudeAgentSpanKind, @@ -27,6 +28,17 @@ # Key: tool_use_id, Value: (span, start_time, tool_name) _active_tool_spans: Dict[str, Tuple[Span, float, str]] = {} +# Tool spans the PostToolUse hook completed while tool-result media capture is on. The SDK streams +# the tool result AFTER that hook, so the span stays recording until its ToolResultBlock has been +# captured (its media refs are projected on it), then ends at the time the hook recorded. +# Key: tool_use_id, Value: (span, end_time_ns, timer handle or None). Bounded and time-limited. +_deferred_tool_ends: "OrderedDict[str, Tuple[Span, int, Any]]" = OrderedDict() +_MAX_DEFERRED_ENDS = 256 +# Tool results streamed while their span was still open (result-first): PostToolUse ends those now. +_results_seen: "OrderedDict[str, bool]" = OrderedDict() +#: A deferred tool span ends at most this long after its hook, even if its result never arrives. +DEFERRED_END_TIMEOUT_S = 30.0 + # Tools that launch a subagent: "Task" in older Claude Code releases, "Agent" in current ones SUBAGENT_TOOLS = ("Task", "Agent") @@ -263,11 +275,80 @@ async def pre_tool_use_hook( return pre_tool_use_hook -def create_post_tool_use_hook(tracer: trace_api.Tracer) -> Callable: +def _end_deferred(entry: Tuple[Span, int, Any]) -> None: + span, end_ns, handle = entry + try: + if handle is not None: + handle.cancel() + except Exception: + pass + try: + span.end(end_time=end_ns) + except Exception as e: + logger.debug(f"Failed to end deferred tool span: {e}") + + +def _expire_deferred(tool_use_id: str, entry: Tuple[Span, int, Any]) -> None: + if _deferred_tool_ends.get(tool_use_id) is entry: + del _deferred_tool_ends[tool_use_id] + _end_deferred(entry) + + +def _defer_tool_end(tool_use_id: str, span: Span, end_ns: int) -> bool: + """Keep a completed tool span recording until its result is captured. False: end it now.""" + try: + import asyncio + + loop = asyncio.get_running_loop() + except Exception: + return False # No asyncio loop to bound the wait (another async backend): end now. + if _results_seen.pop(tool_use_id, None): + return False # Its result was already captured on the open span. + try: + previous = _deferred_tool_ends.pop(tool_use_id, None) + if previous is not None: + _end_deferred(previous) + holder: List[Any] = [] + handle = loop.call_later(DEFERRED_END_TIMEOUT_S, lambda: _expire_deferred(tool_use_id, holder[0])) + entry = (span, end_ns, handle) + holder.append(entry) + _deferred_tool_ends[tool_use_id] = entry + while len(_deferred_tool_ends) > _MAX_DEFERRED_ENDS: + _, oldest = _deferred_tool_ends.popitem(last=False) + _end_deferred(oldest) + return True + except Exception: + return False + + +def end_deferred_tool(tool_use_id: Any) -> None: + """End one deferred tool span (its result was processed), at the time its hook recorded.""" + if not isinstance(tool_use_id, str): + return + entry = _deferred_tool_ends.pop(tool_use_id, None) + if entry is not None: + _end_deferred(entry) + elif tool_use_id in _active_tool_spans: + _results_seen[tool_use_id] = True + while len(_results_seen) > _MAX_DEFERRED_ENDS: + _results_seen.popitem(last=False) + + +def end_deferred_tool_spans() -> None: + """End every deferred tool span at its recorded time (conversation end).""" + _results_seen.clear() + while _deferred_tool_ends: + _, entry = _deferred_tool_ends.popitem(last=False) + _end_deferred(entry) + + +def create_post_tool_use_hook(tracer: trace_api.Tracer, defer_end: bool = False) -> Callable: """Create a PostToolUse hook for completing tool traces. Args: tracer: OpenTelemetry tracer instance + defer_end: keep the completed span recording until its tool result is captured + (only when tool-result media capture is on; the span keeps the hook's end time) Returns: Async hook function compatible with Claude Agent SDK @@ -303,6 +384,7 @@ async def post_tool_use_hook( return {} span, start_time, _ = span_info + end_ns = time.time_ns() duration_ms = (time.time() - start_time) * 1000 # Set output and duration. Inline media bytes never go through text serialization @@ -343,10 +425,10 @@ async def post_tool_use_hook( }, ) - # End the span. Its exact context stays available (bounded) so the tool's - # result media, which the SDK streams after this hook, keep this causal parent. - span.end() - remember_ended_tool(tool_use_id, span) + # End the span. With media capture on, the tool result (streamed after this + # hook) is captured on the still-recording span first; it ends at end_ns. + if not (defer_end and _defer_tool_end(tool_use_id, span, end_ns)): + span.end() # Clean up subagent span if this launched a subagent if tool_name in SUBAGENT_TOOLS: @@ -442,6 +524,9 @@ def clear_active_tool_spans() -> None: """ global _active_tool_spans, _subagent_spans + # Completed tool spans still waiting for their result end normally, at their hook's time. + end_deferred_tool_spans() + # End any orphaned spans for tool_use_id, (span, start_time, tool_name) in _active_tool_spans.items(): try: diff --git a/python/frameworks/mcp/README.md b/python/frameworks/mcp/README.md index 508874f25..ac7bf5cac 100644 --- a/python/frameworks/mcp/README.md +++ b/python/frameworks/mcp/README.md @@ -66,7 +66,8 @@ and `BlobResourceContents.blob` (in an `EmbeddedResource` or a parts and 256 KiB of decoded bytes per result. `resource_link` blocks, resource URIs and `structuredContent` are never read. A MIME of another modality than the block type, or outside the attachment MIME list, uploads -nothing. `attachments=` overrides the provider's manager; `trace_config=` and +nothing, and so do bytes whose leading signature names another modality than +the declared MIME (a PNG labelled `audio/wav` or `application/octet-stream`). `attachments=` overrides the provider's manager; `trace_config=` and `attachment_capture=` only add restrictions. Wrapping a wrapped session returns it unchanged. `capture_mcp_result(manager, span, result, operation)` is the same capture for an existing span. diff --git a/python/frameworks/smolagents/README.md b/python/frameworks/smolagents/README.md index 720973c55..4a59348c1 100644 --- a/python/frameworks/smolagents/README.md +++ b/python/frameworks/smolagents/README.md @@ -88,8 +88,9 @@ Pass the shared manager: `SmolagentsInstrumentor().instrument(tracer_provider=.. blocks: task images and `ActionStep.observations_images`, such as a browser screenshot a step callback stored) are uploaded on the model span as `model_input`, once per image object (memory replays are not re-uploaded). - When the model span captured an image, provider instrumentors below it are - told the call's media is already owned. + Every model call that carries a captured image, new or replayed, tells + provider instrumentors below it that the call's media is already owned, so + a replayed image is never uploaded again from the provider request. - A tool's returned in-memory PIL image or `AgentImage` is uploaded on the tool span as `tool_result`. diff --git a/python/frameworks/smolagents/traceai_smolagents/_attachments.py b/python/frameworks/smolagents/traceai_smolagents/_attachments.py index 20ebe166b..5058e85b3 100644 --- a/python/frameworks/smolagents/traceai_smolagents/_attachments.py +++ b/python/frameworks/smolagents/traceai_smolagents/_attachments.py @@ -13,16 +13,19 @@ A file-backed ``AgentImage`` or PIL image that is not loaded yet, image URLs, ``image_url`` strings and every ``AgentAudio`` (a path or a tensor) are not read (``source_unavailable``). smolagents replays memory images on every -model call; the same image object is uploaded once per manager. +model call; the same image object is uploaded once per manager, and every +model call that carries a captured image (new or replayed) is marked as the +owner of its media so a provider adapter below never uploads it again. """ +import os import threading import weakref from collections import OrderedDict from types import MappingProxyType from typing import Any, Mapping -__all__ = ("capture_model_images", "capture_tool_image", "smolagents_attachment_capabilities") +__all__ = ("capture_model_images", "capture_tool_image", "owner_context", "smolagents_attachment_capabilities") smolagents_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -40,17 +43,28 @@ class _Seen: """Image objects already uploaded, per manager (identity only; PIL images are unhashable).""" def __init__(self) -> None: + self._reset() + + def _reset(self) -> None: + self._pid = os.getpid() self._lock = threading.Lock() self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() + def _owned_lock(self) -> threading.Lock: + # A forked child gets a fresh lock and table: a lock another thread held at fork + # time would never be released there. No fork hook is registered. + if self._pid != os.getpid(): + self._reset() + return self._lock + def __contains__(self, item) -> bool: manager, image = item - with self._lock: + with self._owned_lock(): entry = self._managers.get(manager, {}).get(id(image)) return entry is not None and entry() is image def add(self, manager, image) -> None: - with self._lock: + with self._owned_lock(): seen = self._managers.get(manager) if seen is None: seen = self._managers[manager] = OrderedDict() @@ -78,7 +92,11 @@ def _in_memory(value: Any) -> Any: def capture_model_images(manager, span, messages: Any, *, trace_config=None, capture=None) -> int: - """Capture the in-memory images of one model call's input messages. Returns the admitted count.""" + """Capture the in-memory images of one model call's input messages. + + Returns the number of images this call owns: admitted now, or already uploaded + (a memory replay). A positive count means the caller marks the model call as owner. + """ try: from fi_instrumentation.attachments.media_capture import active from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia @@ -100,7 +118,10 @@ def capture_model_images(manager, span, messages: Any, *, trace_config=None, cap if _get(part, "type") != "image": continue image = _in_memory(_get(part, "image")) - if image is None or (manager, image) in _SEEN: + if image is None: + continue + if (manager, image) in _SEEN: + admitted += 1 # Replayed: already uploaded, still owned here. continue found = media.image("model/%d/%d" % (message_index, part_index), image, purpose="model_input", role="user", message_index=message_index, part_index=part_index) @@ -112,6 +133,13 @@ def capture_model_images(manager, span, messages: Any, *, trace_config=None, cap return admitted +def owner_context(): + """The current context marked as owner of this call's media (by the shared key name).""" + from fi_instrumentation.attachments.tool_media import with_ownership + + return with_ownership() + + def capture_tool_image(manager, span, response: Any, *, trace_config=None, capture=None) -> int: """Capture a tool's returned in-memory image on its span. Returns 1 when admitted.""" try: diff --git a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py index 49b37074f..5897279b3 100644 --- a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py +++ b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py @@ -418,9 +418,6 @@ def _input_value_and_mime_type( yield INPUT_VALUE, safe_json_dumps(arguments) -_OWNER_KEY = "traceai.auto.attachment_owner.v1" - - class _Media: """The shared attachment manager and its local restrictions (None manager: capture off).""" @@ -461,12 +458,13 @@ def __call__( ) as span: owner_token = None if self._media.manager is not None: - from traceai_smolagents._attachments import capture_model_images + from traceai_smolagents._attachments import capture_model_images, owner_context if capture_model_images(self._media.manager, span, arguments.get("messages"), trace_config=self._media.trace_config, capture=self._media.capture): - # These images are captured here; a provider adapter below must not upload them again. - owner_token = context_api.attach(context_api.set_value(_OWNER_KEY, True)) + # These images are captured here (now or on an earlier call); a provider + # adapter below must not upload them again. + owner_token = context_api.attach(owner_context()) try: output_message = wrapped(*args, **kwargs) finally: diff --git a/python/tests/attachments_mcp_agents/_fake_claude_cli.py b/python/tests/attachments_mcp_agents/_fake_claude_cli.py index 29a56adba..a2bf55a9e 100644 --- a/python/tests/attachments_mcp_agents/_fake_claude_cli.py +++ b/python/tests/attachments_mcp_agents/_fake_claude_cli.py @@ -4,13 +4,16 @@ For every user message it replays one tool call: the assistant ``tool_use``, the PreToolUse and PostToolUse hook callbacks the SDK registered, the ``tool_result`` carrying the media in ``FAKE_CLAUDE_RESULT`` (a JSON file), then a final answer. -``FAKE_CLAUDE_ORDER=result-first`` streams the tool result before PostToolUse. -No model, network or file other than the result file is touched. +``FAKE_CLAUDE_ORDER=result-first`` streams the tool result before PostToolUse; +``no-result`` never streams it. ``FAKE_CLAUDE_GAP`` (seconds) waits between +PostToolUse and the tool result. No model, network or file other than the +result file is touched. """ import json import os import sys +import time if "-v" in sys.argv[1:] or "--version" in sys.argv[1:]: print("2.1.999 (Claude Code)") @@ -19,6 +22,7 @@ with open(os.environ["FAKE_CLAUDE_RESULT"]) as handle: TOOL_RESULT = json.load(handle) ORDER = os.environ.get("FAKE_CLAUDE_ORDER", "hooks-first") +GAP = float(os.environ.get("FAKE_CLAUDE_GAP", "0")) TOOL = "mcp__screen__capture" hooks = {} counter = [0] @@ -67,7 +71,9 @@ def turn(): hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) else: hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) - send(result) + time.sleep(GAP) + if ORDER != "no-result": + send(result) assistant([{"type": "text", "text": "done"}]) send({"type": "result", "subtype": "success", "duration_ms": 1, "duration_api_ms": 1, "is_error": False, "num_turns": 1, "session_id": "s1", "total_cost_usd": 0.0, "result": "done", diff --git a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py index 2b3f4dd86..fc15da106 100644 --- a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py +++ b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py @@ -19,6 +19,7 @@ from fi_instrumentation import TraceConfig # noqa: E402 from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 from traceai_claude_agent_sdk import ClaudeAgentInstrumentor # noqa: E402 +from traceai_claude_agent_sdk._attributes import ClaudeAgentAttributes # noqa: E402 from _harness import PDF, b64, flush, png, refs # noqa: E402 @@ -32,7 +33,7 @@ def image(data, mime="image/png"): @pytest.fixture def cli(tmp_path, monkeypatch): - def make(content, order="hooks-first", is_error=False): + def make(content, order="hooks-first", is_error=False, gap=0.0): result = tmp_path / "result.json" result.write_text(json.dumps({"content": content, "is_error": is_error})) launcher = tmp_path / "claude" @@ -40,6 +41,7 @@ def make(content, order="hooks-first", is_error=False): launcher.chmod(launcher.stat().st_mode | stat.S_IEXEC) monkeypatch.setenv("FAKE_CLAUDE_RESULT", str(result)) monkeypatch.setenv("FAKE_CLAUDE_ORDER", order) + monkeypatch.setenv("FAKE_CLAUDE_GAP", str(gap)) monkeypatch.setenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "1") # Releases without ClaudeAgentOptions.cli_path find the CLI as "claude" on PATH. monkeypatch.setenv("PATH", str(tmp_path) + os.pathsep + os.environ.get("PATH", "")) @@ -99,8 +101,10 @@ def test_tool_result_image_and_pdf_upload_before_truncation_on_the_tool_span(f01 (tool,) = tool_spans(exporter) parents = {u["association"]["parent"]["span_id"] for u in f01.uploads} assert parents == {format(tool.context.span_id, "016x")} - if order == "result-first": - assert len(refs(tool)) == 2 # still open: projected on the span + # In both orders the tool span is still recording when its result arrives, so every + # uploaded occurrence is projected (fi.attachment.refs) on its upload parent. + assert sorted(r["occurrence_id"] for r in refs(tool)) == sorted(u["occurrence_id"] for u in f01.uploads) + assert tool.status.is_ok # Bytes never reach a text attribute (tool output, conversation output). for span in exporter.get_finished_spans(): for value in span.attributes.values(): @@ -108,6 +112,35 @@ def test_tool_result_image_and_pdf_upload_before_truncation_on_the_tool_span(f01 manager.shutdown(5) +def _duration_ms(span): + return (span.end_time - span.start_time) / 1e6 + + +def test_hooks_first_tool_span_keeps_the_hook_end_time_while_waiting_for_its_result(f01, instrument, cli): + manager = f01.manager() + exporter = instrument(manager) + converse(cli([image(SHOT)], gap=0.6)) + flush(manager) + (tool,) = tool_spans(exporter) + assert len(refs(tool)) == 1 and len(f01.uploads) == 1 + hook_ms = tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS] + # Ended at the PostToolUse time, not when the result streamed 600 ms later. + assert abs(_duration_ms(tool) - hook_ms) < 150 and _duration_ms(tool) < 450 + manager.shutdown(5) + + +def test_a_result_that_never_arrives_still_ends_the_tool_span_at_its_hook_time(f01, instrument, cli): + manager = f01.manager() + exporter = instrument(manager) + messages = converse(cli([image(SHOT)], order="no-result", gap=0.3)) + flush(manager) + assert type(messages[-1]).__name__ == "ResultMessage" + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + manager.shutdown(5) + + def test_url_and_file_sources_never_fetch(f01, instrument, cli): manager = f01.manager() instrument(manager) diff --git a/python/tests/attachments_mcp_agents/test_smolagents_attachments.py b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py index f2fc93f5a..1762d5a49 100644 --- a/python/tests/attachments_mcp_agents/test_smolagents_attachments.py +++ b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py @@ -162,3 +162,70 @@ def test_disabled_manager_uploads_nothing(f01, instrument, provider): instrument(create_attachment_manager(config=AttachmentConfig())) call(model(provider), [user_message(picture(7))]) assert f01.uploads == [] + + +def test_every_model_call_carrying_a_captured_image_is_marked_owner(f01, instrument, provider, monkeypatch): + """A replayed memory image is already uploaded: the provider adapter below must not upload it again.""" + import openai.resources.chat.completions as completions + from fi_instrumentation.attachments import tool_media + from opentelemetry import context as otel_context + + marks = [] + real = completions.Completions.create + + def create(self, *args, **kwargs): + marks.append(otel_context.get_value(tool_media.OWNER_KEY)) + return real(self, *args, **kwargs) + + monkeypatch.setattr(completions.Completions, "create", create) + manager = f01.manager() + instrument(manager) + shot = picture(11) + m = model(provider) + call(m, [user_message(shot)]) + call(m, [user_message(shot)]) # memory replay: the same image object goes to the model again + call(m, [user_message()]) # no image: no mark + flush(manager) + assert marks == [True, True, None] + assert len(f01.uploads) == 1 + assert sum(raw.count(b"data:image/png;base64,") for _, _, raw in provider.requests) == 2 + manager.shutdown(5) + + +def test_seen_table_is_fresh_in_a_forked_child_even_if_its_lock_was_held(): + import os + import signal + import threading + + from traceai_smolagents import _attachments as sa + + if not hasattr(os, "fork"): + pytest.skip("no fork") + held, release = threading.Event(), threading.Event() + + def holder(): + with sa._SEEN._owned_lock(): + held.set() + release.wait(5) + + thread = threading.Thread(target=holder) + thread.start() + held.wait(5) + + class Manager: + pass + + keep = Manager() + pid = os.fork() + if pid == 0: # child: must not block on the lock the parent's thread held at fork + signal.alarm(5) + try: + image = picture(1) + sa._SEEN.add(keep, image) + os._exit(0 if (keep, image) in sa._SEEN else 4) + except BaseException: + os._exit(3) + release.set() + thread.join() + _, status = os.waitpid(pid, 0) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 diff --git a/python/tests/attachments_mcp_agents/test_tool_media.py b/python/tests/attachments_mcp_agents/test_tool_media.py index d3125549f..dddfec99b 100644 --- a/python/tests/attachments_mcp_agents/test_tool_media.py +++ b/python/tests/attachments_mcp_agents/test_tool_media.py @@ -106,3 +106,36 @@ def test_pil_images_are_encoded_only_when_in_memory(f01, spans, tmp_path): flush(manager) assert [u["mime_type"] for u in f01.uploads] == ["image/png"] manager.shutdown(5) + + +def test_signature_of_another_modality_is_denied_and_never_relabelled(f01, spans): + """A PNG declared as audio, a generic file or a PDF is not uploaded (hide_input_images cannot be escaped).""" + from _harness import PDF, wav + + manager = f01.manager() + provider, _ = spans + shot = png(seed=21) + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_input_images=True)) + assert media.base64("a", lambda: b64(shot), "audio/wav", purpose="tool_result", expect="audio") is None + assert media.base64("b", lambda: b64(shot), "application/octet-stream", purpose="tool_result") is None + assert media.base64("c", lambda: b64(shot), "application/pdf", purpose="tool_result") is None + assert media.data_uri("d", lambda: "data:audio/wav;base64," + b64(shot), ("audio/wav",), + purpose="tool_result") is None + # Matching signatures and signature-less bytes still upload. + assert media.base64("e", lambda: b64(wav()), "audio/wav", purpose="tool_result", expect="audio") + assert media.base64("f", lambda: b64(PDF), "application/pdf", purpose="tool_result") + flush(manager) + assert shot not in f01.payloads + assert sorted(u["mime_type"] for u in f01.uploads) == ["application/pdf", "audio/wav"] + manager.shutdown(5) + + +@pytest.mark.parametrize("head,kind", [ + (b"\x89PNG\r\n\x1a\n\x00\x00\x00\x00", "image"), (b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01", "image"), + (b"RIFF\x00\x00\x00\x00WEBP", "image"), (b"RIFF\x00\x00\x00\x00WAVE", "audio"), (b"OggS\x00\x02", "audio"), + (b"fLaC\x00\x00", "audio"), (b"ID3\x04\x00", "audio"), (b"%PDF-1.7\n", "document"), + (b"\x00\x00\x00\x18ftypmp42", None), (b"\x1aE\xdf\xa3\x9fB\x86", None), (b"RIFF\x00\x00\x00\x00AVI ", None), + (b"plain text", None)]) +def test_signature_kind(head, kind): + assert tool_media.signature_kind(head) == kind From 92939ee9922c1d455a31756ae8ddf70a0214a3f9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:56:25 -0800 Subject: [PATCH 11/16] test(python): MCP error results follow hide_error_details Co-Authored-By: Claude Opus 5.5 --- .../test_mcp_attachments.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/tests/attachments_mcp_agents/test_mcp_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_attachments.py index fd5a3edc3..155cef0e3 100644 --- a/python/tests/attachments_mcp_agents/test_mcp_attachments.py +++ b/python/tests/attachments_mcp_agents/test_mcp_attachments.py @@ -223,3 +223,22 @@ def get(self, key, default=None): assert traceai_mcp.capture_mcp_result(manager, span, object()) == {} assert traceai_mcp.capture_mcp_result(None, span, {"content": []}) == {} manager.shutdown(5) + + +def test_error_results_follow_hide_error_details(f01, spans): + """An isError result's media is error detail: hide_error_details denies it, a success result still uploads.""" + manager = f01.manager() + provider, _ = spans + shot, ok = server.png(31), server.png(32) + + def result(data, error): + return {"isError": error, "content": [{"type": "image", "data": server.b64(data), "mimeType": "image/png"}]} + + config = TraceConfig(hide_error_details=True) + with provider.get_tracer("t").start_as_current_span("failed") as span: + assert traceai_mcp.capture_mcp_result(manager, span, result(shot, True), trace_config=config) == {} + with provider.get_tracer("t").start_as_current_span("succeeded") as span: + assert len(traceai_mcp.capture_mcp_result(manager, span, result(ok, False), trace_config=config)) == 1 + flush(manager) + assert f01.payloads == [ok] + manager.shutdown(5) From c53acf38e365f51589205af04a8e0e30a78ada27 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:19:40 -0800 Subject: [PATCH 12/16] feat(python): auto registry and report rows for provider and framework media adapters - _MEDIA_ADAPTERS gains google_genai, vertexai, bedrock, litellm, fireworks, xai, mistralai, openrouter, portkey, ollama, vllm and minimax (lazy descriptors from provider_media), plus langchain, llamaindex, agno and pydantic_ai. together, deepseek and cerebras stay unclaimed, and a descriptor with no operation is reported unqualified. - auto_instrument accepts openrouter, portkey, vllm, minimax, pydantic_ai and agno. Agno gets its own keyword arguments: no fi TraceConfig or attachment_* alias, auto's hide flags as an OpenInference TraceConfig, attachments only while capturing. - An instrumentor that skipped instrumentation on a dependency conflict is reported dependency_unavailable instead of registered. - Real-SDK tests: report rows equal each installed descriptor; LiteLLM over the OpenAI instrumentor uploads once through auto; OFF never reaches F01; Agno and Pydantic AI through auto. Co-Authored-By: Claude Opus 5.5 --- .../test_auto_framework_media.py | 204 ++++++++++++++++ .../attachments_providers/test_auto_init.py | 10 +- .../test_auto_provider_media.py | 227 ++++++++++++++++++ python/traceai/__init__.py | 67 +++++- python/traceai/tests/test_attachments.py | 196 ++++++++++++++- 5 files changed, 699 insertions(+), 5 deletions(-) create mode 100644 python/tests/attachments_providers/test_auto_framework_media.py create mode 100644 python/tests/attachments_providers/test_auto_provider_media.py diff --git a/python/tests/attachments_providers/test_auto_framework_media.py b/python/tests/attachments_providers/test_auto_framework_media.py new file mode 100644 index 000000000..09c71c561 --- /dev/null +++ b/python/tests/attachments_providers/test_auto_framework_media.py @@ -0,0 +1,204 @@ +"""Auto with the Agno and Pydantic AI media adapters (#691): real frameworks, loopback F01. + +traceai.init(auto_instrument=["agno"] / ["pydantic_ai"]) must hand each framework the ONE +shared manager in the keyword arguments it accepts, carry auto's hide flags, and stay a strict +no-op with attachments OFF. Skips when the installed framework version has no media adapter, +unless TRACEAI_EXPECT_FRAMEWORK_ADAPTERS=1, which turns the skip into a failure. +""" + +import importlib +import json +import os + +import pytest + +traceai = pytest.importorskip("traceai") + +from conftest import flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +EXPECT = os.environ.get("TRACEAI_EXPECT_FRAMEWORK_ADAPTERS") == "1" +IMAGE = png(seed=81) +PROMPT = "describe this synthetic picture" + + +def _adapter(path, name): + try: + return list(getattr(importlib.import_module(path), name)["operations"]) + except Exception: + if EXPECT: + raise + pytest.skip("%s has no media adapter in this version" % path) + + +@pytest.fixture +def auto_state(monkeypatch): + for name, value in (("_tracer_provider", None), ("_init_options", None), ("_instrumented", set()), + ("_adapter_status", {}), ("_attachment_manager_borrowed", False)): + monkeypatch.setattr(traceai, name, value) + monkeypatch.setenv("AGNO_TELEMETRY", "false") + for key in ("OPENINFERENCE_HIDE_INPUTS", "OPENINFERENCE_HIDE_INPUT_IMAGES", "OPENINFERENCE_HIDE_OUTPUTS"): + monkeypatch.delenv(key, raising=False) + yield + try: + from traceai_agno import AgnoInstrumentorWrapper + + AgnoInstrumentorWrapper().uninstrument() + except ImportError: + pass + try: + from traceai_pydantic_ai import PydanticAIInstrumentor + + instrumentor = PydanticAIInstrumentor() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + except ImportError: + pass + + +def _init(f01, monkeypatch, libraries, enabled=True, **options): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + if enabled: + options["attachment_config"] = AttachmentConfig(enabled=True, endpoint=f01.endpoint) + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=libraries, + verbose=False, set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + return tracer_provider, exporter + + +def _exported(exporter): + return json.dumps([dict(span.attributes or {}) for span in exporter.get_finished_spans()], default=str) + + +# ------------------------------------------------------------------------------------- Agno +def _agno_run(provider): + pytest.importorskip("agno") + pytest.importorskip("openinference.instrumentation.agno") + from agno.agent import Agent + from agno.media import Image + from agno.models.openai import OpenAIChat + + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + try: + agent = Agent(model=model, telemetry=False) + except TypeError: # older Agno has no telemetry switch; AGNO_TELEMETRY=false covers it + agent = Agent(model=model) + agent.run(PROMPT, images=[Image(content=IMAGE, format="png")]) + + +def test_auto_agno_uploads_the_image_on_the_model_span(f01, provider, auto_state, monkeypatch): + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + row = {r["name"]: r for r in traceai.report()["integrations"]}["agno"] + assert (row["status"], row["media"]) == ("registered", "provider_adapter") + _agno_run(provider) + flush(manager) + assert f01.payloads == [IMAGE] + owners = [span.name for span in exporter.get_finished_spans() for _ in refs(span)] + assert len(owners) == 1 and owners[0].endswith((".invoke", ".ainvoke")), owners + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_agno_off_is_a_no_op(f01, provider, auto_state, monkeypatch): + pytest.importorskip("agno") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], enabled=False) + try: + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + _agno_run(provider) + assert f01.state["requests"] == [] + assert exporter.get_finished_spans() and all(not refs(s) for s in exporter.get_finished_spans()) + assert PROMPT in _exported(exporter) # tracing itself is unchanged + finally: + tracer_provider.shutdown() + + +def test_auto_agno_honours_auto_hide_flags(f01, provider, auto_state, monkeypatch): + """Auto's fi TraceConfig reaches Agno's OpenInference config: input text AND image stay out.""" + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], trace_config=TraceConfig(hide_inputs=True)) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + _agno_run(provider) + flush(manager) + assert f01.payloads == [] + assert exporter.get_finished_spans() + assert PROMPT not in _exported(exporter) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +# ------------------------------------------------------------------------------ Pydantic AI +def _pydantic_ai_run(): + pytest.importorskip("pydantic_ai") + from pydantic_ai import Agent, BinaryContent + from pydantic_ai.messages import ModelResponse, TextPart + from pydantic_ai.models.function import FunctionModel + + def echo(messages, info): + return ModelResponse(parts=[TextPart("ok")]) + + Agent(FunctionModel(echo)).run_sync([PROMPT, BinaryContent(data=IMAGE, media_type="image/png")]) + + +def _pydantic_ai_installed_by_name(): + """BaseInstrumentor requires the "pydantic-ai" distribution; pydantic-ai-slim alone fails its check.""" + from importlib.metadata import PackageNotFoundError, version + + try: + version("pydantic-ai") + return True + except PackageNotFoundError: + return False + + +def test_auto_pydantic_ai_uploads_the_image_or_reports_the_dependency_gap(f01, auto_state, monkeypatch): + pytest.importorskip("pydantic_ai") + _adapter("traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["pydantic-ai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + row = {r["name"]: r for r in traceai.report()["integrations"]}["pydantic_ai"] + if not _pydantic_ai_installed_by_name(): + # Accurate, not "registered": nothing was instrumented. + assert row["status"] == "dependency_unavailable" + _pydantic_ai_run() + flush(manager) + assert f01.payloads == [] and not exporter.get_finished_spans() + return + assert (row["status"], row["media"]) == ("registered", "provider_adapter") + _pydantic_ai_run() + flush(manager) + assert f01.payloads == [IMAGE] + assert len([s for s in exporter.get_finished_spans() if refs(s)]) == 1 + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_pydantic_ai_off_is_a_no_op(f01, auto_state, monkeypatch): + pytest.importorskip("pydantic_ai") + if not _pydantic_ai_installed_by_name(): + pytest.skip("pydantic-ai-slim only: auto cannot instrument it (dependency check)") + tracer_provider, exporter = _init(f01, monkeypatch, ["pydantic-ai"], enabled=False) + try: + _pydantic_ai_run() + assert f01.state["requests"] == [] + assert exporter.get_finished_spans() and all(not refs(s) for s in exporter.get_finished_spans()) + finally: + tracer_provider.shutdown() diff --git a/python/tests/attachments_providers/test_auto_init.py b/python/tests/attachments_providers/test_auto_init.py index cd70f8880..973537ae2 100644 --- a/python/tests/attachments_providers/test_auto_init.py +++ b/python/tests/attachments_providers/test_auto_init.py @@ -143,7 +143,15 @@ def test_report_rows_for_installed_media_adapters(auto_state): rows = {row["name"]: row for row in traceai.report()["integrations"]} assert {name: rows[name]["media"] for name in ("openai", "anthropic", "groq")} == dict.fromkeys( ("openai", "anthropic", "groq"), "provider_adapter") - assert rows["litellm"]["media"] == "unqualified" + # traceai_litellm is a media adapter only in a version that declares the descriptor, on a core with + # provider_media; otherwise (or when it is not installed) its row is unqualified. + try: + import traceai_litellm + + declared = list(traceai_litellm.litellm_attachment_capabilities["operations"]) + except Exception: + declared = [] + assert rows["litellm"]["media"] == ("provider_adapter" if declared else "unqualified") def test_auto_borrowed_manager_is_shared_and_tightened(f01, provider, auto_state, monkeypatch): diff --git a/python/tests/attachments_providers/test_auto_provider_media.py b/python/tests/attachments_providers/test_auto_provider_media.py new file mode 100644 index 000000000..666844150 --- /dev/null +++ b/python/tests/attachments_providers/test_auto_provider_media.py @@ -0,0 +1,227 @@ +"""Auto with the provider-media (#697) and framework (#691) adapters: real SDKs, loopback F01. + +- report() rows equal each INSTALLED integration's own descriptor, and every integration + whose installed version declares none is unqualified; +- LiteLLM over the OpenAI instrumentor, both enabled through traceai.init, uploads the + call's image ONCE, on the LiteLLM span (LiteLLM's owner mark governs the nested call); +- with attachments OFF, the same call never reaches F01. + +Rows that need a core with ``provider_media`` skip on an older core unless +TRACEAI_EXPECT_PROVIDER_ADAPTERS=1, which turns the skip into a failure. +""" + +import importlib +import importlib.util +import os + +import pytest + +traceai = pytest.importorskip("traceai") + +from conftest import b64, flush, png, refs # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +EXPECT = os.environ.get("TRACEAI_EXPECT_PROVIDER_ADAPTERS") == "1" + +# report row name -> (instrumentor package, descriptor module, descriptor name). Independent of +# traceai._MEDIA_ADAPTERS on purpose: a wrong module or name there fails these rows. +PROVIDERS = { + "google_genai": ("traceai_google_genai", "traceai_google_genai", "google_genai_attachment_capabilities"), + "vertexai": ("traceai_vertexai", "traceai_vertexai", "vertexai_attachment_capabilities"), + "bedrock": ("traceai_bedrock", "traceai_bedrock", "bedrock_attachment_capabilities"), + "litellm": ("traceai_litellm", "traceai_litellm", "litellm_attachment_capabilities"), + "groq": ("traceai_groq", "traceai_groq._audio", "groq_attachment_capabilities"), + "fireworks": ("traceai_fireworks", "traceai_fireworks", "fireworks_attachment_capabilities"), + "xai": ("traceai_xai", "traceai_xai", "xai_attachment_capabilities"), + "mistralai": ("traceai_mistralai", "traceai_mistralai", "mistralai_attachment_capabilities"), + "openrouter": ("traceai_openrouter", "traceai_openrouter", "openrouter_attachment_capabilities"), + "portkey": ("traceai_portkey", "traceai_portkey", "portkey_attachment_capabilities"), + "ollama": ("traceai_ollama", "traceai_ollama", "ollama_attachment_capabilities"), + "vllm": ("traceai_vllm", "traceai_vllm", "vllm_attachment_capabilities"), + "minimax": ("traceai_minimax", "traceai_minimax", "minimax_attachment_capabilities"), +} +FRAMEWORKS = { + "agno": ("traceai_agno", "traceai_agno._attachments", "agno_attachment_capabilities"), + "pydantic_ai": ("traceai_pydantic_ai", "traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities"), + "langchain": ("traceai_langchain", "traceai_langchain._attachments", "langchain_attachment_capabilities"), + "llamaindex": ("traceai_llamaindex", "traceai_llamaindex._attachments", "llamaindex_attachment_capabilities"), +} +UNCLAIMED = {"together": "traceai_together", "deepseek": "traceai_deepseek", "cerebras": "traceai_cerebras"} + + +def _installed(module): + try: + return importlib.util.find_spec(module) is not None + except (ImportError, ValueError): + return False + + +def _declared(path, name): + """The installed version's own descriptor operations, or None when it declares none.""" + try: + return list(getattr(importlib.import_module(path), name)["operations"]) + except Exception: + return None + + +@pytest.fixture +def provider_media(): + try: + return importlib.import_module("fi_instrumentation.attachments.provider_media") + except ImportError: + if EXPECT: + raise + pytest.skip("this core has no provider_media") + + +@pytest.fixture +def auto_state(monkeypatch): + for name, value in (("_tracer_provider", None), ("_init_options", None), ("_instrumented", set()), + ("_adapter_status", {}), ("_attachment_manager_borrowed", False)): + monkeypatch.setattr(traceai, name, value) + yield + for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_litellm", "LiteLLMInstrumentor")): + try: + instrumentor = getattr(importlib.import_module(module), name)() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + except ImportError: + pass + + +def _rows(): + return {row["name"]: row for row in traceai.report()["integrations"]} + + +def test_report_rows_equal_each_installed_descriptor(provider_media, auto_state): + rows, checked = _rows(), [] + for name, (package, path, attr) in {**PROVIDERS, **FRAMEWORKS}.items(): + row = rows[name] + assert row["package"] == package + if not _installed(package): + assert (row["status"], row["media"]) == ("not_installed", "unqualified"), name + continue + declared = _declared(path, attr) + if not declared: + assert row["media"] == "unqualified" and "media_operations" not in row, name + continue + assert (row["media"], row["media_operations"]) == ("provider_adapter", declared), name + if name in PROVIDERS and name != "groq": + # The package's lazy descriptor IS the core's matrix row for it. + assert declared == list(provider_media.capabilities(package)["operations"]), name + checked.append(name) + for name, package in UNCLAIMED.items(): + assert rows[name]["media"] == "unqualified", name + assert list(provider_media.capabilities(package)["operations"]) == [], name + # A VM run names the rows it installed, so a silently missing package cannot pass. + expected = [n for n in os.environ.get("TRACEAI_EXPECT_MEDIA_ROWS", "").split(",") if n] + assert sorted(set(expected) - set(checked)) == [], checked + + +def test_rows_without_a_declared_descriptor_are_unqualified(auto_state): + """Runs on every core: an installed version that declares no descriptor is unqualified.""" + rows = _rows() + for name, (package, path, attr) in {**PROVIDERS, **FRAMEWORKS}.items(): + if _installed(package) and not _declared(path, attr): + assert rows[name]["media"] == "unqualified", name + for name in UNCLAIMED: + assert rows[name]["media"] == "unqualified", name + + +def _init(f01, monkeypatch, libraries, enabled=True): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + options = {"attachment_config": AttachmentConfig(enabled=True, endpoint=f01.endpoint)} if enabled else {} + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=libraries, + verbose=False, set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + return tracer_provider, exporter + + +def _litellm_image(litellm, provider, image): + litellm.completion(model="openai/gpt-synthetic", api_base=provider.url + "/v1", api_key="sk-synthetic", + messages=[{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}]}]) + + +def _litellm(provider_media): + litellm = pytest.importorskip("litellm") + pytest.importorskip("traceai_openai") + traceai_litellm = pytest.importorskip("traceai_litellm") + if not _declared("traceai_litellm", "litellm_attachment_capabilities"): + if EXPECT: + pytest.fail("traceai_litellm declares no media descriptor") + pytest.skip("this traceai_litellm has no media adapter") + return litellm, traceai_litellm + + +def _sent_image(provider, image): + return [path for path, body, raw in provider.requests if b64(image).encode() in raw] + + +@pytest.mark.parametrize("libraries", [["openai", "litellm"], ["litellm", "openai"]]) +def test_auto_litellm_over_openai_uploads_once(f01, provider, provider_media, auto_state, monkeypatch, libraries): + litellm, _ = _litellm(provider_media) + tracer_provider, exporter = _init(f01, monkeypatch, libraries) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + rows = _rows() + assert (rows["litellm"]["status"], rows["litellm"]["media"]) == ("registered", "provider_adapter") + assert (rows["openai"]["status"], rows["openai"]["media"]) == ("registered", "provider_adapter") + image = png(seed=71) + _litellm_image(litellm, provider, image) + flush(manager) + assert len(_sent_image(provider, image)) == 1 # the nested OpenAI SDK call sent the image + assert f01.payloads == [image] + owners = [span.name for span in exporter.get_finished_spans() for _ in refs(span)] + assert owners == ["completion"], owners + # The OpenAI instrumentor traced the nested call, and captured nothing for it. + nested = [span for span in exporter.get_finished_spans() if span.name != "completion"] + assert nested and all(not refs(span) for span in nested), [s.name for s in exporter.get_finished_spans()] + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_control_openai_alone_captures_the_litellm_call(f01, provider, provider_media, auto_state, monkeypatch): + """Without the LiteLLM instrumentor, the OpenAI one owns the same bytes: the claim above matters.""" + litellm, _ = _litellm(provider_media) + tracer_provider, exporter = _init(f01, monkeypatch, ["openai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + image = png(seed=72) + _litellm_image(litellm, provider, image) + flush(manager) + assert f01.payloads == [image] + owners = [span.name for span in exporter.get_finished_spans() for _ in refs(span)] + assert len(owners) == 1 and owners != ["completion"], owners + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +@pytest.mark.parametrize("off", ["default", "kill_switch"]) +def test_auto_litellm_over_openai_off_never_reaches_f01(f01, provider, provider_media, auto_state, monkeypatch, + off): + litellm, _ = _litellm(provider_media) + if off == "kill_switch": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + tracer_provider, exporter = _init(f01, monkeypatch, ["openai", "litellm"], enabled=(off == "kill_switch")) + try: + assert traceai.report()["attachments"]["state"] == "disabled" + image = png(seed=73) + _litellm_image(litellm, provider, image) + assert len(_sent_image(provider, image)) == 1 + assert f01.state["requests"] == [] # not even a session bootstrap + assert all(not refs(span) for span in exporter.get_finished_spans()) + assert {span.name for span in exporter.get_finished_spans()} >= {"completion"} + finally: + tracer_provider.shutdown() diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 441cbe043..6f13f0190 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -69,6 +69,19 @@ def _env_disabled() -> bool: "traceai_claude_agent_sdk": ("traceai_claude_agent_sdk._attachments", "claude_agent_attachment_capabilities"), "traceai_smolagents": ("traceai_smolagents._attachments", "smolagents_attachment_capabilities"), "traceai_firecrawl": ("traceai_firecrawl._attachments", "firecrawl_attachment_capabilities"), + # Provider media (shared provider_media walkers). Each package resolves its descriptor lazily from + # fi_instrumentation.attachments.provider_media, so an installed version without the binding, or a core + # without provider_media, reports unqualified. together, deepseek and cerebras claim no media operation + # and are deliberately absent (their rows stay unqualified). + **{module: (module, module[len("traceai_"):] + "_attachment_capabilities") for module in ( + "traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_litellm", "traceai_fireworks", + "traceai_xai", "traceai_mistralai", "traceai_openrouter", "traceai_portkey", "traceai_ollama", + "traceai_vllm", "traceai_minimax")}, + # Framework media. + "traceai_langchain": ("traceai_langchain._attachments", "langchain_attachment_capabilities"), + "traceai_llamaindex": ("traceai_llamaindex._attachments", "llamaindex_attachment_capabilities"), + "traceai_agno": ("traceai_agno._attachments", "agno_attachment_capabilities"), + "traceai_pydantic_ai": ("traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities"), } # Voice integrations capture only through an explicit session binding (bind_* below). _SESSION_BINDINGS = {"traceai_pipecat": "bind_audio_processor", "traceai_livekit": "bind_session"} @@ -97,6 +110,10 @@ def _env_disabled() -> bool: "ollama": "traceai_ollama", "google_genai": "traceai_google_genai", "google-genai": "traceai_google_genai", + "openrouter": "traceai_openrouter", + "portkey": "traceai_portkey", + "vllm": "traceai_vllm", + "minimax": "traceai_minimax", # Framework instrumentors "langchain": "traceai_langchain", "llamaindex": "traceai_llamaindex", @@ -111,6 +128,9 @@ def _env_disabled() -> bool: "mcp": "traceai_mcp", "mcp_use": "traceai_mcp_use", "mcp-use": "traceai_mcp_use", + "agno": "traceai_agno", + "pydantic_ai": "traceai_pydantic_ai", + "pydantic-ai": "traceai_pydantic_ai", # Agent frameworks and tools "openai_agents": "traceai_openai_agents", "openai-agents": "traceai_openai_agents", @@ -138,6 +158,11 @@ def _env_disabled() -> bool: "traceai_guardrails": "GuardrailsInstrumentor", "traceai_mcp": "MCPInstrumentor", "traceai_openai_agents": "OpenAIAgentsInstrumentor", "traceai_claude_agent_sdk": "ClaudeAgentInstrumentor", "traceai_smolagents": "SmolagentsInstrumentor", "traceai_firecrawl": "FirecrawlInstrumentor", + "traceai_openrouter": "OpenRouterInstrumentor", "traceai_portkey": "PortkeyInstrumentor", + "traceai_vllm": "VLLMInstrumentor", "traceai_minimax": "MiniMaxInstrumentor", + "traceai_pydantic_ai": "PydanticAIInstrumentor", + # A process-wide wrapper over OpenInference Agno, with its own keyword arguments (_agno_kwargs). + "traceai_agno": "AgnoInstrumentorWrapper", } _tracer_provider: Optional[TracerProvider] = None @@ -320,7 +345,9 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: instrumentor = instrumentor_class() provider = _tracer_provider kwargs = dict(tracer_provider=provider, config=getattr(provider, "trace_config", None)) - if _ATTACHMENTS_SUPPORTED: + if instrumentor_module == "traceai_agno": + kwargs = _agno_kwargs(provider) + elif _ATTACHMENTS_SUPPORTED: manager = get_attachment_manager() if _capturing(manager): # The ONE shared manager, as ``attachments`` (``attachment_manager`` is the @@ -330,6 +357,13 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: # Attachments OFF: exactly the keyword arguments auto passed before. kwargs.update(attachment_manager=manager, attachment_policy=getattr(manager, "policy", None)) instrumentor.instrument(**kwargs) + if getattr(instrumentor, "is_instrumented_by_opentelemetry", True) is False: + # BaseInstrumentor returns without instrumenting on a dependency conflict (for + # example pydantic-ai-slim for a "pydantic-ai" requirement): never report it registered. + _adapter_status[lib_name_lower] = "dependency_unavailable" + if verbose: + print(f"Warning: {lib_name} was not instrumented (dependency check failed)") + return False _instrumented.add(instrumentor_module) _adapter_status[lib_name_lower] = "registered" @@ -350,6 +384,29 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: return False +def _agno_kwargs(provider) -> Dict[str, Any]: + """Keyword arguments for ``AgnoInstrumentorWrapper.instrument``. + + It takes OpenInference's TraceConfig (not fi's) and no ``attachment_*`` alias. Auto's + hide flags are carried over, and only when one is set, so an unrestricted setup keeps + Agno's own default and environment policy. ``attachments`` is passed only while capturing. + """ + kwargs: Dict[str, Any] = {"tracer_provider": provider} + trace_config = getattr(provider, "trace_config", None) + hidden = {field.name: True for field in dataclasses.fields(TraceConfig) + if field.name.startswith("hide_") and getattr(trace_config, field.name, False) is True} + if hidden: + from openinference.instrumentation import TraceConfig as OpenInferenceTraceConfig + + names = {field.name for field in dataclasses.fields(OpenInferenceTraceConfig)} + kwargs["config"] = OpenInferenceTraceConfig(**{name: True for name in hidden if name in names}) + if _ATTACHMENTS_SUPPORTED: + manager = get_attachment_manager() + if _capturing(manager): + kwargs["attachments"] = manager + return kwargs + + def get_tracer_provider() -> Optional[TracerProvider]: """Get the current tracer provider, if initialized.""" return _tracer_provider @@ -447,6 +504,8 @@ def instrument(*libraries: str, verbose: bool = True) -> None: "OpenAI and Groq transcription/translation audio the SDK holds as bytes", "Anthropic Messages base64 images and PDFs, and computer-use tool_result screenshots (traceai_anthropic)", "Pipecat and LiveKit recordings, only through an explicit bind_* call", + "for every other integration, exactly the media_operations of its provider_adapter row below " + "(the installed instrumentor version decides)", ) _AUTO_UNSUPPORTED = ( "integrations whose media row is 'unqualified'", @@ -500,7 +559,11 @@ def _media_support(module: str) -> Dict[str, Any]: if module in _MEDIA_ADAPTERS: path, name = _MEDIA_ADAPTERS[module] descriptor = getattr(importlib.import_module(path), name) - return {"media": "provider_adapter", "media_operations": list(descriptor["operations"])} + operations = list(descriptor["operations"]) + if operations: + return {"media": "provider_adapter", "media_operations": operations} + # A descriptor that claims no operation (tracing only) is not a media adapter. + return {"media": "unqualified"} if module in _SESSION_BINDINGS and callable(getattr(importlib.import_module(module), _SESSION_BINDINGS[module], None)): return {"media": "session_binding"} diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index ba2660091..1c33105ce 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -337,9 +337,20 @@ def test_mcp_and_agent_packages_are_registered_with_explicit_entry_points(): assert "traceai_mcp_use" in traceai._EXPLICIT_ONLY -@pytest.mark.parametrize("name", sorted(_AGENT_INSTRUMENTORS)) +# Provider and framework instrumentors newly reachable from auto (#697 and #691). They take +# the generic keyword arguments; Agno has its own (see test_agno_*). +_PROVIDER_FRAMEWORK_INSTRUMENTORS = { + "openrouter": ("traceai_openrouter", "OpenRouterInstrumentor"), + "portkey": ("traceai_portkey", "PortkeyInstrumentor"), + "vllm": ("traceai_vllm", "VLLMInstrumentor"), + "minimax": ("traceai_minimax", "MiniMaxInstrumentor"), + "pydantic_ai": ("traceai_pydantic_ai", "PydanticAIInstrumentor"), +} + + +@pytest.mark.parametrize("name", sorted({**_AGENT_INSTRUMENTORS, **_PROVIDER_FRAMEWORK_INSTRUMENTORS})) def test_agent_instrumentors_get_the_shared_manager_and_the_off_kwargs_when_off(monkeypatch, name): - module_name, entry = _AGENT_INSTRUMENTORS[name] + module_name, entry = {**_AGENT_INSTRUMENTORS, **_PROVIDER_FRAMEWORK_INSTRUMENTORS}[name] for env, expected in ((None, "on"), ("false", "off")): if env is None: monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) @@ -473,3 +484,184 @@ def test_explicit_entry_points_need_init(monkeypatch): traceai.mcp_use_callback() with pytest.raises(RuntimeError): traceai.wrap_mcp_client(object()) + + +# ------------------------------------- provider (#697) and framework (#691) media adapters +# report row name -> (descriptor module, descriptor name). The provider packages define the +# descriptor as a lazy module attribute; the framework packages in their _attachments module. +_PROVIDER_FRAMEWORK_DESCRIPTORS = { + "google_genai": ("traceai_google_genai", "google_genai_attachment_capabilities"), + "vertexai": ("traceai_vertexai", "vertexai_attachment_capabilities"), + "bedrock": ("traceai_bedrock", "bedrock_attachment_capabilities"), + "litellm": ("traceai_litellm", "litellm_attachment_capabilities"), + "groq": ("traceai_groq._audio", "groq_attachment_capabilities"), + "fireworks": ("traceai_fireworks", "fireworks_attachment_capabilities"), + "xai": ("traceai_xai", "xai_attachment_capabilities"), + "mistralai": ("traceai_mistralai", "mistralai_attachment_capabilities"), + "openrouter": ("traceai_openrouter", "openrouter_attachment_capabilities"), + "portkey": ("traceai_portkey", "portkey_attachment_capabilities"), + "ollama": ("traceai_ollama", "ollama_attachment_capabilities"), + "vllm": ("traceai_vllm", "vllm_attachment_capabilities"), + "minimax": ("traceai_minimax", "minimax_attachment_capabilities"), + "langchain": ("traceai_langchain._attachments", "langchain_attachment_capabilities"), + "llamaindex": ("traceai_llamaindex._attachments", "llamaindex_attachment_capabilities"), + "agno": ("traceai_agno._attachments", "agno_attachment_capabilities"), + "pydantic_ai": ("traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities"), +} +# Instrumented, but their descriptors claim no media operation: never a media adapter. +_UNCLAIMED = {"together": "traceai_together", "deepseek": "traceai_deepseek", "cerebras": "traceai_cerebras"} + + +def test_provider_and_framework_packages_are_registered_with_explicit_entry_points(): + for name, (module, entry) in _PROVIDER_FRAMEWORK_INSTRUMENTORS.items(): + assert traceai._INSTRUMENTOR_REGISTRY[name] == module + assert traceai._ENTRY_POINTS[module] == entry + assert traceai._INSTRUMENTOR_REGISTRY["pydantic-ai"] == "traceai_pydantic_ai" + assert traceai._INSTRUMENTOR_REGISTRY["agno"] == "traceai_agno" + assert traceai._ENTRY_POINTS["traceai_agno"] == "AgnoInstrumentorWrapper" + for name, (path, attr) in _PROVIDER_FRAMEWORK_DESCRIPTORS.items(): + module = traceai._INSTRUMENTOR_REGISTRY[name] + assert traceai._MEDIA_ADAPTERS[module] == (path, attr), name + for module in _UNCLAIMED.values(): + assert module not in traceai._MEDIA_ADAPTERS + assert module in traceai._ENTRY_POINTS # still traced + + +def test_provider_and_framework_report_rows_read_each_installed_descriptor(monkeypatch): + modules = {} + for name, (path, attr) in _PROVIDER_FRAMEWORK_DESCRIPTORS.items(): + modules[path] = SimpleNamespace(**{attr: {"operations": (name + ".input",)}}) + modules.setdefault(traceai._INSTRUMENTOR_REGISTRY[name], SimpleNamespace()) + for name, module in _UNCLAIMED.items(): # installed, with a descriptor that claims nothing + modules[module] = SimpleNamespace(**{name + "_attachment_capabilities": {"operations": ()}}) + + def import_module(requested): + if requested in modules: + return modules[requested] + raise ImportError(requested) + + monkeypatch.setattr(traceai, "_installed", lambda module: module in modules) + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + for name in _PROVIDER_FRAMEWORK_DESCRIPTORS: + assert rows[name]["media"] == "provider_adapter", name + assert rows[name]["media_operations"] == [name + ".input"], name + assert rows[name]["status"] == "not_enabled", name + for name, module in _UNCLAIMED.items(): + assert rows[name] == {"name": name, "package": module, "status": "not_enabled", "media": "unqualified"} + for alias in ("pydantic-ai", "google-genai", "vertex", "mistral"): + assert alias not in rows # aliases share one row + # An installed version without the descriptor: the lazy attribute raises AttributeError (old + # package) or ImportError (a core without provider_media); both are unqualified. + for name, (path, attr) in _PROVIDER_FRAMEWORK_DESCRIPTORS.items(): + saved = modules[path] + package = traceai._INSTRUMENTOR_REGISTRY[name] + for broken in ("attribute", "import"): + if path != package: + del modules[path] + elif broken == "attribute": + modules[path] = SimpleNamespace() + else: + modules[path] = _RaisingModule(ImportError("fi_instrumentation.attachments.provider_media")) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows[name] == {"name": name, "package": package, "status": "not_enabled", + "media": "unqualified"}, (name, broken) + modules[path] = saved + # A descriptor that lists no operation is never reported as a media adapter. + modules["traceai_litellm"] = SimpleNamespace(litellm_attachment_capabilities={"operations": []}) + assert {row["name"]: row for row in traceai.report()["integrations"]}["litellm"]["media"] == "unqualified" + + +class _RaisingModule: + def __init__(self, error): + self._error = error + + def __getattr__(self, name): + raise self._error + + +def _stub_openinference(monkeypatch): + """OpenInference's TraceConfig shape (hide_* flags plus an image cap), without the package.""" + import dataclasses + import sys + from types import ModuleType + + @dataclasses.dataclass(frozen=True) + class OpenInferenceTraceConfig: + hide_llm_invocation_parameters: bool = False + hide_inputs: bool = False + hide_outputs: bool = False + hide_input_messages: bool = False + hide_output_messages: bool = False + hide_input_images: bool = False + hide_input_text: bool = False + hide_output_text: bool = False + hide_embedding_vectors: bool = False + base64_image_max_length: int = 32000 + + package, module = ModuleType("openinference"), ModuleType("openinference.instrumentation") + module.TraceConfig = OpenInferenceTraceConfig + package.instrumentation = module + monkeypatch.setitem(sys.modules, "openinference", package) + monkeypatch.setitem(sys.modules, "openinference.instrumentation", module) + return OpenInferenceTraceConfig + + +def _agno_init(monkeypatch, manager, config): + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai, "_init_options", None) + monkeypatch.setattr(traceai, "_instrumented", set()) + provider = SimpleNamespace(attachment_manager=manager, trace_config=config) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + wrapper = Mock() + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(AgnoInstrumentorWrapper=wrapper))) + traceai.init(project_name="synthetic", auto_instrument=["agno"], verbose=False) + wrapper.assert_called_once_with() + wrapper.return_value.instrument.assert_called_once() + return provider, wrapper.return_value.instrument.call_args + + +def test_agno_gets_its_own_keyword_arguments_on_and_off(monkeypatch): + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + manager = _enabled_like_manager() + provider, call = _agno_init(monkeypatch, manager, TraceConfig()) + # No fi TraceConfig and no attachment_* alias: Agno's wrapper accepts neither. + assert call.args == () and call.kwargs == {"tracer_provider": provider, "attachments": manager} + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + for off in ("env", "disabled"): + if off == "env": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = _enabled_like_manager() + else: + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + manager = traceai.DisabledAttachmentManager() + provider, call = _agno_init(monkeypatch, manager, TraceConfig()) + # Attachments OFF: exactly what configure_agno_tracing(tracer_provider=...) gets. + assert call.kwargs == {"tracer_provider": provider}, off + + +def test_agno_carries_auto_hide_flags_as_an_openinference_config(monkeypatch): + oi_config = _stub_openinference(monkeypatch) + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + config = TraceConfig(hide_inputs=True, hide_output_text=True, hide_input_images=True) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), config) + passed = call.kwargs["config"] + assert isinstance(passed, oi_config) + assert (passed.hide_inputs, passed.hide_output_text, passed.hide_input_images) == (True, True, True) + assert (passed.hide_outputs, passed.hide_input_text, passed.base64_image_max_length) == (False, False, 32000) + assert set(call.kwargs) == {"tracer_provider", "config"} + + +def test_a_dependency_conflict_is_never_reported_registered(monkeypatch): + provider = SimpleNamespace(attachment_manager=traceai.DisabledAttachmentManager(), trace_config=TraceConfig()) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + instrumentor = Mock(is_instrumented_by_opentelemetry=False) # BaseInstrumentor skipped _instrument + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(PydanticAIInstrumentor=Mock(return_value=instrumentor)))) + traceai.init(project_name="synthetic", auto_instrument=["pydantic-ai"], verbose=False) + instrumentor.instrument.assert_called_once() + assert traceai.attachment_status()["adapters"]["pydantic-ai"] == "dependency_unavailable" + assert "traceai_pydantic_ai" not in traceai._instrumented + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows["pydantic_ai"]["status"] == "dependency_unavailable" From dd79fcfa9fd6774e3b15eb196ab96db50234a064 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:25:32 -0800 Subject: [PATCH 13/16] fix(python): pass attachments to Agno only when its media adapter is installed A traceai-agno release before the adapter has no `attachments` parameter, so auto with attachments on raised TypeError there and left Agno untraced. Auto now passes `attachments` only while capturing AND when the installed package declares its descriptor; otherwise Agno gets the OFF arguments. Co-Authored-By: Claude Opus 5.5 --- python/traceai/__init__.py | 7 +++++-- python/traceai/tests/test_attachments.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 6f13f0190..d3e6a8c07 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -389,7 +389,8 @@ def _agno_kwargs(provider) -> Dict[str, Any]: It takes OpenInference's TraceConfig (not fi's) and no ``attachment_*`` alias. Auto's hide flags are carried over, and only when one is set, so an unrestricted setup keeps - Agno's own default and environment policy. ``attachments`` is passed only while capturing. + Agno's own default and environment policy. ``attachments`` is passed only while capturing + and only to a version that has the media adapter. """ kwargs: Dict[str, Any] = {"tracer_provider": provider} trace_config = getattr(provider, "trace_config", None) @@ -402,7 +403,9 @@ def _agno_kwargs(provider) -> Dict[str, Any]: kwargs["config"] = OpenInferenceTraceConfig(**{name: True for name in hidden if name in names}) if _ATTACHMENTS_SUPPORTED: manager = get_attachment_manager() - if _capturing(manager): + # Only a traceai-agno version with the media adapter (it declares the descriptor) accepts + # ``attachments``; an older one keeps tracing with the OFF arguments. + if _capturing(manager) and _media_support("traceai_agno")["media"] == "provider_adapter": kwargs["attachments"] = manager return kwargs diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index 1c33105ce..499a4d717 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -624,6 +624,7 @@ def _agno_init(monkeypatch, manager, config): def test_agno_gets_its_own_keyword_arguments_on_and_off(monkeypatch): monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + monkeypatch.setattr(traceai, "_media_support", lambda module: {"media": "provider_adapter"}) manager = _enabled_like_manager() provider, call = _agno_init(monkeypatch, manager, TraceConfig()) # No fi TraceConfig and no attachment_* alias: Agno's wrapper accepts neither. @@ -665,3 +666,12 @@ def test_a_dependency_conflict_is_never_reported_registered(monkeypatch): assert "traceai_pydantic_ai" not in traceai._instrumented rows = {row["name"]: row for row in traceai.report()["integrations"]} assert rows["pydantic_ai"]["status"] == "dependency_unavailable" + + +def test_agno_without_the_media_adapter_keeps_the_off_arguments(monkeypatch): + """A traceai-agno release before the adapter has no ``attachments`` parameter at all.""" + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + monkeypatch.setattr(traceai, "_media_support", lambda module: {"media": "unqualified"}) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), TraceConfig()) + assert call.kwargs == {"tracer_provider": provider} + assert traceai.attachment_status()["adapters"]["agno"] == "registered" From 4901f03dae34c7d1328e3daf10d98bba60ea8cfb Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:17:30 -0800 Subject: [PATCH 14/16] fix(python): per-client Claude tool-span deferral, live-conversation gate, image signatures - Claude Agent SDK: a completed tool span waits for its result only while that client's receive_response() conversation is live; receive_messages() and hooks that run before receive_response() end the span at the hook, as before. - Deferral tables and results-seen are per client (ToolDeferrals); a conversation end or disconnect() flushes only that client's waiting spans. - tool_media: BMP, TIFF and ISO-BMFF HEIF/AVIF image brands join the signature table, so those images cannot escape hide_input_images under an audio, video or octet-stream label. Octet-stream blobs with a recognised signature are skipped (documented). - Tests: receive_messages() without disconnect(), two concurrent clients, mislabelled BMP/TIFF/HEIF rows and the octet-stream rule. Co-Authored-By: Claude Opus 5.5 --- .../attachments/tool_media.py | 22 +++- python/frameworks/claude-agent-sdk/README.md | 13 +- .../traceai_claude_agent_sdk/_attachments.py | 11 +- .../_client_wrapper.py | 32 +++-- .../traceai_claude_agent_sdk/_hooks.py | 99 +++++++++------ python/frameworks/mcp/README.md | 11 +- .../_fake_claude_cli.py | 13 +- .../test_claude_agent_sdk_attachments.py | 113 +++++++++++++++++- .../attachments_mcp_agents/test_tool_media.py | 66 +++++++++- 9 files changed, 310 insertions(+), 70 deletions(-) diff --git a/python/fi_instrumentation/attachments/tool_media.py b/python/fi_instrumentation/attachments/tool_media.py index 525b8130b..1ef0925e7 100644 --- a/python/fi_instrumentation/attachments/tool_media.py +++ b/python/fi_instrumentation/attachments/tool_media.py @@ -14,9 +14,15 @@ - Every value is preflighted before it is read or decoded. Each observation has a bounded decode budget and part count; parts beyond them are skipped. - A declared MIME is never trusted over the bytes: when the first bytes are a - known signature of ANOTHER modality (a PNG labelled ``audio/wav`` or - ``application/octet-stream``), the part is denied, so a mislabel cannot - escape ``hide_input_images`` or a modality restriction. Never relabelled. + known signature (``_SIGNATURES``) of ANOTHER modality (a PNG labelled + ``audio/wav``), the part is denied, so a mislabel cannot escape + ``hide_input_images`` or a modality restriction. Never relabelled. The + signatures checked are PNG, JPEG, GIF, WebP, BMP, TIFF, ISO-BMFF image + brands (``ftyp`` avif/heic/heix/mif1/msf1), WAV, Ogg, FLAC, ID3-tagged MP3 + and PDF; other bytes (MP4/WebM containers, MP3 without an ID3 tag) are + judged by their declared MIME only. ``application/octet-stream`` is the + ``file`` modality, so an octet-stream blob whose bytes carry a recognised + image, audio or PDF signature is skipped (fail-closed, never relabelled). - A context marked by another adapter as the owner of this call's media (``OWNER_KEY``, shared by name with ``traceai``) captures nothing here. @@ -90,11 +96,15 @@ def mime_kind(mime: Any) -> Optional[str]: return MIME_MODALITIES.get(mime) if isinstance(mime, str) else None -#: Leading-byte signatures with an unambiguous F01 modality. Containers shared by audio and -#: video (MP4 ``ftyp``, WebM/Matroska) have none: they can only be denied by their declared MIME. +#: Leading-byte signatures with an unambiguous F01 modality. ISO-BMFF ``ftyp`` boxes name an +#: image only for the HEIF/AVIF still-image brands listed here; other brands (mp42, isom, +#: ``M4A ``, ``qt ``) and WebM/Matroska are shared by audio and video, so those bytes are +#: judged by their declared MIME only. ``consistent`` decodes the first 12 bytes. _SIGNATURES = ( (0, b"\x89PNG\r\n\x1a\n", "image"), (0, b"\xff\xd8\xff", "image"), (0, b"GIF87a", "image"), - (0, b"GIF89a", "image"), (8, b"WEBP", "image"), (8, b"WAVE", "audio"), + (0, b"GIF89a", "image"), (8, b"WEBP", "image"), (0, b"BM", "image"), (0, b"II*\x00", "image"), + (0, b"MM\x00*", "image"), (4, b"ftypavif", "image"), (4, b"ftypheic", "image"), (4, b"ftypheix", "image"), + (4, b"ftypmif1", "image"), (4, b"ftypmsf1", "image"), (8, b"WAVE", "audio"), (0, b"OggS", "audio"), (0, b"fLaC", "audio"), (0, b"ID3", "audio"), (0, b"%PDF-", "document"), ) diff --git a/python/frameworks/claude-agent-sdk/README.md b/python/frameworks/claude-agent-sdk/README.md index 7bc316869..71fb0e01e 100644 --- a/python/frameworks/claude-agent-sdk/README.md +++ b/python/frameworks/claude-agent-sdk/README.md @@ -192,10 +192,15 @@ uploaded from the original SDK message, before any text flattening or truncation, as `tool_result` occurrences with the `tool_use_id`. They belong to the tool's span and are projected on it (`fi.attachment.refs`). The CLI usually runs the PostToolUse hook before it streams the tool result, so with -attachments on the tool span stays recording until its result has been -captured (at most 30 seconds, or until the conversation ends) and then ends -at the time the hook recorded; its duration is unchanged. With attachments -off, the hook ends the span immediately, as before. +attachments on, while that client's `receive_response()` conversation is +running, the tool span stays recording until its result has been captured (at +most 30 seconds, until that conversation ends, or until `disconnect()`) and +then ends at the time the hook recorded; its duration is unchanged. Each +client keeps its own waiting spans, so one conversation ending never ends +another client's. Tool media is captured from `receive_response()` only: +with `receive_messages()`, or for a hook that fires before +`receive_response()` starts, the hook ends the span immediately, as it +always does with attachments off. Media bytes never reach `tool.output` or the conversation output text. The SDK drops top-level user image blocks while parsing, so prompt images are not captured. URL and file sources are never read. diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py index 5d8abe732..3a42f9096 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py @@ -6,10 +6,13 @@ flattening or truncation, at the tool's span (with ``tool_call_id``): - on the tool span, while it is still recording: before the PostToolUse hook - (result-first) or after it, because with capture on the hook defers the - span's end until its result has been captured (hooks-first, the CLI's - usual order); the span then ends at the time the hook recorded; -- for a result with no traced tool, on the conversation span. + (result-first) or after it, because with capture on and the client's + ``receive_response()`` conversation live, the hook defers the span's end + until its result has been captured (hooks-first, the CLI's usual order); + the span then ends at the time the hook recorded. Each client has its own + deferral tables (``_hooks.ToolDeferrals``); +- for a result with no traced or recording tool span (for example a hook that + ran before ``receive_response()`` started), on the conversation span. Every occurrence is projected (``fi.attachment.refs``) on the live span that is its upload parent. diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py index 671aade46..296812e27 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py @@ -17,8 +17,8 @@ from ._attachments import capture_tool_results, tool_result_ids from ._attributes import ClaudeAgentAttributes, ClaudeAgentSpanKind from ._hooks import ( + ToolDeferrals, _active_tool_spans, - _deferred_tool_ends, create_pre_tool_use_hook, create_subagent_start_hook, create_post_tool_use_hook, @@ -171,13 +171,15 @@ def _flatten_content(self, content: Any) -> str: return str(content) -def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer, defer_tool_end: bool = False) -> None: +def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer, + deferrals: Optional[ToolDeferrals] = None) -> None: """Inject OpenTelemetry tracing hooks into ClaudeAgentOptions. Args: options: ClaudeAgentOptions instance tracer: OpenTelemetry tracer - defer_tool_end: tool-result media capture is on, so a completed tool span stays + deferrals: the client's deferral tables when tool-result media capture is on: while + its receive_response() conversation is live, a completed tool span stays recording until its result is captured (it keeps the hook's end time) """ if not hasattr(options, "hooks"): @@ -199,7 +201,7 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer, defer_tool_end: from claude_agent_sdk import HookMatcher pre_hook = create_pre_tool_use_hook(tracer) - post_hook = (create_post_tool_use_hook(tracer, defer_end=True) if defer_tool_end + post_hook = (create_post_tool_use_hook(tracer, deferrals=deferrals) if deferrals is not None else create_post_tool_use_hook(tracer)) start_hook = create_subagent_start_hook(tracer) @@ -246,9 +248,11 @@ class TracedClaudeSDKClient(original_class): def __init__(self, *args: Any, **kwargs: Any): # Extract options for hook injection options = kwargs.get("options") or (args[0] if args else None) + # This client's deferred tool ends (tool-result media capture only). + deferrals = ToolDeferrals() if attachments is not None else None if options: - if attachments is not None: - inject_tracing_hooks(options, tracer, defer_tool_end=True) + if deferrals is not None: + inject_tracing_hooks(options, tracer, deferrals=deferrals) else: inject_tracing_hooks(options, tracer) @@ -264,6 +268,7 @@ def __init__(self, *args: Any, **kwargs: Any): self._prompt: Optional[str] = None self._start_time: Optional[float] = None self._options = options + self._tool_deferrals = deferrals async def query(self, *args: Any, **kwargs: Any) -> Any: """Capture prompt and timestamp when query starts.""" @@ -285,6 +290,9 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: attributes=span_attributes, ) as conversation_span: set_parent_span(conversation_span) + deferrals = getattr(self, "_tool_deferrals", None) + if deferrals is not None: + deferrals.live += 1 # Initialize turn tracker turn_tracker = TurnTracker( @@ -326,14 +334,14 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: msg, conversation_span, _active_tool_spans, - _deferred_tool_ends, + deferrals.ends if deferrals is not None else {}, message_index=message_index, trace_config=trace_config, capture=attachment_capture, ) finally: for tool_use_id in tool_result_ids(msg): - end_deferred_tool(tool_use_id) + end_deferred_tool(deferrals, tool_use_id) # End current turn, mark next start turn_tracker.end_current_turn() turn_tracker.mark_next_start() @@ -388,6 +396,10 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: finally: turn_tracker.end_current_turn() clear_parent_span() + if deferrals is not None: + # Only this conversation's deferred tool spans end here, at their hook's time. + deferrals.live -= 1 + end_deferred_tool_spans(deferrals) clear_active_tool_spans() def _build_span_attributes(self) -> Dict[str, Any]: @@ -539,11 +551,11 @@ async def __aexit__(self, *args: Any) -> None: if attachments is not None and hasattr(original_class, "disconnect"): async def disconnect(self: Any, *args: Any, **kwargs: Any) -> Any: - """Disconnect; completed tool spans still waiting for their result end now.""" + """Disconnect; this client's tool spans still waiting for their result end now.""" try: return await super(TracedClaudeSDKClient, self).disconnect(*args, **kwargs) finally: - end_deferred_tool_spans() + end_deferred_tool_spans(getattr(self, "_tool_deferrals", None)) TracedClaudeSDKClient.disconnect = disconnect diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py index ecae04e36..89a4f08f4 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py @@ -28,17 +28,33 @@ # Key: tool_use_id, Value: (span, start_time, tool_name) _active_tool_spans: Dict[str, Tuple[Span, float, str]] = {} -# Tool spans the PostToolUse hook completed while tool-result media capture is on. The SDK streams -# the tool result AFTER that hook, so the span stays recording until its ToolResultBlock has been -# captured (its media refs are projected on it), then ends at the time the hook recorded. -# Key: tool_use_id, Value: (span, end_time_ns, timer handle or None). Bounded and time-limited. -_deferred_tool_ends: "OrderedDict[str, Tuple[Span, int, Any]]" = OrderedDict() _MAX_DEFERRED_ENDS = 256 -# Tool results streamed while their span was still open (result-first): PostToolUse ends those now. -_results_seen: "OrderedDict[str, bool]" = OrderedDict() #: A deferred tool span ends at most this long after its hook, even if its result never arrives. DEFERRED_END_TIMEOUT_S = 30.0 + +class ToolDeferrals: + """One client's tool spans whose end waits for their result (tool-result media capture on). + + The SDK streams a tool result AFTER the PostToolUse hook, so while that client's + ``receive_response()`` conversation span is live, a completed tool span stays recording + until its ToolResultBlock has been captured (its media refs are projected on it), then + ends at the time the hook recorded. Outside a live conversation (``receive_messages()``, + or a hook that fires before ``receive_response()`` starts) the hook ends the span as + before. Each client has its own tables, so one conversation's end flushes only its own. + + ``ends``: tool_use_id -> (span, end_time_ns, timer handle or None), bounded and + time-limited. ``results_seen``: results streamed while their span was still open + (result-first), so PostToolUse ends those now. ``live``: open conversation spans. + """ + + __slots__ = ("ends", "results_seen", "live") + + def __init__(self) -> None: + self.ends: "OrderedDict[str, Tuple[Span, int, Any]]" = OrderedDict() + self.results_seen: "OrderedDict[str, bool]" = OrderedDict() + self.live = 0 + # Tools that launch a subagent: "Task" in older Claude Code releases, "Agent" in current ones SUBAGENT_TOOLS = ("Task", "Agent") @@ -288,67 +304,73 @@ def _end_deferred(entry: Tuple[Span, int, Any]) -> None: logger.debug(f"Failed to end deferred tool span: {e}") -def _expire_deferred(tool_use_id: str, entry: Tuple[Span, int, Any]) -> None: - if _deferred_tool_ends.get(tool_use_id) is entry: - del _deferred_tool_ends[tool_use_id] +def _expire_deferred(deferrals: ToolDeferrals, tool_use_id: str, entry: Tuple[Span, int, Any]) -> None: + if deferrals.ends.get(tool_use_id) is entry: + del deferrals.ends[tool_use_id] _end_deferred(entry) -def _defer_tool_end(tool_use_id: str, span: Span, end_ns: int) -> bool: +def _defer_tool_end(deferrals: ToolDeferrals, tool_use_id: str, span: Span, end_ns: int) -> bool: """Keep a completed tool span recording until its result is captured. False: end it now.""" + if deferrals.results_seen.pop(tool_use_id, None): + return False # Its result was already captured on the open span. + if deferrals.live <= 0: + return False # No receive_response() conversation will capture its result: end now. try: import asyncio loop = asyncio.get_running_loop() except Exception: return False # No asyncio loop to bound the wait (another async backend): end now. - if _results_seen.pop(tool_use_id, None): - return False # Its result was already captured on the open span. try: - previous = _deferred_tool_ends.pop(tool_use_id, None) + previous = deferrals.ends.pop(tool_use_id, None) if previous is not None: _end_deferred(previous) holder: List[Any] = [] - handle = loop.call_later(DEFERRED_END_TIMEOUT_S, lambda: _expire_deferred(tool_use_id, holder[0])) + handle = loop.call_later(DEFERRED_END_TIMEOUT_S, + lambda: _expire_deferred(deferrals, tool_use_id, holder[0])) entry = (span, end_ns, handle) holder.append(entry) - _deferred_tool_ends[tool_use_id] = entry - while len(_deferred_tool_ends) > _MAX_DEFERRED_ENDS: - _, oldest = _deferred_tool_ends.popitem(last=False) + deferrals.ends[tool_use_id] = entry + while len(deferrals.ends) > _MAX_DEFERRED_ENDS: + _, oldest = deferrals.ends.popitem(last=False) _end_deferred(oldest) return True except Exception: return False -def end_deferred_tool(tool_use_id: Any) -> None: - """End one deferred tool span (its result was processed), at the time its hook recorded.""" - if not isinstance(tool_use_id, str): +def end_deferred_tool(deferrals: Optional[ToolDeferrals], tool_use_id: Any) -> None: + """End one of this client's deferred tool spans (its result was processed), at its hook's time.""" + if deferrals is None or not isinstance(tool_use_id, str): return - entry = _deferred_tool_ends.pop(tool_use_id, None) + entry = deferrals.ends.pop(tool_use_id, None) if entry is not None: _end_deferred(entry) elif tool_use_id in _active_tool_spans: - _results_seen[tool_use_id] = True - while len(_results_seen) > _MAX_DEFERRED_ENDS: - _results_seen.popitem(last=False) + deferrals.results_seen[tool_use_id] = True + while len(deferrals.results_seen) > _MAX_DEFERRED_ENDS: + deferrals.results_seen.popitem(last=False) -def end_deferred_tool_spans() -> None: - """End every deferred tool span at its recorded time (conversation end).""" - _results_seen.clear() - while _deferred_tool_ends: - _, entry = _deferred_tool_ends.popitem(last=False) +def end_deferred_tool_spans(deferrals: Optional[ToolDeferrals]) -> None: + """End every deferred tool span of ONE client at its recorded time (its conversation ended).""" + if deferrals is None: + return + deferrals.results_seen.clear() + while deferrals.ends: + _, entry = deferrals.ends.popitem(last=False) _end_deferred(entry) -def create_post_tool_use_hook(tracer: trace_api.Tracer, defer_end: bool = False) -> Callable: +def create_post_tool_use_hook(tracer: trace_api.Tracer, deferrals: Optional[ToolDeferrals] = None) -> Callable: """Create a PostToolUse hook for completing tool traces. Args: tracer: OpenTelemetry tracer instance - defer_end: keep the completed span recording until its tool result is captured - (only when tool-result media capture is on; the span keeps the hook's end time) + deferrals: the client's deferral tables when tool-result media capture is on: while + its receive_response() conversation is live, the completed span stays recording + until its tool result is captured (it keeps the hook's end time) Returns: Async hook function compatible with Claude Agent SDK @@ -425,9 +447,10 @@ async def post_tool_use_hook( }, ) - # End the span. With media capture on, the tool result (streamed after this - # hook) is captured on the still-recording span first; it ends at end_ns. - if not (defer_end and _defer_tool_end(tool_use_id, span, end_ns)): + # End the span. With media capture on and a live conversation, the tool result + # (streamed after this hook) is captured on the still-recording span first; it + # ends at end_ns. + if not (deferrals is not None and _defer_tool_end(deferrals, tool_use_id, span, end_ns)): span.end() # Clean up subagent span if this launched a subagent @@ -524,8 +547,8 @@ def clear_active_tool_spans() -> None: """ global _active_tool_spans, _subagent_spans - # Completed tool spans still waiting for their result end normally, at their hook's time. - end_deferred_tool_spans() + # Deferred tool spans belong to one client; its conversation flushes its own + # (end_deferred_tool_spans) before calling this. # End any orphaned spans for tool_use_id, (span, start_time, tool_name) in _active_tool_spans.items(): diff --git a/python/frameworks/mcp/README.md b/python/frameworks/mcp/README.md index ac7bf5cac..5b344ade3 100644 --- a/python/frameworks/mcp/README.md +++ b/python/frameworks/mcp/README.md @@ -66,8 +66,15 @@ and `BlobResourceContents.blob` (in an `EmbeddedResource` or a parts and 256 KiB of decoded bytes per result. `resource_link` blocks, resource URIs and `structuredContent` are never read. A MIME of another modality than the block type, or outside the attachment MIME list, uploads -nothing, and so do bytes whose leading signature names another modality than -the declared MIME (a PNG labelled `audio/wav` or `application/octet-stream`). `attachments=` overrides the provider's manager; `trace_config=` and +nothing. So do bytes whose leading signature names another modality than the +declared MIME, for example a PNG labelled `audio/wav`. The signatures checked +are PNG, JPEG, GIF, WebP, BMP, TIFF, the HEIF/AVIF image brands of an ISO-BMFF +`ftyp` box (`avif`, `heic`, `heix`, `mif1`, `msf1`), WAV, Ogg, FLAC, MP3 with +an ID3 tag, and PDF. Other bytes, including MP4 and WebM containers and MP3 +without an ID3 tag, are judged by the declared MIME only. +`application/octet-stream` is the generic file modality, so an octet-stream +blob whose bytes carry one of those image, audio or PDF signatures is skipped +rather than relabelled. `attachments=` overrides the provider's manager; `trace_config=` and `attachment_capture=` only add restrictions. Wrapping a wrapped session returns it unchanged. `capture_mcp_result(manager, span, result, operation)` is the same capture for an existing span. diff --git a/python/tests/attachments_mcp_agents/_fake_claude_cli.py b/python/tests/attachments_mcp_agents/_fake_claude_cli.py index a2bf55a9e..83d83f4b0 100644 --- a/python/tests/attachments_mcp_agents/_fake_claude_cli.py +++ b/python/tests/attachments_mcp_agents/_fake_claude_cli.py @@ -6,8 +6,9 @@ carrying the media in ``FAKE_CLAUDE_RESULT`` (a JSON file), then a final answer. ``FAKE_CLAUDE_ORDER=result-first`` streams the tool result before PostToolUse; ``no-result`` never streams it. ``FAKE_CLAUDE_GAP`` (seconds) waits between -PostToolUse and the tool result. No model, network or file other than the -result file is touched. +PostToolUse and the tool result; ``FAKE_CLAUDE_HOLD`` (a path) also waits there, +until that file exists (at most 20 s). ``FAKE_TOOL_ID`` names the tool call. No +model, network or file other than the result and hold files is touched. """ import json @@ -23,6 +24,8 @@ TOOL_RESULT = json.load(handle) ORDER = os.environ.get("FAKE_CLAUDE_ORDER", "hooks-first") GAP = float(os.environ.get("FAKE_CLAUDE_GAP", "0")) +HOLD = os.environ.get("FAKE_CLAUDE_HOLD", "") +TOOL_ID = os.environ.get("FAKE_TOOL_ID", "toolu_screen_1") TOOL = "mcp__screen__capture" hooks = {} counter = [0] @@ -61,7 +64,8 @@ def assistant(content): def turn(): - tool_use_id = "toolu_screen_1" + tool_use_id = TOOL_ID + time.sleep(0.05) # Model latency: the client is iterating receive_response() before any hook. assistant([{"type": "tool_use", "id": tool_use_id, "name": TOOL, "input": {}}]) hook("PreToolUse", tool_use_id, {}) result = {"type": "user", "parent_tool_use_id": None, "session_id": "s1", "message": { @@ -72,6 +76,9 @@ def turn(): else: hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) time.sleep(GAP) + deadline = time.monotonic() + 20 + while HOLD and not os.path.exists(HOLD) and time.monotonic() < deadline: + time.sleep(0.02) if ORDER != "no-result": send(result) assistant([{"type": "text", "text": "done"}]) diff --git a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py index fc15da106..b336f8428 100644 --- a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py +++ b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py @@ -9,6 +9,7 @@ import os import stat import sys +import time from pathlib import Path import pytest @@ -67,10 +68,19 @@ def install(manager, config=None): state["instrumentor"].uninstrument() +def client_options(cli_path, **env): + fields = getattr(claude_agent_sdk.ClaudeAgentOptions, "__dataclass_fields__", {}) + kwargs = {"cli_path": cli_path} if "cli_path" in fields else {} + if env: + if "env" not in fields: + pytest.skip("this claude-agent-sdk release has no ClaudeAgentOptions.env") + kwargs["env"] = env + return claude_agent_sdk.ClaudeAgentOptions(**kwargs) + + def converse(cli_path): async def go(): - fields = getattr(claude_agent_sdk.ClaudeAgentOptions, "__dataclass_fields__", {}) - options = claude_agent_sdk.ClaudeAgentOptions(**({"cli_path": cli_path} if "cli_path" in fields else {})) + options = client_options(cli_path) messages = [] async with claude_agent_sdk.ClaudeSDKClient(options=options) as client: await client.query("take a screenshot") @@ -169,3 +179,102 @@ def test_disabled_manager_uploads_nothing_and_the_conversation_is_unchanged(f01, assert f01.uploads == [] blocks = [m for m in messages if type(m).__name__ == "UserMessage"][0].content assert blocks[0].content[0]["source"]["data"] == b64(SHOT) + + +def test_receive_messages_without_disconnect_ends_the_tool_span_at_its_hook(f01, instrument, cli): + """No receive_response() conversation is live, so PostToolUse ends the span as with attachments off.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)], gap=0.3) + clients = [] + + async def go(): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path)) + clients.append(client) + await client.connect() + await client.query("take a screenshot") + async for message in client.receive_messages(): + if type(message).__name__ == "ResultMessage": + break + # Returns without disconnect() or async-with: nothing flushes deferred spans. + + asyncio.run(go()) + flush(manager) + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + assert not clients[0]._tool_deferrals.ends and clients[0]._tool_deferrals.live == 0 + manager.shutdown(5) + + +def test_conversation_end_flushes_its_waiting_tool_span_without_disconnect(f01, instrument, cli): + """A result that never arrives: the span ends when receive_response() ends, not at the 30 s timer.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)], order="no-result", gap=0.2) + clients = [] + + async def go(): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path)) + clients.append(client) + await client.connect() + await client.query("take a screenshot") + async for _ in client.receive_response(): + pass + # Inspected before any disconnect(): only the conversation end can have ended it. + deferrals = clients[0]._tool_deferrals + return len(tool_spans(exporter)), list(deferrals.ends), deferrals.live + + ended, waiting, live = asyncio.run(go()) + assert (ended, waiting, live) == (1, [], 0) + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + manager.shutdown(5) + + +def test_one_conversation_ending_does_not_end_another_clients_deferred_tool_span(f01, instrument, cli, tmp_path): + """B's tool span waits for its result while A runs to completion; A's end flushes only A's spans.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)]) + other = png(seed=52) + a_result = tmp_path / "result-a.json" + a_result.write_text(json.dumps({"content": [image(other)], "is_error": False})) + release = tmp_path / "release-b" + clients = {} + parked = {} + + async def converse_as(name, **env): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path, FAKE_TOOL_ID="toolu_" + name, **env)) + clients[name] = client + async with client: + await client.query("take a screenshot") + async for _ in client.receive_response(): + pass + + async def go(): + b = asyncio.ensure_future(converse_as("b", FAKE_CLAUDE_HOLD=str(release))) + deadline = time.monotonic() + 15 + while not ("b" in clients and clients["b"]._tool_deferrals.ends): + assert time.monotonic() < deadline and not b.done(), "B never parked its tool span" + await asyncio.sleep(0.02) + await converse_as("a", FAKE_CLAUDE_RESULT=str(a_result)) + parked["b"] = list(clients["b"]._tool_deferrals.ends) + release.write_text("go") + await b + + asyncio.run(go()) + flush(manager) + assert parked["b"] == ["toolu_b"] + tools = {s.attributes[ClaudeAgentAttributes.TOOL_USE_ID]: s for s in tool_spans(exporter)} + assert sorted(tools) == ["toolu_a", "toolu_b"] + assert sorted(u.get("tool_call_id") for u in f01.uploads) == ["toolu_a", "toolu_b"] + assert sorted(f01.payloads) == sorted([SHOT, other]) + for upload in f01.uploads: + tool = tools[upload["tool_call_id"]] + assert tool.status.is_ok + assert upload["association"]["parent"]["span_id"] == format(tool.context.span_id, "016x") + assert [r["occurrence_id"] for r in refs(tool)] == [upload["occurrence_id"]] + assert not clients["a"]._tool_deferrals.ends and not clients["b"]._tool_deferrals.ends + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_tool_media.py b/python/tests/attachments_mcp_agents/test_tool_media.py index dddfec99b..c77c58fd4 100644 --- a/python/tests/attachments_mcp_agents/test_tool_media.py +++ b/python/tests/attachments_mcp_agents/test_tool_media.py @@ -1,5 +1,7 @@ """The shared tool-media helper: deny-wins folding, preflight before read, budgets, dedup, ownership.""" +import struct + import pytest from fi_instrumentation import TraceConfig @@ -136,6 +138,68 @@ def test_signature_of_another_modality_is_denied_and_never_relabelled(f01, spans (b"RIFF\x00\x00\x00\x00WEBP", "image"), (b"RIFF\x00\x00\x00\x00WAVE", "audio"), (b"OggS\x00\x02", "audio"), (b"fLaC\x00\x00", "audio"), (b"ID3\x04\x00", "audio"), (b"%PDF-1.7\n", "document"), (b"\x00\x00\x00\x18ftypmp42", None), (b"\x1aE\xdf\xa3\x9fB\x86", None), (b"RIFF\x00\x00\x00\x00AVI ", None), - (b"plain text", None)]) + (b"plain text", None), (b"BM6\x00\x00\x00\x00\x00\x00\x006\x00", "image"), (b"II*\x00\x08\x00\x00\x00", "image"), + (b"MM\x00*\x00\x00\x00\x08", "image"), (b"\x00\x00\x00\x18ftypavif", "image"), + (b"\x00\x00\x00\x18ftypheic", "image"), (b"\x00\x00\x00\x18ftypheix", "image"), + (b"\x00\x00\x00\x18ftypmif1", "image"), (b"\x00\x00\x00\x18ftypmsf1", "image"), + (b"\x00\x00\x00\x18ftypisom", None), (b"\x00\x00\x00\x18ftypM4A ", None), (b"\x00\x00\x00\x14ftypqt ", None), + (b"\x00\x00\x00\x18avifftyp", None)]) def test_signature_kind(head, kind): assert tool_media.signature_kind(head) == kind + + +def _bmp(): + pixels = b"\x00\x00\xff\x00" * 4 + header = b"BM" + struct.pack("I", 24) + b"ftyp" + brand + b"\x00\x00\x00\x00" + brand + b"mif1" + b"\x00" * 32 + + +_IMAGE_MISLABELS = [ + ("bmp-as-audio/wav", "audio/wav", _bmp), ("bmp-as-audio/L16", "audio/L16", _bmp), + ("bmp-as-octet-stream", "application/octet-stream", _bmp), ("tiff-le-as-audio/L16", "audio/L16", _tiff), + ("tiff-be-as-audio/wav", "audio/wav", lambda: _tiff(b"MM\x00*")), + ("avif-as-video/mp4", "video/mp4", lambda: _heif(b"avif")), ("heic-as-audio/mp4", "audio/mp4", lambda: _heif(b"heic")), + ("heix-as-octet-stream", "application/octet-stream", lambda: _heif(b"heix")), + ("mif1-as-video/mp4", "video/mp4", lambda: _heif(b"mif1")), ("msf1-as-video/mp4", "video/mp4", lambda: _heif(b"msf1"))] + + +@pytest.mark.parametrize("label,mime,make", _IMAGE_MISLABELS, ids=[case[0] for case in _IMAGE_MISLABELS]) +def test_bmp_tiff_and_heif_images_cannot_escape_hide_input_images(f01, spans, label, mime, make): + """Image bytes declared as audio, video or a generic file are not uploaded under hide_input_images.""" + manager = f01.manager() + provider, _ = spans + data = make() + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_input_images=True)) + assert media.base64("a", lambda: b64(data), mime, purpose="tool_result") is None + assert media.data_uri("b", lambda: "data:%s;base64,%s" % (mime, b64(data)), (mime,), + purpose="tool_result") is None + flush(manager) + assert f01.uploads == [] and data not in f01.payloads + manager.shutdown(5) + + +def test_octet_stream_with_a_recognised_signature_is_skipped(f01, spans): + """application/octet-stream is the file modality: image, audio or PDF bytes in it are skipped, never relabelled.""" + from _harness import PDF, wav + + manager = f01.manager() + provider, _ = spans + plain = b"opaque tool output, no known signature" * 4 + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test") + for position, data in (("png", png(seed=31)), ("wav", wav()), ("pdf", PDF), ("bmp", _bmp())): + assert media.base64(position, lambda d=data: b64(d), "application/octet-stream", + purpose="tool_result") is None, position + assert media.base64("plain", lambda: b64(plain), "application/octet-stream", purpose="tool_result") + flush(manager) + assert [(u["mime_type"], p) for u, p in zip(f01.uploads, f01.payloads)] == [("application/octet-stream", plain)] + manager.shutdown(5) From 2010273d5ad2c78f5cf9d254c4d2974222c50ca5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:31:17 -0800 Subject: [PATCH 15/16] fix(python): auto skips Agno media when trace_config sets a flag Agno cannot apply Agno's privacy bridge reads pii_redaction and hide_error_details only from FI_PII_REDACTION / FI_HIDE_ERROR_DETAILS. When auto's trace_config sets either one in code and the environment (plus the forwarded hide flags) would not, auto no longer passes attachments= to Agno and reports the row config_unsupported. OFF arguments are unchanged. README lists the provider and framework rows, including vllm and minimax. Co-Authored-By: Claude Opus 5.5 --- python/traceai/README.md | 23 ++++++-- python/traceai/__init__.py | 69 +++++++++++++++++++----- python/traceai/tests/test_attachments.py | 67 +++++++++++++++++++++++ 3 files changed, 141 insertions(+), 18 deletions(-) diff --git a/python/traceai/README.md b/python/traceai/README.md index 8b98dfb66..2e8c2d8f8 100644 --- a/python/traceai/README.md +++ b/python/traceai/README.md @@ -69,10 +69,25 @@ Both helpers use the provider and hide flags from `traceai.init`, and pass the s manager only while capture is on. `traceai.report()` shows the attachment state and, per integration, `media`: -`provider_adapter` when the installed instrumentor captures media (today `traceAI-openai`, -`traceAI-anthropic`, `traceAI-groq` and the six MCP and agent packages above), -`session_binding` for voice integrations that capture only through an explicit `bind_*` -call, otherwise `unqualified`. It contains no endpoint or credential and is not a playback claim. +`provider_adapter` when the installed instrumentor version captures media, with its +`media_operations`; `session_binding` for voice integrations that capture only through an +explicit `bind_*` call; otherwise `unqualified`. It contains no endpoint or credential and +is not a playback claim. A row can be `provider_adapter` for: + +| Kind | Packages | +|---|---| +| Providers | `openai`, `anthropic`, `groq`, `google_genai`, `vertexai`, `bedrock`, `litellm`, `fireworks`, `xai`, `mistralai`, `openrouter`, `portkey`, `ollama`, `vllm`, `minimax` | +| Frameworks | `langchain`, `llamaindex`, `agno`, `pydantic_ai` | +| MCP and agents | the six packages above | + +An installed version without the adapter reports `unqualified`. `together`, `deepseek` and +`cerebras` capture no media. + +Agno reads `pii_redaction` and `hide_error_details` only from `FI_PII_REDACTION` and +`FI_HIDE_ERROR_DETAILS`. When `trace_config` sets either one in code and the environment does +not, `traceai` still traces Agno but captures no Agno media, and its status is +`config_unsupported`. Agno span text then does not follow that flag; set the environment +variable to apply it. ## License diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index d3e6a8c07..c94119c14 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -345,8 +345,16 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: instrumentor = instrumentor_class() provider = _tracer_provider kwargs = dict(tracer_provider=provider, config=getattr(provider, "trace_config", None)) + status = "registered" if instrumentor_module == "traceai_agno": - kwargs = _agno_kwargs(provider) + kwargs, unreachable = _agno_kwargs(provider) + if unreachable: + # Agno is traced, but cannot apply these flags and captures no media. + status = "config_unsupported" + if verbose: + print(f"Warning: {lib_name} cannot apply trace_config {', '.join(unreachable)} " + "set in code (it reads them only from FI_* environment variables); " + "its media capture is off and its span text does not follow them") elif _ATTACHMENTS_SUPPORTED: manager = get_attachment_manager() if _capturing(manager): @@ -365,7 +373,7 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: print(f"Warning: {lib_name} was not instrumented (dependency check failed)") return False _instrumented.add(instrumentor_module) - _adapter_status[lib_name_lower] = "registered" + _adapter_status[lib_name_lower] = status if verbose: print(f"| Auto-instrumented: {lib_name}") @@ -384,30 +392,58 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: return False -def _agno_kwargs(provider) -> Dict[str, Any]: - """Keyword arguments for ``AgnoInstrumentorWrapper.instrument``. +# FI-only TraceConfig flags. Agno's privacy bridge builds its FI config from the environment and +# takes only the hide_* flags from the OpenInference config auto passes, so these reach Agno only +# through FI_PII_REDACTION / FI_HIDE_ERROR_DETAILS. +_AGNO_ENV_ONLY = ("pii_redaction", "hide_error_details") + + +def _agno_privacy(trace_config): + """``(hide flags for Agno's OpenInference config or None, FI-only flags set on auto that Agno cannot apply)``. + + What Agno applies is its environment policy plus the forwarded hide flags (they also derive + ``hide_error_details``), so a flag counts as unreachable only when that would not set it. + """ + hidden = [field.name for field in dataclasses.fields(TraceConfig) + if field.name.startswith("hide_") and getattr(trace_config, field.name, False) is True] + forwarded: Optional[Dict[str, bool]] = None + if hidden: + from openinference.instrumentation import TraceConfig as OpenInferenceTraceConfig + + names = {field.name for field in dataclasses.fields(OpenInferenceTraceConfig)} + forwarded = {name: True for name in hidden if name in names} + wanted = [name for name in _AGNO_ENV_ONLY if getattr(trace_config, name, False) is True] + if not wanted: + return forwarded, [] + try: + reach = TraceConfig(**(forwarded or {})) + except Exception: + reach = None # Unknown: treat every wanted flag as unreachable. + return forwarded, [name for name in wanted if getattr(reach, name, False) is not True] + + +def _agno_kwargs(provider): + """``(keyword arguments for AgnoInstrumentorWrapper.instrument, unreachable FI-only flags)``. It takes OpenInference's TraceConfig (not fi's) and no ``attachment_*`` alias. Auto's hide flags are carried over, and only when one is set, so an unrestricted setup keeps - Agno's own default and environment policy. ``attachments`` is passed only while capturing - and only to a version that has the media adapter. + Agno's own default and environment policy. ``attachments`` is passed only while capturing, + only to a version that has the media adapter, and never while auto's trace_config sets + ``pii_redaction`` or ``hide_error_details`` that Agno cannot apply (fail closed). """ kwargs: Dict[str, Any] = {"tracer_provider": provider} - trace_config = getattr(provider, "trace_config", None) - hidden = {field.name: True for field in dataclasses.fields(TraceConfig) - if field.name.startswith("hide_") and getattr(trace_config, field.name, False) is True} - if hidden: + forwarded, unreachable = _agno_privacy(getattr(provider, "trace_config", None)) + if forwarded is not None: from openinference.instrumentation import TraceConfig as OpenInferenceTraceConfig - names = {field.name for field in dataclasses.fields(OpenInferenceTraceConfig)} - kwargs["config"] = OpenInferenceTraceConfig(**{name: True for name in hidden if name in names}) - if _ATTACHMENTS_SUPPORTED: + kwargs["config"] = OpenInferenceTraceConfig(**forwarded) + if _ATTACHMENTS_SUPPORTED and not unreachable: manager = get_attachment_manager() # Only a traceai-agno version with the media adapter (it declares the descriptor) accepts # ``attachments``; an older one keeps tracing with the OFF arguments. if _capturing(manager) and _media_support("traceai_agno")["media"] == "provider_adapter": kwargs["attachments"] = manager - return kwargs + return kwargs, unreachable def get_tracer_provider() -> Optional[TracerProvider]: @@ -584,6 +620,11 @@ def report() -> dict: integrations that capture only through an explicit ``bind_*`` call, otherwise ``unqualified``. It describes capability, not that any media was captured, and never a playback claim. No endpoint, header or credential is included. + + ``integrations[].status`` is ``registered`` once enabled, ``config_unsupported`` for an + Agno traced without media because auto's trace_config sets ``pii_redaction`` or + ``hide_error_details`` it cannot apply, ``dependency_unavailable``, ``unsupported_version``, + else ``not_enabled`` or ``not_installed``. """ names = {} for name, module in _INSTRUMENTOR_REGISTRY.items(): diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index 499a4d717..a6369840c 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -675,3 +675,70 @@ def test_agno_without_the_media_adapter_keeps_the_off_arguments(monkeypatch): provider, call = _agno_init(monkeypatch, _enabled_like_manager(), TraceConfig()) assert call.kwargs == {"tracer_provider": provider} assert traceai.attachment_status()["adapters"]["agno"] == "registered" + + +def _agno_on(monkeypatch): + for name in ("FI_ATTACHMENTS_ENABLED", "FI_PII_REDACTION", "FI_HIDE_ERROR_DETAILS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(traceai, "_media_support", lambda module: {"media": "provider_adapter"}) + + +def _agno_row(): + return {row["name"]: row for row in traceai.report()["integrations"]}["agno"]["status"] + + +def test_agno_fails_closed_on_a_pii_or_error_flag_it_cannot_apply(monkeypatch): + """Agno reads pii_redaction / hide_error_details only from the environment: no media, own status.""" + _stub_openinference(monkeypatch) + for config in (TraceConfig(pii_redaction=True), TraceConfig(hide_error_details=True), + TraceConfig(pii_redaction=True, hide_error_details=True)): + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), config) + assert call.kwargs == {"tracer_provider": provider}, config + assert traceai.attachment_status()["adapters"]["agno"] == "config_unsupported" + assert _agno_row() == "config_unsupported" + # A hide flag Agno does receive keeps its config, and the error flag it derives is not lost. + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), + TraceConfig(hide_inputs=True, pii_redaction=True)) + assert set(call.kwargs) == {"tracer_provider", "config"} and call.kwargs["config"].hide_inputs is True + assert _agno_row() == "config_unsupported" + + +def test_agno_keeps_media_when_the_environment_carries_the_flag(monkeypatch): + _stub_openinference(monkeypatch) + for env, config in (("FI_PII_REDACTION", TraceConfig(pii_redaction=True)), + ("FI_HIDE_ERROR_DETAILS", TraceConfig(hide_error_details=True))): + _agno_on(monkeypatch) + monkeypatch.setenv(env, "true") + monkeypatch.setattr(traceai, "_adapter_status", {}) + manager = _enabled_like_manager() + provider, call = _agno_init(monkeypatch, manager, config) + assert call.kwargs == {"tracer_provider": provider, "attachments": manager}, env + assert _agno_row() == "registered" + # hide_error_details derived from a forwarded content hide flag is applied by Agno too. + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + manager = _enabled_like_manager() + config = TraceConfig(hide_inputs=True) + assert config.hide_error_details is True + provider, call = _agno_init(monkeypatch, manager, config) + assert set(call.kwargs) == {"tracer_provider", "config", "attachments"} + assert _agno_row() == "registered" + + +def test_agno_off_arguments_are_unchanged_by_an_unreachable_flag(monkeypatch): + _stub_openinference(monkeypatch) + for off in ("env", "disabled"): + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + if off == "env": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = _enabled_like_manager() + else: + manager = traceai.DisabledAttachmentManager() + provider, call = _agno_init(monkeypatch, manager, TraceConfig(pii_redaction=True)) + assert call.kwargs == {"tracer_provider": provider}, off + assert _agno_row() == "config_unsupported", off From 67e5cbc02c88e536243bb64a280f0b2db1d3dfeb Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:37:05 -0800 Subject: [PATCH 16/16] test(python): auto framework+OpenAI one-upload rows and Agno config_unsupported rows LangChain and Agno over the OpenAI instrumentor through traceai.init upload each call's image once, on the framework span (both enable orders). Agno with pii_redaction or hide_error_details set only in code is config_unsupported and uploads nothing; the same flag from the environment keeps its media; with the OpenAI instrumentor also on, the unmarked call is owned by the OpenAI span. Co-Authored-By: Claude Opus 5.5 --- .../test_auto_framework_media.py | 159 +++++++++++++++++- 1 file changed, 151 insertions(+), 8 deletions(-) diff --git a/python/tests/attachments_providers/test_auto_framework_media.py b/python/tests/attachments_providers/test_auto_framework_media.py index 09c71c561..43699d53c 100644 --- a/python/tests/attachments_providers/test_auto_framework_media.py +++ b/python/tests/attachments_providers/test_auto_framework_media.py @@ -4,6 +4,11 @@ shared manager in the keyword arguments it accepts, carry auto's hide flags, and stay a strict no-op with attachments OFF. Skips when the installed framework version has no media adapter, unless TRACEAI_EXPECT_FRAMEWORK_ADAPTERS=1, which turns the skip into a failure. + +With the OpenAI instrumentor also enabled, each model call is uploaded ONCE, on the framework span +(the framework runs the nested SDK call under the owner mark). When auto's trace_config sets +pii_redaction or hide_error_details in code, Agno (which reads them only from the environment) +gets no media manager and its row is config_unsupported. """ import importlib @@ -49,14 +54,14 @@ def auto_state(monkeypatch): AgnoInstrumentorWrapper().uninstrument() except ImportError: pass - try: - from traceai_pydantic_ai import PydanticAIInstrumentor - - instrumentor = PydanticAIInstrumentor() - if instrumentor.is_instrumented_by_opentelemetry: - instrumentor.uninstrument() - except ImportError: - pass + for module, name in (("traceai_pydantic_ai", "PydanticAIInstrumentor"), ("traceai_openai", "OpenAIInstrumentor"), + ("traceai_langchain", "LangChainInstrumentor")): + try: + instrumentor = getattr(importlib.import_module(module), name)() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + except ImportError: + pass def _init(f01, monkeypatch, libraries, enabled=True, **options): @@ -202,3 +207,141 @@ def test_auto_pydantic_ai_off_is_a_no_op(f01, auto_state, monkeypatch): assert exporter.get_finished_spans() and all(not refs(s) for s in exporter.get_finished_spans()) finally: tracer_provider.shutdown() + + +# --------------------------------------------- framework + OpenAI through auto: one upload +def _is_openai_span(span): + return getattr(span.instrumentation_scope, "name", "") == "traceai_openai" + + +def _owners(exporter): + return [span for span in exporter.get_finished_spans() for _ in refs(span)] + + +def _sent(provider, image): + from conftest import b64 + + return [path for path, body, raw in provider.requests if b64(image).encode() in raw] + + +def _assert_one_upload_on_the_framework_span(f01, provider, manager, exporter, image): + flush(manager) + assert len(_sent(provider, image)) == 1, "the nested OpenAI SDK call did not send the image" + assert f01.payloads == [image], len(f01.payloads) + (owner,) = _owners(exporter) + assert not _is_openai_span(owner), owner.name + assert any(_is_openai_span(s) for s in exporter.get_finished_spans()), "no OpenAI span: not a 2-layer call" + + +def _openai_adapter(): + pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + + +@pytest.mark.parametrize("libraries", [["langchain", "openai"], ["openai", "langchain"]]) +def test_auto_langchain_over_openai_uploads_once(f01, provider, auto_state, monkeypatch, libraries): + _openai_adapter() + pytest.importorskip("langchain_openai") + _adapter("traceai_langchain._attachments", "langchain_attachment_capabilities") + from conftest import b64 + from langchain_core.messages import HumanMessage + from langchain_openai import ChatOpenAI + + tracer_provider, exporter = _init(f01, monkeypatch, libraries) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + rows = {r["name"]: r for r in traceai.report()["integrations"]} + assert [(rows[n]["status"], rows[n]["media"]) for n in ("langchain", "openai")] == [ + ("registered", "provider_adapter")] * 2 + image = png(seed=82) + model = ChatOpenAI(model="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + model.invoke([HumanMessage(content=[{"type": "text", "text": PROMPT}, { + "type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}])]) + _assert_one_upload_on_the_framework_span(f01, provider, manager, exporter, image) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +@pytest.mark.parametrize("libraries", [["agno", "openai"], ["openai", "agno"]]) +def test_auto_agno_over_openai_uploads_once(f01, provider, auto_state, monkeypatch, libraries): + _openai_adapter() + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, libraries) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + _agno_run(provider) + _assert_one_upload_on_the_framework_span(f01, provider, manager, exporter, IMAGE) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +# ------------------------------------------- Agno and a trace_config flag it cannot apply +@pytest.mark.parametrize("flag", ["pii_redaction", "hide_error_details"]) +def test_auto_agno_skips_media_for_a_flag_it_cannot_apply(f01, provider, auto_state, monkeypatch, flag): + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], trace_config=TraceConfig(**{flag: True})) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + row = {r["name"]: r for r in traceai.report()["integrations"]}["agno"] + assert (row["status"], row["media"]) == ("config_unsupported", "provider_adapter") + assert traceai.attachment_status()["adapters"]["agno"] == "config_unsupported" + _agno_run(provider) + flush(manager) + assert f01.payloads == [] and not _owners(exporter) + assert exporter.get_finished_spans() # Agno is still traced + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +@pytest.mark.parametrize("flag,env", [("pii_redaction", "FI_PII_REDACTION"), + ("hide_error_details", "FI_HIDE_ERROR_DETAILS")]) +def test_auto_agno_keeps_media_when_the_environment_sets_the_flag(f01, provider, auto_state, monkeypatch, flag, + env): + """Control: the same flag from the environment reaches Agno, so its media stays on.""" + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + monkeypatch.setenv(env, "true") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], trace_config=TraceConfig(**{flag: True})) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + _agno_run(provider) + flush(manager) + if flag == "pii_redaction": + assert f01.payloads == [] # the shared manager's PII policy denies media anyway + else: + assert f01.payloads == [IMAGE] and len(_owners(exporter)) == 1 + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_agno_unapplied_error_flag_leaves_the_call_to_openai(f01, provider, auto_state, monkeypatch): + """Agno without media marks nothing, so the OpenAI instrumentor (which applies the flag) owns the call.""" + _openai_adapter() + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno", "openai"], + trace_config=TraceConfig(hide_error_details=True)) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + rows = {r["name"]: r for r in traceai.report()["integrations"]} + assert (rows["agno"]["status"], rows["openai"]["status"]) == ("config_unsupported", "registered") + _agno_run(provider) + flush(manager) + assert f01.payloads == [IMAGE] + (owner,) = _owners(exporter) + assert _is_openai_span(owner), owner.name + finally: + manager.shutdown(5) + tracer_provider.shutdown()