diff --git a/.github/workflows/attachments-python-mcp-agents-tests.yml b/.github/workflows/attachments-python-mcp-agents-tests.yml new file mode 100644 index 000000000..31a7a8119 --- /dev/null +++ b/.github/workflows/attachments-python-mcp-agents-tests.yml @@ -0,0 +1,113 @@ +name: Python MCP and agent attachments + +# Real MCP, mcp-use, OpenAI Agents SDK, Claude Agent SDK, smolagents and +# firecrawl-py releases (declared floor and latest) through the Python +# instrumentors to a TLS loopback F01 attachment server. Outbound network is +# disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/mcp/**' + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/openai-agents/**' + - 'python/frameworks/claude-agent-sdk/**' + - 'python/frameworks/smolagents/**' + - 'python/frameworks/firecrawl/**' + - 'python/tests/attachments_mcp_agents/**' + - 'python/tests/attachments_providers/conftest.py' + - 'python/traceai/**' + - '.github/workflows/attachments-python-mcp-agents-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/mcp/**' + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/openai-agents/**' + - 'python/frameworks/claude-agent-sdk/**' + - 'python/frameworks/smolagents/**' + - 'python/frameworks/firecrawl/**' + - 'python/tests/attachments_mcp_agents/**' + - 'python/tests/attachments_providers/conftest.py' + - 'python/traceai/**' + - '.github/workflows/attachments-python-mcp-agents-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +env: + MCP_USE_ANONYMIZED_TELEMETRY: 'false' + MCP_USE_LANGFUSE: 'false' + MCP_USE_LAMINAR: 'false' + +jobs: + attachments: + name: ${{ matrix.package }} attachments (${{ matrix.tier }}) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - {package: core, tier: latest, python: '3.12', path: '', suite: test_tool_media.py, sdks: 'pillow'} + - {package: mcp, tier: floor, python: '3.11', path: mcp, suite: test_mcp_attachments.py, sdks: 'mcp==1.30.0'} + - {package: mcp, tier: latest, python: '3.12', path: mcp, suite: test_mcp_attachments.py, sdks: 'mcp==2.3.0'} + - {package: mcp-use, tier: floor, python: '3.11', path: mcp-use, suite: test_mcp_use_attachments.py, + sdks: 'mcp-use==1.7.1 langchain-core==1.6.12'} + - {package: mcp-use, tier: latest, python: '3.12', path: mcp-use, suite: test_mcp_use_attachments.py, + sdks: 'mcp-use==1.7.1'} + # 0.2.6 is the declared floor; computer calls get a function span from 0.10.0. + - {package: openai-agents, tier: floor, python: '3.11', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.2.6'} + - {package: openai-agents, tier: computer-span, python: '3.12', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.10.0 openai==2.19.0'} + - {package: openai-agents, tier: latest, python: '3.12', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.24.0'} + - {package: claude-agent-sdk, tier: floor, python: '3.11', path: claude-agent-sdk, + suite: test_claude_agent_sdk_attachments.py, sdks: 'claude-agent-sdk==0.1.0'} + - {package: claude-agent-sdk, tier: latest, python: '3.12', path: claude-agent-sdk, + suite: test_claude_agent_sdk_attachments.py, sdks: 'claude-agent-sdk==0.2.180'} + - {package: smolagents, tier: floor, python: '3.11', path: smolagents, suite: test_smolagents_attachments.py, + sdks: 'smolagents==1.12.0 openai'} + - {package: smolagents, tier: latest, python: '3.12', path: smolagents, suite: test_smolagents_attachments.py, + sdks: 'smolagents==1.26.0 openai'} + - {package: firecrawl, tier: floor, python: '3.11', path: firecrawl, suite: test_firecrawl_attachments.py, + sdks: 'firecrawl-py==4.46.2'} + - {package: firecrawl, tier: latest, python: '3.12', path: firecrawl, suite: test_firecrawl_attachments.py, + sdks: 'firecrawl-py==4.55.0'} + # traceai.init enabling all six by name (registry, report rows, capture through auto). + - {package: auto, tier: latest, python: '3.12', expect: '1', suite: test_auto_registry.py, + path: 'mcp mcp-use openai-agents claude-agent-sdk smolagents firecrawl', + sdks: 'mcp>=1.30.0,<2 mcp-use==1.7.1 openai-agents==0.24.0 claude-agent-sdk==0.2.180 smolagents==1.26.0 openai pillow firecrawl-py==4.55.0'} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install the source integration and the selected SDK + env: + SDKS: ${{ matrix.sdks }} + PKG_PATH: ${{ matrix.path }} + run: | + python -m venv .venv + extra="" + for pkg in $PKG_PATH; do extra="$extra -e python/frameworks/$pkg"; done + # shellcheck disable=SC2086 + .venv/bin/pip install -e python $extra opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' packaging $SDKS + .venv/bin/pip freeze | grep -iE '^(mcp|mcp-use|openai-agents|openai|claude-agent-sdk|smolagents|firecrawl-py|pillow|langchain-core)==' || true + - name: Run with outbound network disabled + env: + SUITE: ${{ matrix.suite }} + TRACEAI_EXPECT_MCP_AGENT_ADAPTERS: ${{ matrix.expect || '0' }} + run: | + sudo --preserve-env=MCP_USE_ANONYMIZED_TELEMETRY,MCP_USE_LANGFUSE,MCP_USE_LAMINAR,TRACEAI_EXPECT_MCP_AGENT_ADAPTERS unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + PYTHONPATH=. "$1" -m pytest "tests/attachments_mcp_agents/$3" -q -p no:cacheprovider -o addopts="" \ + -o faulthandler_timeout=120 + ' _ "$PWD/.venv/bin/python" "$PWD" "$SUITE" diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index a2fd637c4..1673fd5f9 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,6 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- A per-call ``CapturePolicy`` set in the OpenTelemetry context under + ``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only + add restrictions. A context marked under ``OWNER_KEY`` + (``traceai.with_attachment_ownership``) belongs to an adapter that captures + the media itself, so nothing is captured here for that call. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -26,6 +31,8 @@ from collections import OrderedDict from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple +from opentelemetry import context as otel_context + from .policy import CapturePolicy from .stream import StreamOptions from .types import Bytes, CorrelationHandle @@ -50,6 +57,13 @@ # mulaw have no F01 MIME, so they are never captured. SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", "wav": "audio/wav"} +# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the +# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. +CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" +OWNER_KEY = "traceai.auto.attachment_owner.v1" +# CapturePolicy facts that grant capture; every other fact only restricts. +_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) +_DENY_ALL = CapturePolicy(capture_content=False, requested=False) class Occurrence(NamedTuple): @@ -93,6 +107,40 @@ def _restricted(config, name: str) -> bool: return True +def merge_capture_policies(*policies: Any) -> CapturePolicy: + """Deny-wins intersection: a grant needs every policy, a restriction needs any one. + + ``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies. + """ + present = [policy for policy in policies if policy is not None] + if any(not isinstance(policy, CapturePolicy) for policy in present): + return _DENY_ALL + if not present: + return CapturePolicy() + values = {} + for item in dataclasses.fields(CapturePolicy): + facts = [getattr(policy, item.name) for policy in present] + values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts) + return CapturePolicy(**values) + + +def context_capture_policy(context=None) -> Any: + """The per-call policy in the (current) OpenTelemetry context; None when unset.""" + try: + return otel_context.get_value(CAPTURE_POLICY_KEY, context) + except Exception: + # An unreadable restriction is a restriction, never permission. + return _DENY_ALL + + +def owned_elsewhere(context=None) -> bool: + """True when another adapter marked this call's media as its own.""" + try: + return otel_context.get_value(OWNER_KEY, context) is not None + except Exception: + return True + + def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, base: Optional[CapturePolicy] = None) -> CapturePolicy: """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. @@ -103,6 +151,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal ancestry, so a tool screenshot is still treated as model input. """ base = base if isinstance(base, CapturePolicy) else CapturePolicy() + base = merge_capture_policies(base, context_capture_policy()) names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") if image and direction == "input" and _restricted(trace_config, "hide_input_images"): @@ -116,6 +165,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal changes["model_input"] = True if error_detail: changes["error_detail"] = True + if owned_elsewhere(): + changes["requested"] = False return dataclasses.replace(base, **changes) @@ -262,7 +313,7 @@ class Observation: def __init__(self, manager, span, namespace: str): self.manager, self.span, self.namespace = manager, span, namespace - self.key = span_key(span) if active(manager) else None + self.key = span_key(span) if active(manager) and not owned_elsewhere() else None self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() self._visited = 0 diff --git a/python/fi_instrumentation/attachments/tool_media.py b/python/fi_instrumentation/attachments/tool_media.py new file mode 100644 index 000000000..1ef0925e7 --- /dev/null +++ b/python/fi_instrumentation/attachments/tool_media.py @@ -0,0 +1,305 @@ +"""Shared tool-media helpers for MCP and agent adapters, built on ``media_capture``. + +MCP clients, ``mcp-use``, the OpenAI Agents SDK, the Claude Agent SDK, +smolagents and Firecrawl all observe media that a TOOL returned. These helpers +give them one set of rules (the TypeScript ``@traceai/mcp`` and +``@traceai/claude-agent-sdk`` semantics): + +- Tool media is tool output AND potential model input, so input and output + hide flags, ``hide_input_images`` for images and ``pii_redaction`` all deny + (deny wins; the core cannot relax a local restriction). +- Only typed inline values are read: base64 fields, data URIs, and in-memory + images. Nothing here fetches a URL, dereferences a resource URI or path, + opens a file, or reads a stream. A URL or file-backed value is skipped. +- Every value is preflighted before it is read or decoded. Each observation + has a bounded decode budget and part count; parts beyond them are skipped. +- A declared MIME is never trusted over the bytes: when the first bytes are a + known signature (``_SIGNATURES``) of ANOTHER modality (a PNG labelled + ``audio/wav``), the part is denied, so a mislabel cannot escape + ``hide_input_images`` or a modality restriction. Never relabelled. The + signatures checked are PNG, JPEG, GIF, WebP, BMP, TIFF, ISO-BMFF image + brands (``ftyp`` avif/heic/heix/mif1/msf1), WAV, Ogg, FLAC, ID3-tagged MP3 + and PDF; other bytes (MP4/WebM containers, MP3 without an ID3 tag) are + judged by their declared MIME only. ``application/octet-stream`` is the + ``file`` modality, so an octet-stream blob whose bytes carry a recognised + image, audio or PDF signature is skipped (fail-closed, never relabelled). +- A context marked by another adapter as the owner of this call's media + (``OWNER_KEY``, shared by name with ``traceai``) captures nothing here. + +Not part of ``fi_instrumentation.attachments.__all__``. Every function is +non-throwing and returns ``None``/empty on denial. +""" + +import base64 +import binascii +import dataclasses +import io +import sys +from typing import Any, Callable, Mapping, Optional + +from opentelemetry import context as otel_context + +from . import media_capture as mc +from .policy import CapturePolicy +from .types import MIME_MODALITIES, Bytes + +KiB = 1024 +MiB = 1024 * KiB +#: Decoded bytes one tool result may contribute (TypeScript MCP_MEDIA_DECODE_BUDGET). +TOOL_RESULT_BUDGET = 256 * KiB +#: Typed parts inspected per tool result. +MAX_TOOL_PARTS = 64 +#: In-memory images are PNG-encoded only up to this many pixels (no huge synchronous encodes). +MAX_IMAGE_PIXELS = 2048 * 2048 +#: The cross-adapter owner mark, by name (``media_capture.OWNER_KEY`` where the core defines it). +OWNER_KEY = getattr(mc, "OWNER_KEY", "traceai.auto.attachment_owner.v1") +URI_PREFIX = "fi-attachment:" +Occurrence = mc.Occurrence + + +def owned_elsewhere() -> bool: + """True when another adapter marked this call's media as its own (by context key name).""" + try: + return otel_context.get_value(OWNER_KEY) is not None + except Exception: + return True + + +def with_ownership(context=None): + """A context that tells downstream observers this call's media is captured already.""" + return otel_context.set_value(OWNER_KEY, True, context) + + +def uri(occurrence: Occurrence) -> str: + return URI_PREFIX + occurrence.occurrence_id + + +def tool_capture_policy(trace_config=None, capture: Optional[CapturePolicy] = None, *, image: bool = False, + error: bool = False) -> CapturePolicy: + """Deny-only policy for media a tool returned: input AND output restrictions both apply.""" + base = capture if isinstance(capture, CapturePolicy) else CapturePolicy() + seen_input = mc.local_capture_policy(trace_config, direction="input", image=image, error_detail=error, + base=base) + seen_output = mc.local_capture_policy(trace_config, direction="output", image=image, error_detail=error, + base=base) + return dataclasses.replace( + seen_input, + capture_content=seen_input.capture_content and seen_output.capture_content, + requested=seen_input.requested and seen_output.requested and not owned_elsewhere(), + contains_output=True, + model_input=True, + ) + + +def mime_kind(mime: Any) -> Optional[str]: + """The F01 modality of an exact MIME; None when the MIME is not in the F01 enum.""" + return MIME_MODALITIES.get(mime) if isinstance(mime, str) else None + + +#: Leading-byte signatures with an unambiguous F01 modality. ISO-BMFF ``ftyp`` boxes name an +#: image only for the HEIF/AVIF still-image brands listed here; other brands (mp42, isom, +#: ``M4A ``, ``qt ``) and WebM/Matroska are shared by audio and video, so those bytes are +#: judged by their declared MIME only. ``consistent`` decodes the first 12 bytes. +_SIGNATURES = ( + (0, b"\x89PNG\r\n\x1a\n", "image"), (0, b"\xff\xd8\xff", "image"), (0, b"GIF87a", "image"), + (0, b"GIF89a", "image"), (8, b"WEBP", "image"), (0, b"BM", "image"), (0, b"II*\x00", "image"), + (0, b"MM\x00*", "image"), (4, b"ftypavif", "image"), (4, b"ftypheic", "image"), (4, b"ftypheix", "image"), + (4, b"ftypmif1", "image"), (4, b"ftypmsf1", "image"), (8, b"WAVE", "audio"), + (0, b"OggS", "audio"), (0, b"fLaC", "audio"), (0, b"ID3", "audio"), (0, b"%PDF-", "document"), +) + + +def signature_kind(head: bytes) -> Optional[str]: + """The modality the first bytes name, or None when they name none (or are ambiguous).""" + for offset, magic, kind in _SIGNATURES: + if head[offset:offset + len(magic)] == magic: + if offset == 8 and head[:4] != b"RIFF": + continue + return kind + return None + + +def _declared_kind(value: str, mimes, raw: bool) -> Optional[str]: + if raw: + return mime_kind(mimes[0]) + end = value.find(";", 5, 128) + return mime_kind(value[5:end]) if value.startswith("data:") and end > 5 else None + + +def consistent(value: Any, mimes, raw: bool) -> bool: + """False when the base64 value's first bytes are a signature of another modality than declared.""" + if not isinstance(value, str): + return True # Not inline base64: the structural parse rejects it. + start = 0 + if not raw: + comma = value.find(",", 0, 128) + if comma < 0: + return True + start = comma + 1 + try: + head = base64.b64decode(value[start:start + 16], validate=True) + except (binascii.Error, ValueError): + return True # Malformed base64: strict decoding rejects it later. + found = signature_kind(head) + return found is None or found == _declared_kind(value, mimes, raw) + + +def _pil_image(value: Any) -> bool: + module = sys.modules.get("PIL.Image") + image_type = getattr(module, "Image", None) if module is not None else None + return isinstance(image_type, type) and isinstance(value, image_type) + + +def model_input_policy(trace_config=None, capture: Optional[CapturePolicy] = None, *, + image: bool = False) -> CapturePolicy: + """Deny-only policy for media an agent framework hands to a model as input.""" + policy = mc.local_capture_policy(trace_config, direction="input", image=image, base=capture) + return dataclasses.replace(policy, requested=policy.requested and not owned_elsewhere()) + + +class ToolMedia: + """One adapter's observation of one span, with a decode budget and registry dedup. + + ``namespace`` keeps positions of different adapters apart in the shared + per-manager registry; a repeated observation of the same span and + position returns the first occurrence and uploads nothing. + ``direction`` is ``"output"`` for media a tool returned (tool policy) or + ``"input"`` for media a framework sends to a model (model-input policy). + """ + + def __init__(self, manager, span, namespace: str, *, trace_config=None, + capture: Optional[CapturePolicy] = None, budget: int = TOOL_RESULT_BUDGET, + max_parts: int = MAX_TOOL_PARTS, direction: str = "output"): + self.manager, self.span = manager, span + self.direction = "input" if direction == "input" else "output" + self._trace_config, self._capture = trace_config, capture + self.remaining = budget + self._parts = 0 + self._max_parts = max_parts + self._observation = mc.Observation(manager, span, namespace) + if owned_elsewhere(): + self._observation.key = None + + def _policy(self, image: bool, error: bool) -> CapturePolicy: + if self.direction == "input": + return model_input_policy(self._trace_config, self._capture, image=image) + return tool_capture_policy(self._trace_config, self._capture, image=image, error=error) + + @property + def usable(self) -> bool: + return self._observation.usable and self.remaining > 0 + + @property + def captured(self) -> Mapping[str, Occurrence]: + return dict(self._observation.captured) + + def _visit(self, position: str, capture: Callable[[], Optional[Occurrence]]) -> Optional[Occurrence]: + if not self.usable or self._parts >= self._max_parts: + return None + self._parts += 1 + self._observation.visit(position, capture) + return self._observation.captured.get(position) + + def base64(self, position: str, read: Callable[[], Any], mime: Any, *, purpose: str, + expect: Optional[str] = None, role: str = "tool", tool_call_id: Optional[str] = None, + message_index: Optional[int] = None, part_index: Optional[int] = None, + error: bool = False) -> Optional[Occurrence]: + """A raw base64 value with a declared MIME (MCP ImageContent/AudioContent/blob, Anthropic source). + + ``expect`` ("image", "audio", ...) rejects a MIME of another modality, so a + misleading MIME cannot relabel an image to escape ``hide_input_images``. + ``read`` is called at most once, and only after a preflight admitted the part. + """ + kind = mime_kind(mime) + if kind is None or (expect is not None and kind != expect): + return None + return self._inline(position, read, (mime,), True, purpose, kind == "image", role, tool_call_id, + message_index, part_index, error) + + def data_uri(self, position: str, read: Callable[[], Any], mimes, *, purpose: str, role: str = "tool", + tool_call_id: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, error: bool = False) -> Optional[Occurrence]: + """A ``data:;base64,`` string whose MIME must be one of ``mimes`` (first is preflighted).""" + mimes = tuple(mime for mime in mimes if mime_kind(mime)) + if not mimes: + return None + image = any(mime_kind(mime) == "image" for mime in mimes) + return self._inline(position, read, mimes, False, purpose, image, role, tool_call_id, message_index, + part_index, error) + + def _inline(self, position, read, mimes, raw, purpose, image, role, tool_call_id, message_index, part_index, + error) -> Optional[Occurrence]: + seen = {} + + def once(): + if "value" not in seen: + value = read() + seen["value"] = value if consistent(value, mimes, raw) else None + return seen["value"] + + def capture(): + policy = self._policy(image, error) + return mc.attach_inline(self.manager, self.span, once, mimes, raw=raw, direction=self.direction, + purpose=purpose, capture=policy, message_index=message_index, + part_index=part_index, role=role, tool_call_id=tool_call_id, + max_bytes=self.remaining) + + before = self._observation.captured.get(position) + occurrence = self._visit(position, capture) + if occurrence is not None and before is None and "value" in seen: + value = seen["value"] + if not raw and isinstance(value, str): + value = value[value.find(",", 0, 128) + 1:] + self.remaining -= mc.encoded_length(value, self.remaining) or self.remaining + return occurrence + + def image(self, position: str, image: Any, *, purpose: str, role: str = "tool", + tool_call_id: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, error: bool = False) -> Optional[Occurrence]: + """An in-memory ``PIL.Image.Image``, PNG-encoded only after a pixel-bounded preflight. + + A file-backed image that is not loaded yet is skipped: encoding it would + read the file. PIL is never imported here; a value is an image only when + the application already imported PIL. + """ + if not _pil_image(image): + return None + + def capture(): + try: + fp = getattr(image, "fp", None) + if fp is not None and not isinstance(fp, io.BytesIO): + return None + width, height = image.size + bands = len(image.getbands()) + if not (0 < width and 0 < height and width * height <= MAX_IMAGE_PIXELS): + return None + # PNG never exceeds the raw pixel bytes plus per-row filter bytes and chunk framing. + bound = min(width * height * max(bands, 1) + height + 4 * KiB, self.remaining) + policy = self._policy(True, error) + options = dict(mime_type="image/png", direction=self.direction, purpose=purpose, + capture_policy=policy) + ticket = self.manager.preflight(self.span, max_bytes=bound, **options) + if not ticket.admitted: + return None + except Exception: + return None + try: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + data = buffer.getvalue() + except Exception: + data = b"" + try: + if not data or len(data) > ticket.max_bytes: + ticket.release() + return None + handle = ticket.attach(Bytes(data), **mc._fields(message_index, part_index, role, tool_call_id)) + if handle.admission != "queued": + return None + self.remaining -= len(data) + return Occurrence(handle.occurrence_id, "image/png") + except Exception: + return None + + return self._visit(position, capture) diff --git a/python/frameworks/claude-agent-sdk/README.md b/python/frameworks/claude-agent-sdk/README.md index 33a9fc14f..71fb0e01e 100644 --- a/python/frameworks/claude-agent-sdk/README.md +++ b/python/frameworks/claude-agent-sdk/README.md @@ -183,3 +183,37 @@ pytest tests/ -v ## License MIT + +## Tool-result media (attachments) + +Pass the shared manager: `ClaudeAgentInstrumentor().instrument(tracer_provider=..., attachments=manager)`. +The base64 `image` and PDF `document` blocks of a `ToolResultBlock` are +uploaded from the original SDK message, before any text flattening or +truncation, as `tool_result` occurrences with the `tool_use_id`. They belong +to the tool's span and are projected on it (`fi.attachment.refs`). The CLI +usually runs the PostToolUse hook before it streams the tool result, so with +attachments on, while that client's `receive_response()` conversation is +running, the tool span stays recording until its result has been captured (at +most 30 seconds, until that conversation ends, or until `disconnect()`) and +then ends at the time the hook recorded; its duration is unchanged. Each +client keeps its own waiting spans, so one conversation ending never ends +another client's. Tool media is captured from `receive_response()` only: +with `receive_messages()`, or for a hook that fires before +`receive_response()` starts, the hook ends the span immediately, as it +always does with attachments off. +Media bytes never reach `tool.output` or the conversation output text. The +SDK drops top-level user image blocks while parsing, so prompt images are not +captured. URL and file sources are never read. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py new file mode 100644 index 000000000..3a42f9096 --- /dev/null +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py @@ -0,0 +1,151 @@ +"""Tool-result media of a Claude Agent SDK conversation, through the shared attachment manager. + +The SDK parses each CLI ``user`` message into a ``UserMessage`` whose +``ToolResultBlock.content`` keeps the original Anthropic content blocks. The +base64 ``image`` and PDF ``document`` blocks there are uploaded BEFORE any text +flattening or truncation, at the tool's span (with ``tool_call_id``): + +- on the tool span, while it is still recording: before the PostToolUse hook + (result-first) or after it, because with capture on and the client's + ``receive_response()`` conversation live, the hook defers the span's end + until its result has been captured (hooks-first, the CLI's usual order); + the span then ends at the time the hook recorded. Each client has its own + deferral tables (``_hooks.ToolDeferrals``); +- for a result with no traced or recording tool span (for example a hook that + ran before ``receive_response()`` started), on the conversation span. + +Every occurrence is projected (``fi.attachment.refs``) on the live span that +is its upload parent. + +URL and file sources, paths and transcripts are never read. The SDK drops +top-level user image blocks during parsing, so prompt images are not +observable here (``source_unavailable``). Model calls run inside the Claude +Code process, so no in-process provider adapter sees the same bytes. +""" + +from types import MappingProxyType +from typing import Any, List, Mapping + +__all__ = ("capture_tool_results", "claude_agent_attachment_capabilities", "media_safe", "tool_result_ids") + +claude_agent_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tool_result.output",), + "sources": ("inline_base64",), + "content_types": ("image", "document"), + "unsupported": ("url_source", "file_source", "prompt_images", "transcript", "local_path"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_MEDIA_TYPES = ("image", "document", "audio") +_MAX_BLOCKS = 64 + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def tool_result_ids(message: Any) -> List[str]: + """The tool_use_ids of the ``ToolResultBlock``s in one ``UserMessage`` (bounded). Never raises.""" + try: + content = _get(message, "content") + if not isinstance(content, list): + return [] + return [block.tool_use_id for block in content[:_MAX_BLOCKS] + if type(block).__name__ == "ToolResultBlock" and isinstance(_get(block, "tool_use_id"), str)] + except Exception: + return [] + + +def capture_tool_results(manager, message: Any, conversation_span: Any, active_spans: Mapping[str, Any], + deferred_spans: Mapping[str, Any] = _EMPTY, *, message_index: int, trace_config=None, + capture=None) -> int: + """Capture the media blocks of every ``ToolResultBlock`` in one ``UserMessage``. Never raises. + + ``active_spans`` maps a tool_use_id to its open span (PostToolUse not run yet) and + ``deferred_spans`` to a span PostToolUse completed whose end waits for this result; + both are the hooks' tables and both spans are still recording. Any other result is + captured on the (recording) conversation span. Returns the number of admitted blocks. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return 0 + admitted = 0 + try: + if not active(manager): + return 0 + content = _get(message, "content") + if not isinstance(content, list): + return 0 + for block in content[:MAX_TOOL_PARTS]: + if type(block).__name__ != "ToolResultBlock": + continue + blocks = _get(block, "content") + if not isinstance(blocks, list): + continue + tool_use_id = _get(block, "tool_use_id") + entry = None + if isinstance(tool_use_id, str): + entry = active_spans.get(tool_use_id) or deferred_spans.get(tool_use_id) + target = entry[0] if entry is not None else conversation_span + if not _recording(target): + target = conversation_span + error = _get(block, "is_error") is True + media = ToolMedia(manager, target, "claude_agent_sdk", trace_config=trace_config, capture=capture) + for part_index, part in enumerate(blocks[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(part, "type") + if kind not in ("image", "document"): + continue + source = _get(part, "source") + if _get(source, "type") != "base64": + continue # url / file sources are never fetched + mime = _get(source, "media_type") + expect = "image" if kind == "image" else "document" + found = media.base64("tool_result/%s/%d" % (tool_use_id, part_index), + lambda s=source: _get(s, "data"), mime, purpose="tool_result", + expect=expect, role="tool", tool_call_id=tool_use_id, + message_index=min(message_index, 65535), part_index=part_index, + error=error) + admitted += found is not None + except Exception: + return admitted + return admitted + + +def _recording(span: Any) -> bool: + try: + return bool(span.is_recording()) + except Exception: + return False + + +def media_safe(value: Any, depth: int = 0) -> Any: + """A telemetry copy without inline media bytes: base64 fields of media blocks become ``"[omitted]"``. + + Used before tool responses are JSON-serialized and truncated, so bytes never + reach span attributes as text. Non-media values are returned unchanged. + """ + if depth > 6: + return value + if isinstance(value, list): + return [media_safe(item, depth + 1) for item in value] + if not isinstance(value, Mapping): + return value + media = value.get("type") in _MEDIA_TYPES + result = {} + for key, item in value.items(): + if media and key in ("data", "base64", "blob") and isinstance(item, str): + result[key] = "[omitted]" + elif media and key in ("source", "file") and isinstance(item, Mapping): + result[key] = {k: ("[omitted]" if k in ("data", "base64") and isinstance(v, str) else v) + for k, v in item.items()} + else: + result[key] = media_safe(item, depth + 1) + return result diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py index 5cdb1cc89..296812e27 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py @@ -14,8 +14,11 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from ._attachments import capture_tool_results, tool_result_ids from ._attributes import ClaudeAgentAttributes, ClaudeAgentSpanKind from ._hooks import ( + ToolDeferrals, + _active_tool_spans, create_pre_tool_use_hook, create_subagent_start_hook, create_post_tool_use_hook, @@ -23,6 +26,8 @@ clear_parent_span, set_mcp_servers, clear_active_tool_spans, + end_deferred_tool, + end_deferred_tool_spans, safe_json_serialize, ) @@ -166,12 +171,16 @@ def _flatten_content(self, content: Any) -> str: return str(content) -def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: +def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer, + deferrals: Optional[ToolDeferrals] = None) -> None: """Inject OpenTelemetry tracing hooks into ClaudeAgentOptions. Args: options: ClaudeAgentOptions instance tracer: OpenTelemetry tracer + deferrals: the client's deferral tables when tool-result media capture is on: while + its receive_response() conversation is live, a completed tool span stays + recording until its result is captured (it keeps the hook's end time) """ if not hasattr(options, "hooks"): return @@ -192,7 +201,8 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: from claude_agent_sdk import HookMatcher pre_hook = create_pre_tool_use_hook(tracer) - post_hook = create_post_tool_use_hook(tracer) + post_hook = (create_post_tool_use_hook(tracer, deferrals=deferrals) if deferrals is not None + else create_post_tool_use_hook(tracer)) start_hook = create_subagent_start_hook(tracer) # Insert at beginning to ensure we trace before user hooks @@ -213,13 +223,20 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: def wrap_claude_sdk_client( - original_class: Any, tracer: trace_api.Tracer + original_class: Any, + tracer: trace_api.Tracer, + attachments: Any = None, + trace_config: Any = None, + attachment_capture: Any = None, ) -> Any: """Wrap ClaudeSDKClient with tracing. Args: original_class: The original ClaudeSDKClient class tracer: OpenTelemetry tracer + attachments: the shared attachment manager for tool-result media (None: off) + trace_config: TraceConfig whose hide flags and PII redaction deny media capture + attachment_capture: CapturePolicy that can only add restrictions Returns: Wrapped class with tracing @@ -231,8 +248,13 @@ class TracedClaudeSDKClient(original_class): def __init__(self, *args: Any, **kwargs: Any): # Extract options for hook injection options = kwargs.get("options") or (args[0] if args else None) + # This client's deferred tool ends (tool-result media capture only). + deferrals = ToolDeferrals() if attachments is not None else None if options: - inject_tracing_hooks(options, tracer) + if deferrals is not None: + inject_tracing_hooks(options, tracer, deferrals=deferrals) + else: + inject_tracing_hooks(options, tracer) # Store MCP server configs for tool attribution mcp_servers = getattr(options, "mcp_servers", None) @@ -246,6 +268,7 @@ def __init__(self, *args: Any, **kwargs: Any): self._prompt: Optional[str] = None self._start_time: Optional[float] = None self._options = options + self._tool_deferrals = deferrals async def query(self, *args: Any, **kwargs: Any) -> Any: """Capture prompt and timestamp when query starts.""" @@ -267,6 +290,9 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: attributes=span_attributes, ) as conversation_span: set_parent_span(conversation_span) + deferrals = getattr(self, "_tool_deferrals", None) + if deferrals is not None: + deferrals.live += 1 # Initialize turn tracker turn_tracker = TurnTracker( @@ -277,10 +303,12 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: collected_messages: List[Dict[str, Any]] = [] result_usage: Optional[Dict[str, Any]] = None + message_index = 0 try: async for msg in messages: msg_type = type(msg).__name__ + message_index += 1 if msg_type == "AssistantMessage": # Start new turn @@ -295,6 +323,25 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: }) elif msg_type == "UserMessage": + # Tool-result media first, from the original typed blocks, + # before any text flattening or truncation. + # A tool span the PostToolUse hook completed is still recording + # (deferred end), so the media refs are projected on it. + if attachments is not None: + try: + capture_tool_results( + attachments, + msg, + conversation_span, + _active_tool_spans, + deferrals.ends if deferrals is not None else {}, + message_index=message_index, + trace_config=trace_config, + capture=attachment_capture, + ) + finally: + for tool_use_id in tool_result_ids(msg): + end_deferred_tool(deferrals, tool_use_id) # End current turn, mark next start turn_tracker.end_current_turn() turn_tracker.mark_next_start() @@ -349,6 +396,10 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: finally: turn_tracker.end_current_turn() clear_parent_span() + if deferrals is not None: + # Only this conversation's deferred tool spans end here, at their hook's time. + deferrals.live -= 1 + end_deferred_tool_spans(deferrals) clear_active_tool_spans() def _build_span_attributes(self) -> Dict[str, Any]: @@ -414,6 +465,15 @@ def _extract_content(self, msg: Any) -> Optional[str]: parts.append(block.text) elif type(block).__name__ == "ToolResultBlock": result = getattr(block, "content", "") + if isinstance(result, list): + # Only text blocks: media bytes never become span text. + result = " ".join( + item.get("text", "") + for item in result + if isinstance(item, dict) and isinstance(item.get("text"), str) + ) + elif not isinstance(result, str): + result = "" parts.append(f"[Tool Result: {result[:100]}...]") return " ".join(parts) if parts else None @@ -488,4 +548,15 @@ async def __aenter__(self) -> "TracedClaudeSDKClient": async def __aexit__(self, *args: Any) -> None: await super().__aexit__(*args) + if attachments is not None and hasattr(original_class, "disconnect"): + + async def disconnect(self: Any, *args: Any, **kwargs: Any) -> Any: + """Disconnect; this client's tool spans still waiting for their result end now.""" + try: + return await super(TracedClaudeSDKClient, self).disconnect(*args, **kwargs) + finally: + end_deferred_tool_spans(getattr(self, "_tool_deferrals", None)) + + TracedClaudeSDKClient.disconnect = disconnect + return TracedClaudeSDKClient diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py index fa2c876b1..89a4f08f4 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py @@ -7,12 +7,14 @@ import json import logging import time +from collections import OrderedDict from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional, Tuple from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from ._attachments import media_safe from ._attributes import ( ClaudeAgentAttributes, ClaudeAgentSpanKind, @@ -26,6 +28,33 @@ # Key: tool_use_id, Value: (span, start_time, tool_name) _active_tool_spans: Dict[str, Tuple[Span, float, str]] = {} +_MAX_DEFERRED_ENDS = 256 +#: A deferred tool span ends at most this long after its hook, even if its result never arrives. +DEFERRED_END_TIMEOUT_S = 30.0 + + +class ToolDeferrals: + """One client's tool spans whose end waits for their result (tool-result media capture on). + + The SDK streams a tool result AFTER the PostToolUse hook, so while that client's + ``receive_response()`` conversation span is live, a completed tool span stays recording + until its ToolResultBlock has been captured (its media refs are projected on it), then + ends at the time the hook recorded. Outside a live conversation (``receive_messages()``, + or a hook that fires before ``receive_response()`` starts) the hook ends the span as + before. Each client has its own tables, so one conversation's end flushes only its own. + + ``ends``: tool_use_id -> (span, end_time_ns, timer handle or None), bounded and + time-limited. ``results_seen``: results streamed while their span was still open + (result-first), so PostToolUse ends those now. ``live``: open conversation spans. + """ + + __slots__ = ("ends", "results_seen", "live") + + def __init__(self) -> None: + self.ends: "OrderedDict[str, Tuple[Span, int, Any]]" = OrderedDict() + self.results_seen: "OrderedDict[str, bool]" = OrderedDict() + self.live = 0 + # Tools that launch a subagent: "Task" in older Claude Code releases, "Agent" in current ones SUBAGENT_TOOLS = ("Task", "Agent") @@ -262,11 +291,86 @@ async def pre_tool_use_hook( return pre_tool_use_hook -def create_post_tool_use_hook(tracer: trace_api.Tracer) -> Callable: +def _end_deferred(entry: Tuple[Span, int, Any]) -> None: + span, end_ns, handle = entry + try: + if handle is not None: + handle.cancel() + except Exception: + pass + try: + span.end(end_time=end_ns) + except Exception as e: + logger.debug(f"Failed to end deferred tool span: {e}") + + +def _expire_deferred(deferrals: ToolDeferrals, tool_use_id: str, entry: Tuple[Span, int, Any]) -> None: + if deferrals.ends.get(tool_use_id) is entry: + del deferrals.ends[tool_use_id] + _end_deferred(entry) + + +def _defer_tool_end(deferrals: ToolDeferrals, tool_use_id: str, span: Span, end_ns: int) -> bool: + """Keep a completed tool span recording until its result is captured. False: end it now.""" + if deferrals.results_seen.pop(tool_use_id, None): + return False # Its result was already captured on the open span. + if deferrals.live <= 0: + return False # No receive_response() conversation will capture its result: end now. + try: + import asyncio + + loop = asyncio.get_running_loop() + except Exception: + return False # No asyncio loop to bound the wait (another async backend): end now. + try: + previous = deferrals.ends.pop(tool_use_id, None) + if previous is not None: + _end_deferred(previous) + holder: List[Any] = [] + handle = loop.call_later(DEFERRED_END_TIMEOUT_S, + lambda: _expire_deferred(deferrals, tool_use_id, holder[0])) + entry = (span, end_ns, handle) + holder.append(entry) + deferrals.ends[tool_use_id] = entry + while len(deferrals.ends) > _MAX_DEFERRED_ENDS: + _, oldest = deferrals.ends.popitem(last=False) + _end_deferred(oldest) + return True + except Exception: + return False + + +def end_deferred_tool(deferrals: Optional[ToolDeferrals], tool_use_id: Any) -> None: + """End one of this client's deferred tool spans (its result was processed), at its hook's time.""" + if deferrals is None or not isinstance(tool_use_id, str): + return + entry = deferrals.ends.pop(tool_use_id, None) + if entry is not None: + _end_deferred(entry) + elif tool_use_id in _active_tool_spans: + deferrals.results_seen[tool_use_id] = True + while len(deferrals.results_seen) > _MAX_DEFERRED_ENDS: + deferrals.results_seen.popitem(last=False) + + +def end_deferred_tool_spans(deferrals: Optional[ToolDeferrals]) -> None: + """End every deferred tool span of ONE client at its recorded time (its conversation ended).""" + if deferrals is None: + return + deferrals.results_seen.clear() + while deferrals.ends: + _, entry = deferrals.ends.popitem(last=False) + _end_deferred(entry) + + +def create_post_tool_use_hook(tracer: trace_api.Tracer, deferrals: Optional[ToolDeferrals] = None) -> Callable: """Create a PostToolUse hook for completing tool traces. Args: tracer: OpenTelemetry tracer instance + deferrals: the client's deferral tables when tool-result media capture is on: while + its receive_response() conversation is live, the completed span stays recording + until its tool result is captured (it keeps the hook's end time) Returns: Async hook function compatible with Claude Agent SDK @@ -302,12 +406,14 @@ async def post_tool_use_hook( return {} span, start_time, _ = span_info + end_ns = time.time_ns() duration_ms = (time.time() - start_time) * 1000 - # Set output and duration + # Set output and duration. Inline media bytes never go through text serialization + # or truncation; the attachment path uploads them from the typed tool result. span.set_attribute( ClaudeAgentAttributes.TOOL_OUTPUT, - safe_json_serialize(tool_response), + safe_json_serialize(media_safe(tool_response)), ) span.set_attribute(ClaudeAgentAttributes.TOOL_DURATION_MS, duration_ms) @@ -341,8 +447,11 @@ async def post_tool_use_hook( }, ) - # End the span - span.end() + # End the span. With media capture on and a live conversation, the tool result + # (streamed after this hook) is captured on the still-recording span first; it + # ends at end_ns. + if not (deferrals is not None and _defer_tool_end(deferrals, tool_use_id, span, end_ns)): + span.end() # Clean up subagent span if this launched a subagent if tool_name in SUBAGENT_TOOLS: @@ -438,6 +547,9 @@ def clear_active_tool_spans() -> None: """ global _active_tool_spans, _subagent_spans + # Deferred tool spans belong to one client; its conversation flushes its own + # (end_deferred_tool_spans) before calling this. + # End any orphaned spans for tool_use_id, (span, start_time, tool_name) in _active_tool_spans.items(): try: diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py index 8322da2b7..74b3bff25 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py @@ -20,6 +20,23 @@ _instruments = ("claude-agent-sdk >= 0.1.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class ClaudeAgentInstrumentor(BaseInstrumentor): """OpenTelemetry Instrumentor for Claude Agent SDK. @@ -97,6 +114,12 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider, ) + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + self._attachments = _attachment_manager(kwargs) + self._trace_config = kwargs.get("config") + self._attachment_capture = kwargs.get("attachment_capture") + # Try to import and patch Claude Agent SDK try: self._patch_claude_agent_sdk() @@ -126,9 +149,16 @@ def _patch_claude_agent_sdk(self) -> None: self._original_client_class = claude_agent_sdk.ClaudeSDKClient # Create wrapped class + attachments = getattr(self, "_attachments", None) + media = {} if attachments is None else dict( + attachments=attachments, + trace_config=getattr(self, "_trace_config", None), + attachment_capture=getattr(self, "_attachment_capture", None), + ) wrapped_class = wrap_claude_sdk_client( self._original_client_class, self._tracer, + **media, ) # Replace in module diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index 03d710941..f68184784 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -114,3 +114,26 @@ Direct `FirecrawlClient` and `AsyncFirecrawlClient` instances look methods up on ## Watchers `Firecrawl.watcher()` polls `get_crawl_status` from its own thread. Each poll produces a separate root span; a swallowed poll error still appears as ERROR. `AsyncFirecrawl.watcher()` polls in the iterating task, so its spans inherit that task's current parent. + +## Screenshots and media (attachments) + +Pass the shared manager: `FirecrawlInstrumentor().instrument(tracer_provider=..., attachments=manager)`. +For `scrape`, `crawl`, `batch_scrape` and their status calls, a returned +Document's `screenshot`, `actions.screenshots`, `audio` and `video` are +uploaded on the operation span when they are inline `data:` URIs (up to 16 +documents and 8 MiB per result). Firecrawl Cloud returns hosted URLs for +these: they are never fetched, so nothing is uploaded for them. No extra +request, crawl or browser session is started. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py index 389ff8edf..748c1a1b4 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -56,8 +56,30 @@ } +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class FirecrawlInstrumentor(BaseInstrumentor): - """Instrument Firecrawl v2 scraping, extraction, agent and browser operations.""" + """Instrument Firecrawl v2 scraping, extraction, agent and browser operations. + + With an enabled shared attachment manager (``attachments=``), inline + ``data:`` screenshots and media of returned Documents are uploaded. + Hosted URLs are never fetched. + """ __slots__ = ("_original_methods",) @@ -73,7 +95,9 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() tracer = FITracer(trace_api.get_tracer(__name__, __version__, tracer_provider), config=config) self._original_methods: Dict[Tuple[str, str, str], Any] = {} - self._state = _State() + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + self._state = _State(_attachment_manager(kwargs), kwargs.get("attachment_capture")) for module_name, class_name, is_async in ( (_SYNC_MODULE, _SYNC_CLASS, False), diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py b/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py new file mode 100644 index 000000000..8925ffd3a --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py @@ -0,0 +1,84 @@ +"""Firecrawl returned screenshots and media, through the shared attachment manager. + +A firecrawl-py v2 ``Document`` carries ``screenshot``, ``audio`` and ``video`` +strings, and browser ``actions`` may carry ``screenshots``. Only values that +are inline ``data:;base64,`` URIs are uploaded, at the operation span, +as tool results. Firecrawl Cloud returns hosted URLs for these: they are never +fetched (the SDK core has no approved remote retrieval yet), so they are +``source_unavailable``. Nothing here starts a crawl, a browser or a request. +""" + +from types import MappingProxyType +from typing import Any, Mapping + +__all__ = ("capture_firecrawl_media", "firecrawl_attachment_capabilities") + +#: Operations whose results carry Documents. +MEDIA_METHODS = frozenset(( + "scrape", "crawl", "batch_scrape", "wait_batch_scrape", "get_crawl_status", "get_batch_scrape_status", +)) +MAX_DOCUMENTS = 16 +MAX_RESULT_BYTES = 8 * 1024 * 1024 +_IMAGE = ("image/png", "image/jpeg", "image/webp") +_AUDIO = ("audio/mpeg", "audio/wav", "audio/ogg", "audio/webm", "audio/mp4", "audio/flac") +_VIDEO = ("video/mp4", "video/webm") +_EMPTY: Mapping[str, Any] = MappingProxyType({}) + +firecrawl_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": tuple(sorted(MEDIA_METHODS)), + "sources": ("inline_data_uri",), + "fields": ("screenshot", "actions.screenshots", "audio", "video"), + "unsupported": ("hosted_url", "remote_url", "images_links", "pdf_url"), + "max_documents": MAX_DOCUMENTS, + "max_decoded_bytes_per_result": MAX_RESULT_BYTES, +}) + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def _documents(method: str, result: Any) -> list: + if method == "scrape": + return [result] + data = _get(result, "data") + return list(data[:MAX_DOCUMENTS]) if isinstance(data, (list, tuple)) else [] + + +def capture_firecrawl_media(manager, span, method: str, result: Any, *, trace_config=None, + capture=None) -> Mapping[str, Any]: + """Capture inline media of one operation's returned Documents. Never raises.""" + if method not in MEDIA_METHODS: + return _EMPTY + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import ToolMedia + except ImportError: + return _EMPTY + try: + if not active(manager): + return _EMPTY + media = ToolMedia(manager, span, "firecrawl", trace_config=trace_config, capture=capture, + budget=MAX_RESULT_BYTES) + for index, document in enumerate(_documents(method, result)[:MAX_DOCUMENTS]): + values = [("screenshot", _get(document, "screenshot"), _IMAGE), + ("audio", _get(document, "audio"), _AUDIO), + ("video", _get(document, "video"), _VIDEO)] + shots = _get(_get(document, "actions"), "screenshots") + if isinstance(shots, (list, tuple)): + values.extend(("actions.screenshots.%d" % n, shot, _IMAGE) for n, shot in enumerate(shots[:16])) + for field, value, mimes in values: + if not media.usable: + break + if not isinstance(value, str) or not value.startswith("data:"): + continue # hosted URLs are never fetched + media.data_uri("document/%d/%s" % (index, field), lambda v=value: v, mimes, + purpose="tool_result", role="tool", message_index=index) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 4e4f6b75e..34e132a2e 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -107,8 +107,11 @@ class _State: instrumentor is removed. """ - def __init__(self) -> None: + def __init__(self, attachments: Any = None, attachment_capture: Any = None) -> None: self.enabled = True + # The shared attachment manager (None unless an enabled one was passed). + self.attachments = attachments + self.attachment_capture = attachment_capture def _span_kind(method_name: str) -> str: @@ -406,6 +409,13 @@ def _finish_ok( ) -> None: try: self._record_result(span, instance, result, kwargs) + if self._state.attachments is not None: + from traceai_firecrawl._attachments import capture_firecrawl_media + + capture_firecrawl_media( + self._state.attachments, span, self._method_name, result, + trace_config=self._config, capture=self._state.attachment_capture, + ) except Exception: logger.debug("Could not read %s result", self._span_name, exc_info=True) finally: diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 6c0c1b30a..450fede1e 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -369,3 +369,27 @@ path is exported either way. The example test runs `examples/agent_with_tools.py` as its own process, once alone and once with `traceai_mcp` instrumented. The suite turns off mcp-use's usage telemetry. + +## Tool media (attachments) + +`FutureAGICallback(attachments=manager)` (default: the tracer provider's +`attachment_manager`) uploads the typed base64 blocks mcp-use 1.7 makes from +an MCP tool result: `image` and `file` (inline resource blob), on the tool +span as `tool_result` occurrences with the tool call id (an `audio` block is +captured when it reaches the callback as a typed block; LangChain 1.x turns +audio tool content into a string first, so nothing is uploaded for it). At +most 64 blocks and 256 KiB of decoded bytes per result. +`attachment_capture=` only adds restrictions. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py new file mode 100644 index 000000000..00c864136 --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py @@ -0,0 +1,69 @@ +"""Tool media of an mcp-use agent run, through the shared attachment manager. + +mcp-use 1.7 converts an MCP ``CallToolResult`` into LangChain content blocks +(``agents/adapters/langchain_adapter.py``): ``ImageContent`` becomes +``{"type": "image", "source_type": "base64", "data", "mime_type"}``, +``AudioContent`` an ``"audio"`` block and an ``EmbeddedResource`` blob a +``"file"`` block. Only those typed inline blocks are observed. Text, URLs, +resource URIs and resource links are never fetched or scraped. Each block is +preflighted before its base64 is read; at most 64 blocks and 256 KiB of +decoded bytes per tool result. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Optional + +__all__ = ("capture_tool_blocks", "mcp_use_attachment_capabilities") + +mcp_use_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tool.output",), + "sources": ("inline_base64",), + "content_types": ("image", "audio", "file"), + # LangChain stringifies tool content it does not know as a block (audio), so no typed block reaches callbacks. + "unsupported": ("url", "resource_uri", "resource_link", "local_path", "audio_stringified_by_langchain"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_EXPECT = {"image": "image", "audio": "audio", "file": None} + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else None + except Exception: + return None + + +def capture_tool_blocks(manager, span, content: Any, *, trace_config=None, capture=None, + tool_call_id: Optional[str] = None, error: bool = False) -> Mapping[str, Any]: + """Capture the typed base64 blocks of one tool result. Never raises; returns admitted positions.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return _EMPTY + try: + if not active(manager) or not isinstance(content, (list, tuple)): + return _EMPTY + media = ToolMedia(manager, span, "mcp_use", trace_config=trace_config, capture=capture) + for index, block in enumerate(content[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(block, "type") + if kind not in _EXPECT: + continue + # LangChain's v0 shape (source_type + data) and its v1 standard shape (base64). + if _get(block, "source_type") == "base64": + field = "data" + elif _get(block, "base64") is not None: + field = "base64" + else: + continue # url / id sources are never fetched + media.base64("tool/%d" % index, lambda b=block, f=field: _get(b, f), _get(block, "mime_type"), + purpose="tool_result", expect=_EXPECT[kind], role="tool", tool_call_id=tool_call_id, + part_index=index, error=error) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index 08b7e6348..a603ba26a 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -140,6 +140,8 @@ class _Run: chunks: int = 0 # A tool run's name as LangChain reported it (not capped or redacted). tool_name: Optional[str] = None + # The tool call id LangChain reported, when it is a safe token. + tool_call_id: Optional[str] = None def _describe(error: BaseException) -> str: @@ -329,6 +331,13 @@ class FutureAGICallback(BaseCallbackHandler): redact: secret strings to remove from everything written (for example an LLM key or an MCP server header the environment does not hold). + attachments: the shared attachment manager for tool media (images, + audio and inline resource blobs an MCP tool returned). Defaults + to ``tracer_provider.attachment_manager``. Capture is off unless + that manager is enabled; ``config`` hide flags and PII redaction + deny it. + attachment_capture: a ``fi_instrumentation.attachments.CapturePolicy`` + that can only add restrictions. """ raise_error = False @@ -345,6 +354,8 @@ def __init__( tracer_provider: Optional[trace_api.TracerProvider] = None, config: Optional[TraceConfig] = None, redact: Iterable[str] = (), + attachments: Any = None, + attachment_capture: Any = None, ) -> None: super().__init__() if config is None: @@ -373,6 +384,11 @@ def __init__( self._hide_outputs = bool(config.hide_outputs) self._pii = bool(config.pii_redaction) self._secrets = Secrets(secrets) + self._config = config + self._attachments = ( + attachments if attachments is not None else getattr(provider, "attachment_manager", None) + ) + self._attachment_capture = attachment_capture self._lock = threading.RLock() self._runs: Dict[UUID, _Run] = {} self._roots: "OrderedDict[UUID, _Root]" = OrderedDict() @@ -1042,6 +1058,9 @@ def on_tool_start( return if isinstance(raw_name, str): run.tool_name = raw_name + raw_call_id = kwargs.get("tool_call_id") + if isinstance(raw_call_id, str): + run.tool_call_id = raw_call_id if run.span is None: return arguments = _json(inputs) if isinstance(inputs, Mapping) else input_str @@ -1092,6 +1111,8 @@ def on_tool_end( ) except Exception: logger.debug("Could not classify the tool result", exc_info=True) + if run.span is not None and self._attachments is not None: + self._capture_tool_media(run, output, failure is not None) text = _tool_output_text(output) self._remember(root, [text], output=True) if failure is not None and root is not None: @@ -1115,6 +1136,16 @@ def on_tool_end( except Exception: logger.debug("mcp-use callback on_tool_end failed", exc_info=True) + def _capture_tool_media(self, run: _Run, output: Any, failed: bool) -> None: + """Upload the typed inline media blocks a tool returned (never URLs or paths).""" + try: + from traceai_mcp_use._attachments import capture_tool_blocks + except ImportError: + return + content = getattr(output, "content", output) + capture_tool_blocks(self._attachments, run.span, content, trace_config=self._config, + capture=self._attachment_capture, tool_call_id=run.tool_call_id, error=failed) + def on_tool_error( self, error: BaseException, diff --git a/python/frameworks/mcp/README.md b/python/frameworks/mcp/README.md index 8f1a00c87..5b344ade3 100644 --- a/python/frameworks/mcp/README.md +++ b/python/frameworks/mcp/README.md @@ -45,3 +45,49 @@ enabled is attached only if the reader is enabled when it consumes the item. An already-running request retains the context under which it started. Native MCP SDK tracing remains independent and can continue when this instrumentor is off, particularly with MCP 2.x. + +## Tool and resource media (attachments) + +`MCPInstrumentor` propagates context only and never captures media. To upload +the images, audio and inline resource blobs a server returns, wrap the client +session: + +```python +from traceai_mcp import wrap_mcp_client + +session = wrap_mcp_client(session, tracer_provider=trace_provider) +result = await session.call_tool("screenshot", {}) # unchanged SDK result +``` + +Each `call_tool` / `read_resource` gets an `mcp.tools/call` / +`mcp.resources/read` CLIENT span. `ImageContent.data`, `AudioContent.data` +and `BlobResourceContents.blob` (in an `EmbeddedResource` or a +`resources/read` result) are uploaded as `tool_result` occurrences: at most 64 +parts and 256 KiB of decoded bytes per result. `resource_link` blocks, +resource URIs and `structuredContent` are never read. A MIME of another +modality than the block type, or outside the attachment MIME list, uploads +nothing. So do bytes whose leading signature names another modality than the +declared MIME, for example a PNG labelled `audio/wav`. The signatures checked +are PNG, JPEG, GIF, WebP, BMP, TIFF, the HEIF/AVIF image brands of an ISO-BMFF +`ftyp` box (`avif`, `heic`, `heix`, `mif1`, `msf1`), WAV, Ogg, FLAC, MP3 with +an ID3 tag, and PDF. Other bytes, including MP4 and WebM containers and MP3 +without an ID3 tag, are judged by the declared MIME only. +`application/octet-stream` is the generic file modality, so an octet-stream +blob whose bytes carry one of those image, audio or PDF signatures is skipped +rather than relabelled. `attachments=` overrides the provider's manager; `trace_config=` and +`attachment_capture=` only add restrictions. Wrapping a wrapped session +returns it unchanged. `capture_mcp_result(manager, span, result, operation)` +is the same capture for an existing span. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/mcp/traceai_mcp/__init__.py b/python/frameworks/mcp/traceai_mcp/__init__.py index 1a8e685e3..d2b3bf458 100644 --- a/python/frameworks/mcp/traceai_mcp/__init__.py +++ b/python/frameworks/mcp/traceai_mcp/__init__.py @@ -8,12 +8,19 @@ from opentelemetry import context, propagate from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from traceai_mcp.package import _instruments +from traceai_mcp._attachments import capture_mcp_result, mcp_attachment_capabilities, wrap_mcp_client from wrapt import ObjectProxy, register_post_import_hook, wrap_function_wrapper +__all__ = ["MCPInstrumentor", "capture_mcp_result", "mcp_attachment_capabilities", "wrap_mcp_client"] + + class MCPInstrumentor(BaseInstrumentor): # type: ignore """ An instrumenter for MCP. + + It propagates trace context over MCP transports only and never captures + media. Use ``wrap_mcp_client(session)`` for tool and resource media. """ def instrumentation_dependencies(self) -> Collection[str]: diff --git a/python/frameworks/mcp/traceai_mcp/_attachments.py b/python/frameworks/mcp/traceai_mcp/_attachments.py new file mode 100644 index 000000000..1851e2fee --- /dev/null +++ b/python/frameworks/mcp/traceai_mcp/_attachments.py @@ -0,0 +1,199 @@ +"""MCP tool and resource media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/mcp`` (``wrapMCPClient`` / ``captureMCPResult``). Only typed +inline bytes are observed: ``ImageContent.data``, ``AudioContent.data`` and an +``EmbeddedResource`` (or ``resources/read``) ``BlobResourceContents.blob``. +Resource URIs, ``resource_link`` blocks, URLs, local paths and +``structuredContent`` are never fetched, opened or scraped. Each part is +preflighted before its base64 is read or decoded; at most 64 parts and +256 KiB of decoded bytes per result. + +This module imports ``fi_instrumentation.attachments`` lazily, so the +transport-only ``MCPInstrumentor`` keeps working on a core without it. +""" + +import functools +import inspect +from types import MappingProxyType +from typing import Any, Mapping, Optional + +from opentelemetry import context as otel_context +from opentelemetry import trace as trace_api +from opentelemetry.trace import SpanKind, Status, StatusCode +from wrapt import ObjectProxy + +from traceai_mcp.version import __version__ + +__all__ = ("capture_mcp_result", "wrap_mcp_client", "mcp_attachment_capabilities") + +#: Declarative handoff for auto and reconciliation. +mcp_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tools/call.output", "resources/read.output"), + "sources": ("inline_base64",), + "content_types": ("image", "audio", "resource.blob"), + "unsupported": ("resource_link", "resource_uri", "remote_url", "local_path", "structured_content", + "transport_only_instrumentor"), + "max_parts": 64, + "max_decoded_bytes_per_result": 256 * 1024, +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_OPERATIONS = ("tools/call", "resources/read") + + +def _get(value: Any, *names: str) -> Any: + """A field of an MCP model (1.x camelCase or 2.x snake_case) or a plain dict; never raises.""" + for name in names: + try: + found = value.get(name) if isinstance(value, Mapping) else getattr(value, name, None) + except Exception: + found = None + if found is not None: + return found + return None + + +def _active(manager) -> bool: + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return False + return active(manager) + + +def capture_mcp_result(manager, span, result: Any, operation: str = "tools/call", capture=None, + trace_config=None) -> Mapping[str, Any]: + """Capture typed inline media of one MCP ``tools/call`` or ``resources/read`` result. + + ``span`` is the span that represents this call. Tool media is tool output + and potential model input, so input and output hide flags both apply. + Returns the admitted positions (``"/"`` -> occurrence); + empty when no manager, disabled, unsampled or denied. Never raises and + never changes ``result``. + """ + if operation not in _OPERATIONS or not _active(manager): + return _EMPTY + try: + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return _EMPTY + try: + media = ToolMedia(manager, span, "mcp", trace_config=trace_config, capture=capture) + if not media.usable: + return _EMPTY + error = _get(result, "isError", "is_error") is True + parts = _get(result, "content" if operation == "tools/call" else "contents") + if not isinstance(parts, (list, tuple)): + return _EMPTY + for index, part in enumerate(parts[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(part, "type") + if operation == "resources/read": + holder, field, expect = part, "blob", None + elif kind in ("image", "audio"): + holder, field, expect = part, "data", kind + elif kind == "resource": + holder, field, expect = _get(part, "resource"), "blob", None + else: + continue # text, resource_link and unknown blocks are never dereferenced + if holder is None: + continue + mime = _get(holder, "mimeType", "mime_type") + media.base64("%s/%d" % (operation, index), lambda h=holder, f=field: _get(h, f), mime, + purpose="tool_result", expect=expect, role="tool", part_index=index, error=error) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY # Malformed or untrusted results must not change the tool response. + + +def _provider_manager(tracer_provider) -> Any: + owner = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + manager = getattr(owner, "attachment_manager", None) + if manager is None: + delegate = getattr(owner, "_real_tracer_provider", None) or getattr(owner, "_tracer_provider", None) + manager = getattr(delegate, "attachment_manager", None) + return manager + + +class _TracedSession(ObjectProxy): # type: ignore[misc] + """Proxies an MCP ``ClientSession``; only ``call_tool`` and ``read_resource`` gain a span.""" + + def __init__(self, wrapped, config): + super().__init__(wrapped) + self._self_config = config + + def _self_traced(self, operation: str, method, args, kwargs): + tracer_provider, attachments, trace_config, capture = self._self_config + + @functools.wraps(method) + async def run(): + span = None + manager = None + try: + manager = attachments if attachments is not None else _provider_manager(tracer_provider) + provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + tracer = provider.get_tracer("traceai_mcp", __version__) + span = tracer.start_span("mcp.%s" % operation, kind=SpanKind.CLIENT, + attributes={"gen_ai.span.kind": "TOOL", "mcp.method.name": operation}) + except Exception: + span = None # Tracing never prevents the SDK call. + if span is None: + return await method(*args, **kwargs) + token = otel_context.attach(trace_api.set_span_in_context(span)) + try: + result = await method(*args, **kwargs) + except BaseException: + otel_context.detach(token) + try: + span.set_status(Status(StatusCode.ERROR)) + finally: + span.end() + raise + otel_context.detach(token) + try: + if manager is not None: + capture_mcp_result(manager, span, result, operation, capture, trace_config) + failed = _get(result, "isError", "is_error") is True + span.set_status(Status(StatusCode.ERROR if failed else StatusCode.OK)) + except Exception: + pass # Never log SDK results or exceptions. + finally: + span.end() + return result + + return run() + + def call_tool(self, *args, **kwargs): + method = self.__wrapped__.call_tool + if not inspect.iscoroutinefunction(method): + return method(*args, **kwargs) + return self._self_traced("tools/call", method, args, kwargs) + + def read_resource(self, *args, **kwargs): + method = self.__wrapped__.read_resource + if not inspect.iscoroutinefunction(method): + return method(*args, **kwargs) + return self._self_traced("resources/read", method, args, kwargs) + + +def wrap_mcp_client(session, *, tracer_provider=None, attachments=None, trace_config=None, + attachment_capture=None): + """Trace a ``mcp.ClientSession``'s ``call_tool`` and ``read_resource`` and capture their media. + + ``attachments`` defaults to the provider's shared manager + (``tracer_provider.attachment_manager``, as ``fi_instrumentation.register`` + and ``traceai.init`` set it); no uploader is created here. ``trace_config`` + and ``attachment_capture`` can only add restrictions. The SDK result, + exception and cancellation are returned unchanged. Wrapping a traced session + again returns it unchanged, so spans never nest and media is never uploaded + twice. + """ + # A traced session is returned as is, so spans never nest and media is never captured twice. + # (ObjectProxy delegates ==/hash to the session, so identity caches would see them as equal.) + if isinstance(session, _TracedSession): + return session + proxy = _TracedSession(session, (tracer_provider, attachments, trace_config, attachment_capture)) + return proxy diff --git a/python/frameworks/openai-agents/README.md b/python/frameworks/openai-agents/README.md index bba264856..2712bfe36 100644 --- a/python/frameworks/openai-agents/README.md +++ b/python/frameworks/openai-agents/README.md @@ -59,3 +59,33 @@ print(result.final_output) ``` +## Tool media and computer-use screenshots (attachments) + +Pass the shared manager: `OpenAIAgentsInstrumentor().instrument(tracer_provider=..., attachments=manager)` +(`traceai.init` passes it for you). On the tool's function span, before +`output.value` is written: + +- the computer tool's screenshot (`computer_screenshot`), which the SDK records + on the `computer` function span from openai-agents 0.10.0; +- `ToolOutputImage` / `{"type": "image", "image_url": "data:..."}` and + `ToolOutputFileContent` with `file_data` (openai-agents 0.4.0+; MCP image + results become these from 0.8.0), as `tool_result`. + +Admitted values become `fi-attachment:` in `output.value`. +Nothing is recorded or uploaded when `RunConfig.trace_include_sensitive_data` +is off. Model-call input media is left to the OpenAI instrumentor's own span: +with both enabled, a screenshot is uploaded once as the tool output and once +as the next model input. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/openai-agents/traceai_openai_agents/__init__.py b/python/frameworks/openai-agents/traceai_openai_agents/__init__.py index fa114bfe7..b5f5e0edc 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/__init__.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/__init__.py @@ -12,6 +12,23 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class OpenAIAgentsInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for openai-agents @@ -36,15 +53,22 @@ def _instrument(self, **kwargs: Any) -> None: from traceai_openai_agents._processor import FiTracingProcessor - + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + processor = FiTracingProcessor( + cast(Tracer, tracer), + attachments=_attachment_manager(kwargs), + trace_config=config, + attachment_capture=kwargs.get("attachment_capture"), + ) if exclusive_processor: from agents import set_trace_processors - set_trace_processors([FiTracingProcessor(cast(Tracer, tracer))]) + set_trace_processors([processor]) else: from agents import add_trace_processor - add_trace_processor(FiTracingProcessor(cast(Tracer, tracer))) + add_trace_processor(processor) def _uninstrument(self, **kwargs: Any) -> None: # TODO : OpenAI Agents does not support uninstrumentation currently diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py b/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py new file mode 100644 index 000000000..6b318edb6 --- /dev/null +++ b/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py @@ -0,0 +1,110 @@ +"""Tool media of OpenAI Agents SDK function spans, through the shared attachment manager. + +The SDK records what a tool returned on its ``FunctionSpanData.output`` (only +when ``RunConfig.trace_include_sensitive_data`` is on). These typed outputs +are observed, at the tool span, before ``OUTPUT_VALUE`` is written: + +- the computer tool's screenshot: a ``data:image/png;base64,`` string on the + ``computer`` function span (SDK 0.10.0 and later record it there); +- ``ToolOutputImage`` / ``{"type": "image", "image_url": }`` + (SDK 0.4.0+; MCP image results become these from 0.8.0); +- ``ToolOutputFileContent`` / ``{"type": "file", "file_data": }``; +- a list of those. + +Remote URLs, ``file_id`` values and plain strings on other tools are never +fetched or scraped. Model-call input media (``ResponseSpanData.input``) is +left to the OpenAI provider adapter, which owns that span. Admitted values +become ``fi-attachment:`` in the span's ``OUTPUT_VALUE``; the +SDK's own objects are never changed. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Tuple + +__all__ = ("capture_function_output", "openai_agents_attachment_capabilities") + +COMPUTER_TOOL_NAMES = ("computer", "computer_use_preview") +_IMAGE_MIMES = ("image/png", "image/jpeg", "image/webp") +_FILE_MIMES = ("application/pdf", "application/octet-stream") +_EMPTY: Mapping[str, Any] = MappingProxyType({}) + +openai_agents_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("function.output", "computer.screenshot"), + "sources": ("inline_data_uri",), + "minimum_sdk": {"computer.screenshot": "0.10.0", "tool_output_image": "0.4.0", "mcp_image": "0.8.0"}, + "unsupported": ("remote_url", "file_id", "response_input", "trace_include_sensitive_data_off"), +}) + + +def _field(value: Any, key: str) -> Any: + try: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + except Exception: + return None + + +def _typed(value: Any) -> bool: + return isinstance(value, Mapping) or type(value).__name__ in ("ToolOutputImage", "ToolOutputFileContent") + + +def capture_function_output(manager, span, name: Any, output: Any, *, trace_config=None, + capture=None) -> Tuple[Mapping[str, Any], Any]: + """Capture one function span's typed media; returns (admitted positions, telemetry output). + + The telemetry output is ``output`` itself when nothing was admitted, else a + copy where only the admitted values are ``fi-attachment:`` URIs. Never raises. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia, uri + except ImportError: + return _EMPTY, output + try: + if not active(manager) or output is None: + return _EMPTY, output + media = ToolMedia(manager, span, "openai_agents", trace_config=trace_config, capture=capture) + if not media.usable: + return _EMPTY, output + + def item(value, index): + position = "function/%d" % index + if isinstance(value, str): + if name in COMPUTER_TOOL_NAMES and value.startswith("data:image/"): + found = media.data_uri(position, lambda: value, _IMAGE_MIMES, purpose="computer_screenshot", + role="tool", part_index=index) + return uri(found) if found else value + return value + if not _typed(value): + return value + kind = _field(value, "type") + if kind == "image": + key, mimes = "image_url", _IMAGE_MIMES + elif kind == "file": + key, mimes = "file_data", _FILE_MIMES + else: + return value + encoded = _field(value, key) + if not isinstance(encoded, str) or not encoded.startswith("data:"): + return value # URLs and file ids are never fetched + found = media.data_uri(position, lambda: encoded, mimes, purpose="tool_result", role="tool", + part_index=index) + if not found: + return value + plain = dict(value) if isinstance(value, Mapping) else value.model_dump(exclude_none=True) + plain[key] = uri(found) + return plain + + if isinstance(output, list): + projected = [item(value, index) if index < MAX_TOOL_PARTS else value + for index, value in enumerate(output)] + else: + projected = item(output, 0) + captured = media.captured + if not captured: + return _EMPTY, output + return MappingProxyType(captured), projected + except Exception: + return _EMPTY, output diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py index 48684260b..ea23b690c 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py @@ -67,8 +67,19 @@ class FiTracingProcessor(TracingProcessor): _MAX_HANDOFFS_IN_FLIGHT = 1000 - def __init__(self, tracer: Tracer) -> None: + def __init__( + self, + tracer: Tracer, + *, + attachments: Any = None, + trace_config: Any = None, + attachment_capture: Any = None, + ) -> None: self._tracer = tracer + # The shared attachment manager (None unless an enabled one was passed). + self._attachments = attachments + self._trace_config = trace_config + self._attachment_capture = attachment_capture self._root_spans: dict[str, OtelSpan] = {} self._otel_spans: dict[str, OtelSpan] = {} self._tokens: dict[str, object] = {} @@ -227,7 +238,19 @@ def _set_span_attributes(self, span: Span[Any], otel_span: OtelSpan) -> None: for k, v in _get_attributes_from_generation_span_data(data): otel_span.set_attribute(k, v) elif isinstance(data, FunctionSpanData): - for k, v in _get_attributes_from_function_span_data(data): + output = data.output + if self._attachments is not None and output is not None: + from traceai_openai_agents._attachments import capture_function_output + + _, output = capture_function_output( + self._attachments, + otel_span, + data.name, + output, + trace_config=self._trace_config, + capture=self._attachment_capture, + ) + for k, v in _get_attributes_from_function_span_data(data, output): otel_span.set_attribute(k, v) elif isinstance(data, MCPListToolsSpanData): for k, v in _get_attributes_from_mcp_list_tool_span_data(data): @@ -560,20 +583,27 @@ def _convert_to_primitive(value: Any) -> Union[bool, str, bytes, int, float]: return str(value) +_UNSET = object() + + def _get_attributes_from_function_span_data( obj: FunctionSpanData, + output: Any = _UNSET, ) -> Iterator[tuple[str, AttributeValue]]: + # ``output`` is a telemetry copy in which admitted media are fi-attachment URIs. + if output is _UNSET: + output = obj.output yield GEN_AI_TOOL_NAME, obj.name if obj.input: yield INPUT_VALUE, obj.input yield INPUT_MIME_TYPE, JSON - if obj.output is not None: - yield OUTPUT_VALUE, _convert_to_primitive(obj.output) + if output is not None: + yield OUTPUT_VALUE, _convert_to_primitive(output) if ( - isinstance(obj.output, str) - and len(obj.output) > 1 - and obj.output[0] == "{" - and obj.output[-1] == "}" + isinstance(output, str) + and len(output) > 1 + and output[0] == "{" + and output[-1] == "}" ): yield OUTPUT_MIME_TYPE, JSON diff --git a/python/frameworks/smolagents/README.md b/python/frameworks/smolagents/README.md index f942a29d0..4a59348c1 100644 --- a/python/frameworks/smolagents/README.md +++ b/python/frameworks/smolagents/README.md @@ -80,3 +80,33 @@ manager_agent.run( ) ``` +## Image observations (attachments) + +Pass the shared manager: `SmolagentsInstrumentor().instrument(tracer_provider=..., attachments=manager)`. + +- Images smolagents hands to a model (`{"type": "image", "image": }` + blocks: task images and `ActionStep.observations_images`, such as a browser + screenshot a step callback stored) are uploaded on the model span as + `model_input`, once per image object (memory replays are not re-uploaded). + Every model call that carries a captured image, new or replayed, tells + provider instrumentors below it that the call's media is already owned, so + a replayed image is never uploaded again from the provider request. +- A tool's returned in-memory PIL image or `AgentImage` is uploaded on the + tool span as `tool_result`. + +Images are PNG-encoded only after the manager admitted them, up to +2048 x 2048 pixels. File-backed images that are not loaded yet, image URLs +and every `AgentAudio` are never read. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/smolagents/traceai_smolagents/__init__.py b/python/frameworks/smolagents/traceai_smolagents/__init__.py index e6aac0cdd..642e55863 100644 --- a/python/frameworks/smolagents/traceai_smolagents/__init__.py +++ b/python/frameworks/smolagents/traceai_smolagents/__init__.py @@ -4,6 +4,7 @@ from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from traceai_smolagents._wrappers import ( + _Media, _ModelWrapper, _RunWrapper, _StepWrapper, @@ -16,6 +17,23 @@ _instruments = ("smolagents >= 1.2.2",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class SmolagentsInstrumentor(BaseInstrumentor): __slots__ = ( "_original_run_method", @@ -43,6 +61,9 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + media = _Media(_attachment_manager(kwargs), config, kwargs.get("attachment_capture")) run_wrapper = _RunWrapper(tracer=self._tracer) self._original_run_method = getattr(MultiStepAgent, "run", None) @@ -72,9 +93,10 @@ def _instrument(self, **kwargs: Any) -> None: if method not in vars(model_subclass): continue self._original_model_call_methods[(model_subclass, method)] = vars(model_subclass)[method] - wrap_function_wrapper(model_subclass, method, _ModelWrapper(tracer=self._tracer, method_name=method)) + wrap_function_wrapper(model_subclass, method, + _ModelWrapper(tracer=self._tracer, method_name=method, media=media)) - tool_call_wrapper = _ToolCallWrapper(tracer=self._tracer) + tool_call_wrapper = _ToolCallWrapper(tracer=self._tracer, media=media) self._original_tool_call_method = getattr(Tool, "__call__", None) wrap_function_wrapper( module="smolagents", diff --git a/python/frameworks/smolagents/traceai_smolagents/_attachments.py b/python/frameworks/smolagents/traceai_smolagents/_attachments.py new file mode 100644 index 000000000..5058e85b3 --- /dev/null +++ b/python/frameworks/smolagents/traceai_smolagents/_attachments.py @@ -0,0 +1,164 @@ +"""smolagents typed media observations, through the shared attachment manager. + +Observed, and nothing else: + +- images smolagents hands to a model: ``{"type": "image", "image": }`` + content blocks (task images and ``ActionStep.observations_images``, such as + a browser screenshot a step callback stored), on the model span as model + input, before the model converts them; +- a tool's returned image: an in-memory ``PIL.Image.Image`` or an + ``AgentImage`` whose image is already in memory, on the tool span. + +Images are PNG-encoded only after a pixel-bounded preflight admitted them. +A file-backed ``AgentImage`` or PIL image that is not loaded yet, image URLs, +``image_url`` strings and every ``AgentAudio`` (a path or a tensor) are not +read (``source_unavailable``). smolagents replays memory images on every +model call; the same image object is uploaded once per manager, and every +model call that carries a captured image (new or replayed) is marked as the +owner of its media so a provider adapter below never uploads it again. +""" + +import os +import threading +import weakref +from collections import OrderedDict +from types import MappingProxyType +from typing import Any, Mapping + +__all__ = ("capture_model_images", "capture_tool_image", "owner_context", "smolagents_attachment_capabilities") + +smolagents_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.input.image", "tool.output.image"), + "sources": ("in_memory_pil_image",), + "unsupported": ("agent_audio", "file_backed_image", "image_url", "tensor"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_MAX_SEEN = 1024 +_MAX_MESSAGES = 256 + + +class _Seen: + """Image objects already uploaded, per manager (identity only; PIL images are unhashable).""" + + def __init__(self) -> None: + self._reset() + + def _reset(self) -> None: + self._pid = os.getpid() + self._lock = threading.Lock() + self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() + + def _owned_lock(self) -> threading.Lock: + # A forked child gets a fresh lock and table: a lock another thread held at fork + # time would never be released there. No fork hook is registered. + if self._pid != os.getpid(): + self._reset() + return self._lock + + def __contains__(self, item) -> bool: + manager, image = item + with self._owned_lock(): + entry = self._managers.get(manager, {}).get(id(image)) + return entry is not None and entry() is image + + def add(self, manager, image) -> None: + with self._owned_lock(): + seen = self._managers.get(manager) + if seen is None: + seen = self._managers[manager] = OrderedDict() + seen[id(image)] = weakref.ref(image) + while len(seen) > _MAX_SEEN: + seen.popitem(last=False) + + +_SEEN = _Seen() + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def _in_memory(value: Any) -> Any: + """The PIL image an ``AgentImage`` already holds in memory, else the value itself.""" + if type(value).__name__ == "AgentImage": + raw = getattr(value, "_raw", None) + return raw # None for path- or tensor-backed values: never loaded here + return value + + +def capture_model_images(manager, span, messages: Any, *, trace_config=None, capture=None) -> int: + """Capture the in-memory images of one model call's input messages. + + Returns the number of images this call owns: admitted now, or already uploaded + (a memory replay). A positive count means the caller marks the model call as owner. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return 0 + admitted = 0 + try: + if not active(manager) or not isinstance(messages, (list, tuple)): + return 0 + media = ToolMedia(manager, span, "smolagents", trace_config=trace_config, capture=capture, + budget=8 * 1024 * 1024, direction="input") + for message_index, message in enumerate(messages[:_MAX_MESSAGES]): + content = _get(message, "content") + if not isinstance(content, list): + continue + for part_index, part in enumerate(content[:MAX_TOOL_PARTS]): + if not media.usable: + return admitted + if _get(part, "type") != "image": + continue + image = _in_memory(_get(part, "image")) + if image is None: + continue + if (manager, image) in _SEEN: + admitted += 1 # Replayed: already uploaded, still owned here. + continue + found = media.image("model/%d/%d" % (message_index, part_index), image, purpose="model_input", + role="user", message_index=message_index, part_index=part_index) + if found is not None: + _SEEN.add(manager, image) + admitted += 1 + except Exception: + return admitted + return admitted + + +def owner_context(): + """The current context marked as owner of this call's media (by the shared key name).""" + from fi_instrumentation.attachments.tool_media import with_ownership + + return with_ownership() + + +def capture_tool_image(manager, span, response: Any, *, trace_config=None, capture=None) -> int: + """Capture a tool's returned in-memory image on its span. Returns 1 when admitted.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import ToolMedia + except ImportError: + return 0 + try: + if not active(manager): + return 0 + image = _in_memory(response) + if image is None or (manager, image) in _SEEN: + return 0 + media = ToolMedia(manager, span, "smolagents", trace_config=trace_config, capture=capture, + budget=8 * 1024 * 1024) + found = media.image("tool/0", image, purpose="tool_result", role="tool", part_index=0) + if found is None: + return 0 + _SEEN.add(manager, image) + return 1 + except Exception: + return 0 diff --git a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py index 0c3cd0cb1..5897279b3 100644 --- a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py +++ b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py @@ -418,10 +418,20 @@ def _input_value_and_mime_type( yield INPUT_VALUE, safe_json_dumps(arguments) +class _Media: + """The shared attachment manager and its local restrictions (None manager: capture off).""" + + __slots__ = ("manager", "trace_config", "capture") + + def __init__(self, manager: Any = None, trace_config: Any = None, capture: Any = None) -> None: + self.manager, self.trace_config, self.capture = manager, trace_config, capture + + class _ModelWrapper: - def __init__(self, tracer: trace_api.Tracer, method_name: str = "__call__") -> None: + def __init__(self, tracer: trace_api.Tracer, method_name: str = "__call__", media: Any = None) -> None: self._tracer = tracer self._method_name = method_name + self._media = media if media is not None else _Media() def __call__( self, @@ -446,7 +456,20 @@ def __call__( **dict(_get_raw_input(args, **kwargs)), }, ) as span: - output_message = wrapped(*args, **kwargs) + owner_token = None + if self._media.manager is not None: + from traceai_smolagents._attachments import capture_model_images, owner_context + + if capture_model_images(self._media.manager, span, arguments.get("messages"), + trace_config=self._media.trace_config, capture=self._media.capture): + # These images are captured here (now or on an earlier call); a provider + # adapter below must not upload them again. + owner_token = context_api.attach(owner_context()) + try: + output_message = wrapped(*args, **kwargs) + finally: + if owner_token is not None: + context_api.detach(owner_token) span.set_status(trace_api.StatusCode.OK) span.set_attribute(OUTPUT_VALUE, _get_raw_output(output_message)) # Modern SDKs return usage on ChatMessage; historical releases @@ -473,8 +496,9 @@ def __call__( class _ToolCallWrapper: - def __init__(self, tracer: trace_api.Tracer) -> None: + def __init__(self, tracer: trace_api.Tracer, media: Any = None) -> None: self._tracer = tracer + self._media = media if media is not None else _Media() def __call__( self, @@ -501,6 +525,11 @@ def __call__( }, ) as span: response = wrapped(*args, **kwargs) + if self._media.manager is not None: + from traceai_smolagents._attachments import capture_tool_image + + capture_tool_image(self._media.manager, span, response, + trace_config=self._media.trace_config, capture=self._media.capture) span.set_attribute(OUTPUT_VALUE, _get_raw_output(response)) span.set_status(trace_api.StatusCode.OK) span.set_attributes( diff --git a/python/tests/attachments_mcp_agents/_fake_claude_cli.py b/python/tests/attachments_mcp_agents/_fake_claude_cli.py new file mode 100644 index 000000000..83d83f4b0 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_fake_claude_cli.py @@ -0,0 +1,110 @@ +"""A stand-in for the Claude Code CLI that speaks the SDK's stream-json control protocol. + +The real ``claude_agent_sdk`` spawns this through ``ClaudeAgentOptions(cli_path=...)``. +For every user message it replays one tool call: the assistant ``tool_use``, the +PreToolUse and PostToolUse hook callbacks the SDK registered, the ``tool_result`` +carrying the media in ``FAKE_CLAUDE_RESULT`` (a JSON file), then a final answer. +``FAKE_CLAUDE_ORDER=result-first`` streams the tool result before PostToolUse; +``no-result`` never streams it. ``FAKE_CLAUDE_GAP`` (seconds) waits between +PostToolUse and the tool result; ``FAKE_CLAUDE_HOLD`` (a path) also waits there, +until that file exists (at most 20 s). ``FAKE_TOOL_ID`` names the tool call. No +model, network or file other than the result and hold files is touched. +""" + +import json +import os +import sys +import time + +if "-v" in sys.argv[1:] or "--version" in sys.argv[1:]: + print("2.1.999 (Claude Code)") + sys.exit(0) + +with open(os.environ["FAKE_CLAUDE_RESULT"]) as handle: + TOOL_RESULT = json.load(handle) +ORDER = os.environ.get("FAKE_CLAUDE_ORDER", "hooks-first") +GAP = float(os.environ.get("FAKE_CLAUDE_GAP", "0")) +HOLD = os.environ.get("FAKE_CLAUDE_HOLD", "") +TOOL_ID = os.environ.get("FAKE_TOOL_ID", "toolu_screen_1") +TOOL = "mcp__screen__capture" +hooks = {} +counter = [0] + + +def send(message): + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +def wait_response(request_id): + for line in sys.stdin: + message = json.loads(line) + if message.get("type") == "control_response": + if message.get("response", {}).get("request_id") == request_id: + return message + else: + handle(message) + + +def hook(event, tool_use_id, extra): + for callback_id in hooks.get(event, []): + counter[0] += 1 + request_id = "cli_%d" % counter[0] + send({"type": "control_request", "request_id": request_id, "request": { + "subtype": "hook_callback", "callback_id": callback_id, "tool_use_id": tool_use_id, + "input": dict({"hook_event_name": event, "session_id": "s1", "tool_name": TOOL, "tool_input": {}, + "cwd": "/", "transcript_path": ""}, **extra)}}) + wait_response(request_id) + + +def assistant(content): + send({"type": "assistant", "parent_tool_use_id": None, "session_id": "s1", "message": { + "id": "msg_%d" % counter[0], "type": "message", "role": "assistant", "model": "claude-test", + "content": content, "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 1}}}) + + +def turn(): + tool_use_id = TOOL_ID + time.sleep(0.05) # Model latency: the client is iterating receive_response() before any hook. + assistant([{"type": "tool_use", "id": tool_use_id, "name": TOOL, "input": {}}]) + hook("PreToolUse", tool_use_id, {}) + result = {"type": "user", "parent_tool_use_id": None, "session_id": "s1", "message": { + "role": "user", "content": [dict(TOOL_RESULT, type="tool_result", tool_use_id=tool_use_id)]}} + if ORDER == "result-first": + send(result) + hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) + else: + hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) + time.sleep(GAP) + deadline = time.monotonic() + 20 + while HOLD and not os.path.exists(HOLD) and time.monotonic() < deadline: + time.sleep(0.02) + if ORDER != "no-result": + send(result) + assistant([{"type": "text", "text": "done"}]) + send({"type": "result", "subtype": "success", "duration_ms": 1, "duration_api_ms": 1, "is_error": False, + "num_turns": 1, "session_id": "s1", "total_cost_usd": 0.0, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}) + + +def handle(message): + kind = message.get("type") + if kind == "control_request": + request = message.get("request", {}) + if request.get("subtype") == "initialize": + for event, matchers in (request.get("hooks") or {}).items(): + hooks[event] = [cid for matcher in matchers for cid in matcher.get("hookCallbackIds", [])] + send({"type": "control_response", "response": { + "subtype": "success", "request_id": message.get("request_id"), + "response": {"commands": [], "models": [], "account": {}} if request.get("subtype") == "initialize" + else {}}}) + elif kind == "user": + turn() + + +send({"type": "system", "subtype": "init", "session_id": "s1", "tools": [], "mcp_servers": [], "model": "claude-test", + "cwd": "/", "permissionMode": "default", "apiKeySource": "none"}) +for line in sys.stdin: + line = line.strip() + if line: + handle(json.loads(line)) diff --git a/python/tests/attachments_mcp_agents/_harness.py b/python/tests/attachments_mcp_agents/_harness.py new file mode 100644 index 000000000..b7e2d2a53 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_harness.py @@ -0,0 +1,44 @@ +"""Shared helpers: the provider-attachments harness, loaded by path (one F01 loopback for every lane).""" + +import importlib.util +import json +import sys +from pathlib import Path + +_NAME = "_attachments_providers_harness" +if _NAME in sys.modules: + harness = sys.modules[_NAME] +else: + _path = Path(__file__).resolve().parent.parent / "attachments_providers" / "conftest.py" + _spec = importlib.util.spec_from_file_location(_NAME, _path) + harness = importlib.util.module_from_spec(_spec) + sys.modules[_NAME] = harness + _spec.loader.exec_module(harness) + +png, wav, b64, refs, wait_uploads, flush, PDF = (harness.png, harness.wav, harness.b64, harness.refs, + harness.wait_uploads, harness.flush, harness.PDF) + + +def data_uri(mime, data): + return "data:%s;base64,%s" % (mime, b64(data)) + + +def uploaded(f01): + """(mime_type, direction, purpose, role, tool_call_id) of each initialize body, sorted.""" + return sorted((u.get("mime_type"), u.get("direction"), u.get("purpose"), u.get("role"), + (u.get("correlation") or {}).get("tool_call_id") or u.get("tool_call_id")) + for u in f01.uploads) + + +def span_named(exporter, name): + found = [s for s in exporter.get_finished_spans() if s.name == name] + assert found, [s.name for s in exporter.get_finished_spans()] + return found[-1] + + +def no_bytes(exporter, data): + """No exported span attribute carries the media's base64 text.""" + text = b64(data)[:40] + for span in exporter.get_finished_spans(): + for value in (span.attributes or {}).values(): + assert text not in json.dumps(value, default=str), span.name diff --git a/python/tests/attachments_mcp_agents/_mcp_media_server.py b/python/tests/attachments_mcp_agents/_mcp_media_server.py new file mode 100644 index 000000000..726dde217 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_mcp_media_server.py @@ -0,0 +1,116 @@ +"""A real MCP server over stdio whose tools return synthetic typed media. + +Used by the MCP (``wrap_mcp_client``) and mcp-use tests. It talks to nothing +else. ``reads`` reports how many times the client read each resource, so a +test can prove a ``resource_link`` URI was never fetched. +""" + +import base64 +import json +import struct +import zlib +from importlib.metadata import version + +from mcp import types + +if int(version("mcp").split(".")[0]) >= 2: + from mcp.server.mcpserver import MCPServer as Server +else: + from mcp.server.fastmcp import FastMCP as Server + + +def png(seed=0, width=2, height=2) -> bytes: + raw = b"".join(b"\x00" + bytes((seed + x + y) % 256 for x in range(width * 3)) for y in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +WAV = b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00@\x1f\x00\x00\x80>\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00" +PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n" +SECRET_FILE = "file:///etc/hostname" +READS = {"shot": 0, "secret": 0} + + +def b64(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def block(kind, **fields): + """A content block from its wire (camelCase) shape, valid on MCP 1.x and 2.x.""" + model = {"image": types.ImageContent, "audio": types.AudioContent, "text": types.TextContent, + "resource": types.EmbeddedResource, "resource_link": types.ResourceLink}[kind] + return model.model_validate({"type": kind, **fields}) + + +server = Server("media-fixture") + + +@server.tool() +def screenshot(): + """A screenshot and a caption.""" + return [block("text", text="CAPTION-MARKER"), block("image", data=b64(png(1)), mimeType="image/png")] + + +@server.tool() +def voice(): + """A short WAV clip.""" + return [block("audio", data=b64(WAV), mimeType="audio/wav")] + + +@server.tool() +def report(): + """An embedded PDF resource blob plus a resource link that must never be fetched.""" + return [ + block("resource", resource={"uri": "file:///report.pdf", "mimeType": "application/pdf", "blob": b64(PDF)}), + block("resource_link", uri=SECRET_FILE, name="hostname", mimeType="image/png"), + ] + + +@server.tool() +def relabeled(): + """An image block whose MIME claims another modality, and one outside the F01 enum.""" + return [block("image", data=b64(png(2)), mimeType="audio/wav"), + block("image", data=b64(png(3)), mimeType="image/gif")] + + +@server.tool() +def malformed(): + """Not valid base64.""" + return [block("image", data="!!!not-base64!!!", mimeType="image/png")] + + +@server.tool() +def many(count: int): + """``count`` distinct images.""" + return [block("image", data=b64(png(10 + n)), mimeType="image/png") for n in range(count)] + + +@server.tool() +def big(): + """An image larger than the 256 KiB per-result budget.""" + return [block("image", data=b64(png(4) + b"\x00" * (300 * 1024)), mimeType="image/png")] + + +@server.tool() +def reads() -> str: + return json.dumps(READS) + + +@server.resource("file:///shot.png", mime_type="image/png") +def shot() -> bytes: + READS["shot"] += 1 + return png(5) + + +@server.resource(SECRET_FILE, mime_type="image/png") +def secret() -> bytes: + READS["secret"] += 1 + return png(6) + + +if __name__ == "__main__": + server.run(transport="stdio") diff --git a/python/tests/attachments_mcp_agents/conftest.py b/python/tests/attachments_mcp_agents/conftest.py new file mode 100644 index 000000000..41d06ea86 --- /dev/null +++ b/python/tests/attachments_mcp_agents/conftest.py @@ -0,0 +1,27 @@ +"""Loopback harness for MCP and agent tool-media tests: synthetic data only. + +Reuses the provider-attachments harness (``tests/attachments_providers``): +``f01`` is a TLS loopback F01 attachment server and ``spans`` an in-memory +OpenTelemetry provider. Nothing here reaches a network beyond 127.0.0.1. +""" + +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("fi_instrumentation.attachments.tool_media") +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _harness import harness # noqa: E402 + +# Fixtures, registered by being attributes of this conftest. +clean_env = harness.clean_env +f01 = harness.f01 +spans = harness.spans + + +@pytest.fixture(autouse=True) +def mock_all_http_requests(): + """Overrides python/tests/conftest.py: these suites talk to real loopback servers through requests.""" + yield None diff --git a/python/tests/attachments_mcp_agents/test_auto_registry.py b/python/tests/attachments_mcp_agents/test_auto_registry.py new file mode 100644 index 000000000..1c0abf730 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_auto_registry.py @@ -0,0 +1,256 @@ +"""The auto module with the real MCP and agent packages: registry, report rows and capture. + +``traceai.init`` borrows a loopback F01 manager (or runs with attachments OFF) and +enables each installed package by name. Every test skips the packages that are not +installed; ``TRACEAI_EXPECT_MCP_AGENT_ADAPTERS=1`` (set by CI) turns a missing +package into a failure so a broken install cannot pass silently. +""" + +import asyncio +import importlib +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + +traceai = pytest.importorskip("traceai") + +from fi_instrumentation import TraceConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +from _harness import data_uri, flush, harness, png, refs # noqa: E402 + +provider = harness.provider # the loopback OpenAI-compatible API (fixture) + +# report row name -> (instrumentor package, SDK module, descriptor module, descriptor name) +PACKAGES = { + "mcp": ("traceai_mcp", "mcp", "traceai_mcp", "mcp_attachment_capabilities"), + "mcp_use": ("traceai_mcp_use", "mcp_use", "traceai_mcp_use._attachments", "mcp_use_attachment_capabilities"), + "openai_agents": ("traceai_openai_agents", "agents", "traceai_openai_agents._attachments", + "openai_agents_attachment_capabilities"), + "claude_agent_sdk": ("traceai_claude_agent_sdk", "claude_agent_sdk", "traceai_claude_agent_sdk._attachments", + "claude_agent_attachment_capabilities"), + "smolagents": ("traceai_smolagents", "smolagents", "traceai_smolagents._attachments", + "smolagents_attachment_capabilities"), + "firecrawl": ("traceai_firecrawl", "firecrawl", "traceai_firecrawl._attachments", + "firecrawl_attachment_capabilities"), +} +INSTRUMENTORS = {"openai_agents": "OpenAIAgentsInstrumentor", "claude_agent_sdk": "ClaudeAgentInstrumentor", + "smolagents": "SmolagentsInstrumentor", "firecrawl": "FirecrawlInstrumentor"} +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" +EXPECT_ALL = os.environ.get("TRACEAI_EXPECT_MCP_AGENT_ADAPTERS") == "1" + + +def require(name): + package, sdk = PACKAGES[name][:2] + for module in (package, sdk): + if importlib.util.find_spec(module) is None: + if EXPECT_ALL: + pytest.fail("%s is not installed" % module) + pytest.skip("%s is not installed" % module) + + +@pytest.fixture +def auto(monkeypatch): + """Fresh auto state; uninstruments whatever a test enabled.""" + for attr, value in (("_tracer_provider", None), ("_init_options", None), ("_instrumented", set()), + ("_adapter_status", {}), ("_attachment_manager_borrowed", False)): + monkeypatch.setattr(traceai, attr, value) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + started = [] + + def init(libraries=(), **options): + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", + auto_instrument=list(libraries), verbose=False, + set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + started.append((tracer_provider, list(libraries))) + return tracer_provider, exporter + + yield init + for tracer_provider, libraries in started: + for name in libraries: + module = PACKAGES[name][0] + if module in traceai._ENTRY_POINTS: # mcp-use has no instrumentor + instrumentor = getattr(importlib.import_module(module), traceai._ENTRY_POINTS[module])() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + tracer_provider.shutdown() + + +@pytest.mark.parametrize("name", sorted(PACKAGES)) +def test_report_row_is_the_installed_descriptor(name): + require(name) + _, _, path, attr = PACKAGES[name] + descriptor = getattr(importlib.import_module(path), attr) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows[name]["media"] == "provider_adapter" + assert rows[name]["media_operations"] == list(descriptor["operations"]) + assert rows[name]["package"] == PACKAGES[name][0] + + +@pytest.mark.parametrize("name", sorted(INSTRUMENTORS)) +def test_enabling_by_name_instruments_with_attachments_on_and_off(name, auto, f01): + require(name) + # OFF (no manager): the instrumentor accepts auto's OFF keyword arguments. + auto([name]) + assert traceai.attachment_status()["adapters"][name] == "registered" + instrumentor = getattr(importlib.import_module(PACKAGES[name][0]), INSTRUMENTORS[name])() + assert instrumentor.is_instrumented_by_opentelemetry + instrumentor.uninstrument() + traceai._tracer_provider = None + traceai._init_options = None + # ON (a borrowed, enabled manager): the same, with attachments=. + manager = f01.manager() + try: + auto([name], attachment_manager=manager) + assert traceai.attachment_status()["adapters"][name] == "registered" + assert traceai.get_attachment_manager() is manager + finally: + manager.shutdown(5) + + +def _mcp_session(calls): + from mcp import ClientSession, StdioServerParameters + from mcp.client.stdio import stdio_client + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + return await calls(traceai.wrap_mcp_client(session)) + + return asyncio.run(go()) + + +def test_wrap_mcp_client_captures_on_the_shared_manager(auto, f01): + require("mcp") + sys.path.insert(0, str(SERVER.parent)) + import _mcp_media_server as server + + manager = f01.manager() + try: + _, exporter = auto(["mcp"], attachment_manager=manager) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + _mcp_session(calls) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] + assert f01.payloads == [server.png(1)] + assert sum(len(refs(s)) for s in exporter.get_finished_spans() if s.name == "mcp.tools/call") == 1 + finally: + manager.shutdown(5) + + +def test_wrap_mcp_client_with_attachments_off_uploads_nothing(auto, f01): + require("mcp") + _, exporter = auto(["mcp"]) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + result = _mcp_session(calls) + assert result.content # the SDK result is unchanged + assert f01.uploads == [] and f01.payloads == [] + assert all(not refs(s) for s in exporter.get_finished_spans()) + + +def _mcp_use_run(callback, tool): + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "mcp-use" / "tests")) + import _mcp_use_support as support + from mcp_use import MCPAgent, MCPClient + + async def go(): + client = MCPClient.from_dict({"mcpServers": {"media": {"command": sys.executable, "args": [str(SERVER)]}}}) + try: + agent = MCPAgent(llm=support.ChatFake(script=[support.tool_call(tool, {}, "call_media_1"), + support.answer()]), + client=client, callbacks=[callback]) + return await agent.run(support.PROMPT) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +def test_mcp_use_callback_captures_on_the_shared_manager(auto, f01): + require("mcp_use") + manager = f01.manager() + try: + auto(["mcp_use"], attachment_manager=manager) + _mcp_use_run(traceai.mcp_use_callback(), "screenshot") + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] + finally: + manager.shutdown(5) + + +def test_mcp_use_callback_carries_auto_hide_flags(auto, f01): + require("mcp_use") + manager = f01.manager() + try: + auto(["mcp_use"], attachment_manager=manager, trace_config=TraceConfig(hide_outputs=True)) + _mcp_use_run(traceai.mcp_use_callback(), "screenshot") + flush(manager) + assert f01.uploads == [] + finally: + manager.shutdown(5) + + +def test_smolagents_enabled_by_name_captures_a_model_input_image(auto, f01, provider): + require("smolagents") + pil = pytest.importorskip("PIL.Image") + pytest.importorskip("openai") + import smolagents + + manager = f01.manager() + try: + auto(["smolagents"], attachment_manager=manager) + image = pil.new("RGB", (8, 6), (7, 10, 20)) + content = [{"type": "text", "text": "look"}, {"type": "image", "image": image}] + model = smolagents.OpenAIServerModel(model_id="m", api_base=provider.url + "/v1", api_key="SYNTHETIC_KEY") + if hasattr(model, "generate"): + from smolagents.models import ChatMessage, MessageRole + + model.generate([ChatMessage(role=MessageRole.USER, content=content)]) + else: + model([{"role": "user", "content": content}]) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "input", "model_input")] + finally: + manager.shutdown(5) + + +def test_firecrawl_enabled_by_name_captures_an_inline_screenshot(auto, f01): + require("firecrawl") + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "firecrawl" / "tests")) + from _firecrawl_fake import FakeFirecrawl + from firecrawl import Firecrawl + + shot = png(seed=71) + fake = FakeFirecrawl() + manager = f01.manager() + try: + _, exporter = auto(["firecrawl"], attachment_manager=manager) + fake.routes[("POST", "/v2/scrape")] = (200, {"success": True, "data": { + "markdown": "body", "screenshot": data_uri("image/png", shot), + "metadata": {"sourceURL": "https://example.com"}}}) + document = Firecrawl(api_key="fc-SYNTHETIC-PLACEHOLDER", api_url=fake.origin, max_retries=1).scrape( + "https://example.com", formats=["screenshot"]) + flush(manager) + assert document.screenshot == data_uri("image/png", shot) + assert f01.payloads == [shot] + finally: + manager.shutdown(5) + fake.close() diff --git a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py new file mode 100644 index 000000000..b336f8428 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py @@ -0,0 +1,280 @@ +"""Real claude_agent_sdk ClaudeSDKClient over its subprocess transport -> loopback F01. + +The CLI is ``_fake_claude_cli.py`` (the SDK's real stream-json control protocol: +initialize, hook callbacks, user/assistant/result messages); no model or key. +""" + +import asyncio +import json +import os +import stat +import sys +import time +from pathlib import Path + +import pytest + +claude_agent_sdk = pytest.importorskip("claude_agent_sdk") +pytest.importorskip("traceai_claude_agent_sdk") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_claude_agent_sdk import ClaudeAgentInstrumentor # noqa: E402 +from traceai_claude_agent_sdk._attributes import ClaudeAgentAttributes # noqa: E402 + +from _harness import PDF, b64, flush, png, refs # noqa: E402 + +CLI = Path(__file__).resolve().parent / "_fake_claude_cli.py" +SHOT = png(seed=51) + + +def image(data, mime="image/png"): + return {"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64(data)}} + + +@pytest.fixture +def cli(tmp_path, monkeypatch): + def make(content, order="hooks-first", is_error=False, gap=0.0): + result = tmp_path / "result.json" + result.write_text(json.dumps({"content": content, "is_error": is_error})) + launcher = tmp_path / "claude" + launcher.write_text("#!%s\nimport runpy\nrunpy.run_path(%r, run_name='__main__')\n" % (sys.executable, str(CLI))) + launcher.chmod(launcher.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("FAKE_CLAUDE_RESULT", str(result)) + monkeypatch.setenv("FAKE_CLAUDE_ORDER", order) + monkeypatch.setenv("FAKE_CLAUDE_GAP", str(gap)) + monkeypatch.setenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "1") + # Releases without ClaudeAgentOptions.cli_path find the CLI as "claude" on PATH. + monkeypatch.setenv("PATH", str(tmp_path) + os.pathsep + os.environ.get("PATH", "")) + return str(launcher) + + return make + + +@pytest.fixture +def instrument(spans): + provider, exporter = spans + state = {} + + def install(manager, config=None): + instrumentor = ClaudeAgentInstrumentor() + instrumentor.instrument(tracer_provider=provider, attachments=manager, config=config or TraceConfig(), + skip_dep_check=True) + state["instrumentor"] = instrumentor + return exporter + + yield install + if "instrumentor" in state: + state["instrumentor"].uninstrument() + + +def client_options(cli_path, **env): + fields = getattr(claude_agent_sdk.ClaudeAgentOptions, "__dataclass_fields__", {}) + kwargs = {"cli_path": cli_path} if "cli_path" in fields else {} + if env: + if "env" not in fields: + pytest.skip("this claude-agent-sdk release has no ClaudeAgentOptions.env") + kwargs["env"] = env + return claude_agent_sdk.ClaudeAgentOptions(**kwargs) + + +def converse(cli_path): + async def go(): + options = client_options(cli_path) + messages = [] + async with claude_agent_sdk.ClaudeSDKClient(options=options) as client: + await client.query("take a screenshot") + async for message in client.receive_response(): + messages.append(message) + return messages + + return asyncio.run(go()) + + +def tool_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.startswith("tool.")] + + +@pytest.mark.parametrize("order", ["hooks-first", "result-first"]) +def test_tool_result_image_and_pdf_upload_before_truncation_on_the_tool_span(f01, instrument, cli, order): + manager = f01.manager() + exporter = instrument(manager) + messages = converse(cli([{"type": "text", "text": "captured"}, image(SHOT), + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", + "data": b64(PDF)}}], order=order)) + flush(manager) + assert type(messages[-1]).__name__ == "ResultMessage" + assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ + ("application/pdf", "output", "tool_result", "toolu_screen_1"), + ("image/png", "output", "tool_result", "toolu_screen_1")] + assert sorted(f01.payloads) == sorted([SHOT, PDF]) + (tool,) = tool_spans(exporter) + parents = {u["association"]["parent"]["span_id"] for u in f01.uploads} + assert parents == {format(tool.context.span_id, "016x")} + # In both orders the tool span is still recording when its result arrives, so every + # uploaded occurrence is projected (fi.attachment.refs) on its upload parent. + assert sorted(r["occurrence_id"] for r in refs(tool)) == sorted(u["occurrence_id"] for u in f01.uploads) + assert tool.status.is_ok + # Bytes never reach a text attribute (tool output, conversation output). + for span in exporter.get_finished_spans(): + for value in span.attributes.values(): + assert b64(SHOT)[:40] not in str(value) + manager.shutdown(5) + + +def _duration_ms(span): + return (span.end_time - span.start_time) / 1e6 + + +def test_hooks_first_tool_span_keeps_the_hook_end_time_while_waiting_for_its_result(f01, instrument, cli): + manager = f01.manager() + exporter = instrument(manager) + converse(cli([image(SHOT)], gap=0.6)) + flush(manager) + (tool,) = tool_spans(exporter) + assert len(refs(tool)) == 1 and len(f01.uploads) == 1 + hook_ms = tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS] + # Ended at the PostToolUse time, not when the result streamed 600 ms later. + assert abs(_duration_ms(tool) - hook_ms) < 150 and _duration_ms(tool) < 450 + manager.shutdown(5) + + +def test_a_result_that_never_arrives_still_ends_the_tool_span_at_its_hook_time(f01, instrument, cli): + manager = f01.manager() + exporter = instrument(manager) + messages = converse(cli([image(SHOT)], order="no-result", gap=0.3)) + flush(manager) + assert type(messages[-1]).__name__ == "ResultMessage" + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + manager.shutdown(5) + + +def test_url_and_file_sources_never_fetch(f01, instrument, cli): + manager = f01.manager() + instrument(manager) + converse(cli([{"type": "image", "source": {"type": "url", "url": "http://127.0.0.1:9/x.png"}}, + {"type": "image", "source": {"type": "file", "file_id": "file_1"}}, + {"type": "image", "source": {"type": "base64", "media_type": "text/plain", "data": b64(SHOT)}}])) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny(f01, instrument, cli, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + converse(cli([image(SHOT)])) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing_and_the_conversation_is_unchanged(f01, instrument, cli): + instrument(create_attachment_manager(config=AttachmentConfig())) + messages = converse(cli([image(SHOT)])) + assert f01.uploads == [] + blocks = [m for m in messages if type(m).__name__ == "UserMessage"][0].content + assert blocks[0].content[0]["source"]["data"] == b64(SHOT) + + +def test_receive_messages_without_disconnect_ends_the_tool_span_at_its_hook(f01, instrument, cli): + """No receive_response() conversation is live, so PostToolUse ends the span as with attachments off.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)], gap=0.3) + clients = [] + + async def go(): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path)) + clients.append(client) + await client.connect() + await client.query("take a screenshot") + async for message in client.receive_messages(): + if type(message).__name__ == "ResultMessage": + break + # Returns without disconnect() or async-with: nothing flushes deferred spans. + + asyncio.run(go()) + flush(manager) + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + assert not clients[0]._tool_deferrals.ends and clients[0]._tool_deferrals.live == 0 + manager.shutdown(5) + + +def test_conversation_end_flushes_its_waiting_tool_span_without_disconnect(f01, instrument, cli): + """A result that never arrives: the span ends when receive_response() ends, not at the 30 s timer.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)], order="no-result", gap=0.2) + clients = [] + + async def go(): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path)) + clients.append(client) + await client.connect() + await client.query("take a screenshot") + async for _ in client.receive_response(): + pass + # Inspected before any disconnect(): only the conversation end can have ended it. + deferrals = clients[0]._tool_deferrals + return len(tool_spans(exporter)), list(deferrals.ends), deferrals.live + + ended, waiting, live = asyncio.run(go()) + assert (ended, waiting, live) == (1, [], 0) + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + manager.shutdown(5) + + +def test_one_conversation_ending_does_not_end_another_clients_deferred_tool_span(f01, instrument, cli, tmp_path): + """B's tool span waits for its result while A runs to completion; A's end flushes only A's spans.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)]) + other = png(seed=52) + a_result = tmp_path / "result-a.json" + a_result.write_text(json.dumps({"content": [image(other)], "is_error": False})) + release = tmp_path / "release-b" + clients = {} + parked = {} + + async def converse_as(name, **env): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path, FAKE_TOOL_ID="toolu_" + name, **env)) + clients[name] = client + async with client: + await client.query("take a screenshot") + async for _ in client.receive_response(): + pass + + async def go(): + b = asyncio.ensure_future(converse_as("b", FAKE_CLAUDE_HOLD=str(release))) + deadline = time.monotonic() + 15 + while not ("b" in clients and clients["b"]._tool_deferrals.ends): + assert time.monotonic() < deadline and not b.done(), "B never parked its tool span" + await asyncio.sleep(0.02) + await converse_as("a", FAKE_CLAUDE_RESULT=str(a_result)) + parked["b"] = list(clients["b"]._tool_deferrals.ends) + release.write_text("go") + await b + + asyncio.run(go()) + flush(manager) + assert parked["b"] == ["toolu_b"] + tools = {s.attributes[ClaudeAgentAttributes.TOOL_USE_ID]: s for s in tool_spans(exporter)} + assert sorted(tools) == ["toolu_a", "toolu_b"] + assert sorted(u.get("tool_call_id") for u in f01.uploads) == ["toolu_a", "toolu_b"] + assert sorted(f01.payloads) == sorted([SHOT, other]) + for upload in f01.uploads: + tool = tools[upload["tool_call_id"]] + assert tool.status.is_ok + assert upload["association"]["parent"]["span_id"] == format(tool.context.span_id, "016x") + assert [r["occurrence_id"] for r in refs(tool)] == [upload["occurrence_id"]] + assert not clients["a"]._tool_deferrals.ends and not clients["b"]._tool_deferrals.ends + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py new file mode 100644 index 000000000..f710e16e0 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py @@ -0,0 +1,128 @@ +"""Real firecrawl-py v2 clients against a loopback Firecrawl API -> loopback F01.""" + +import asyncio +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("firecrawl") +pytest.importorskip("traceai_firecrawl") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from firecrawl import AsyncFirecrawl, Firecrawl # noqa: E402 +from traceai_firecrawl import FirecrawlInstrumentor # noqa: E402 + +from _harness import data_uri, flush, png, refs, span_named # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "firecrawl" / "tests")) +from _firecrawl_fake import JOB_ID, FakeFirecrawl, crawl_status # noqa: E402 + +SHOT = png(seed=61) +ACTION_SHOT = png(seed=62) +KEY = "fc-SYNTHETIC-PLACEHOLDER" + + +@pytest.fixture +def fake(): + server = FakeFirecrawl() + yield server + server.close() + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + FirecrawlInstrumentor().uninstrument() + + +def page(screenshot, actions=None): + data = {"markdown": "body", "screenshot": screenshot, "metadata": {"sourceURL": "https://example.com"}} + if actions is not None: + data["actions"] = {"screenshots": actions} + return {"success": True, "data": data} + + +def test_inline_screenshots_upload_and_hosted_urls_are_never_fetched(f01, instrument, fake): + manager = f01.manager() + exporter = instrument(manager) + hosted = fake.origin + "/hosted/screenshot.png" + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT), [data_uri("image/jpeg", ACTION_SHOT), hosted])) + document = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert document.screenshot == data_uri("image/png", SHOT) # the SDK result is unchanged + assert sorted((u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads) == [ + ("image/jpeg", "output", "tool_result"), ("image/png", "output", "tool_result")] + assert sorted(f01.payloads) == sorted([SHOT, ACTION_SHOT]) + assert fake.calls_to("GET", "/hosted/screenshot.png") == 0 + assert len(refs(span_named(exporter, "firecrawl.scrape"))) == 2 + manager.shutdown(5) + + +def test_hosted_only_screenshot_uploads_nothing(f01, instrument, fake): + manager = f01.manager() + instrument(manager) + fake.routes[("POST", "/v2/scrape")] = (200, page(fake.origin + "/hosted/x.png")) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert f01.uploads == [] and fake.calls_to("GET", "/hosted/x.png") == 0 + manager.shutdown(5) + + +def test_async_client_scrape(f01, instrument, fake): + manager = f01.manager() + instrument(manager) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + + async def go(): + return await AsyncFirecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape( + "https://example.com", formats=["screenshot"]) + + asyncio.run(go()) + flush(manager) + assert f01.payloads == [SHOT] + manager.shutdown(5) + + +def test_crawl_status_documents_upload_on_the_status_span(f01, instrument, fake): + manager = f01.manager() + exporter = instrument(manager) + body = crawl_status("completed", pages=2) + body["data"] = [dict(page, screenshot=data_uri("image/png", png(seed=63 + n))) for n, page in enumerate(body["data"])] + fake.routes[("GET", "/v2/crawl/" + JOB_ID)] = (200, body) + job = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).get_crawl_status(JOB_ID) + flush(manager) + assert len(job.data) == 2 + assert sorted(f01.payloads) == sorted([png(seed=63), png(seed=64)]) + assert len(refs(span_named(exporter, "firecrawl.get_crawl_status"))) == 2 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_inputs", "pii_redaction"]) +def test_local_restrictions_deny(f01, instrument, fake, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, instrument, fake): + instrument(create_attachment_manager(config=AttachmentConfig())) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_mcp_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_attachments.py new file mode 100644 index 000000000..155cef0e3 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_mcp_attachments.py @@ -0,0 +1,244 @@ +"""Real ``mcp`` ClientSession over stdio -> wrap_mcp_client -> TLS loopback F01 server.""" + +import asyncio +import json +import sys +import threading +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") +traceai_mcp = pytest.importorskip("traceai_mcp") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from mcp import ClientSession, StdioServerParameters # noqa: E402 +from mcp.client.stdio import stdio_client # noqa: E402 + +from _harness import PDF, flush, no_bytes, refs, span_named, uploaded # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _mcp_media_server as server # noqa: E402 + +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" + + +def run(spans, manager, calls, **wrap): + """Open a real stdio session, wrap it, and run ``calls(session)``.""" + provider, _ = spans + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + traced = traceai_mcp.wrap_mcp_client(session, tracer_provider=provider, attachments=manager, **wrap) + return await calls(traced) + + return asyncio.run(go()) + + +def test_tool_image_audio_and_embedded_blob_upload_with_tool_identity(f01, spans): + manager = f01.manager() + _, exporter = spans + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + result = await session.call_tool("report", {}) + reads = await session.call_tool("reads", {}) + return result, reads + + result, reads = run(spans, manager, calls) + flush(manager) + assert sorted(u["mime_type"] for u in f01.uploads) == ["application/pdf", "audio/wav", "image/png"] + assert {(u["direction"], u["purpose"]) for u in f01.uploads} == {("output", "tool_result")} + assert server.png(1) in f01.payloads and server.WAV in f01.payloads and PDF in f01.payloads + # The resource_link URI was never dereferenced. + assert json.loads(reads.content[0].text)["secret"] == 0 + # The SDK result is unchanged. + assert result.content[0].resource.blob == server.b64(PDF) + shot = [s for s in exporter.get_finished_spans() if s.name == "mcp.tools/call"] + assert len(shot) == 4 + assert sum(len(refs(s)) for s in shot) == 3 + assert {r["role"] for s in shot for r in refs(s)} == {"tool"} + no_bytes(exporter, server.png(1)) + manager.shutdown(5) + + +def test_resources_read_blob_uploads_and_text_is_ignored(f01, spans): + manager = f01.manager() + + async def calls(session): + return await session.read_resource("file:///shot.png") + + result = run(spans, manager, calls) + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["image/png"] + assert f01.payloads == [server.png(5)] + assert result.contents[0].blob == server.b64(server.png(5)) + manager.shutdown(5) + + +def test_relabeled_unknown_malformed_and_oversized_media_upload_nothing(f01, spans): + manager = f01.manager() + + async def calls(session): + for name in ("relabeled", "malformed", "big"): + await session.call_tool(name, {}) + + run(spans, manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_part_budget_caps_uploads_per_result(f01, spans): + manager = f01.manager() + + async def calls(session): + await session.call_tool("many", {"count": 70}) + + run(spans, manager, calls) + flush(manager) + assert len(f01.uploads) == 64 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_messages", "hide_output_text", + "pii_redaction"]) +def test_local_hide_flags_and_pii_deny_every_tool_media(f01, spans, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + + run(spans, manager, calls, trace_config=TraceConfig(**{flag: True})) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_hide_input_images_denies_images_but_not_audio(f01, spans): + manager = f01.manager() + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + + run(spans, manager, calls, trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["audio/wav"] + manager.shutdown(5) + + +def test_env_hide_flag_denies(f01, spans, monkeypatch): + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + manager = f01.manager() + + async def calls(session): + await session.call_tool("screenshot", {}) + + run(spans, manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_disabled_manager_and_unsampled_span_upload_nothing_and_start_no_threads(f01, spans): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + + before = {t.name for t in threading.enumerate()} + disabled = create_attachment_manager(config=AttachmentConfig()) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + result = run(spans, disabled, calls) + assert result.content[1].data == server.b64(server.png(1)) + assert not {t.name for t in threading.enumerate()} - before + manager = f01.manager() + unsampled = TracerProvider(sampler=ALWAYS_OFF) + run((unsampled, None), manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_provider_manager_is_the_default_and_rewrapping_is_idempotent(f01, spans): + manager = f01.manager() + provider, exporter = spans + provider.attachment_manager = manager + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + traced = traceai_mcp.wrap_mcp_client(session, tracer_provider=provider) + assert traceai_mcp.wrap_mcp_client(traced) is traced + await traced.call_tool("screenshot", {}) + + asyncio.run(go()) + flush(manager) + assert len(f01.uploads) == 1 + assert len([s for s in exporter.get_finished_spans() if s.name == "mcp.tools/call"]) == 1 + manager.shutdown(5) + + +def test_tool_errors_keep_the_sdk_exception_and_end_the_span(f01, spans): + manager = f01.manager() + _, exporter = spans + + async def calls(session): + with pytest.raises(Exception): + await session.read_resource("file:///missing.png") + + run(spans, manager, calls) + span = span_named(exporter, "mcp.resources/read") + assert span.status.status_code.name == "ERROR" + manager.shutdown(5) + + +def test_capture_mcp_result_is_preflight_first_and_never_raises(f01, spans): + manager = f01.manager() + provider, _ = spans + reads = [] + + class Part(dict): + def get(self, key, default=None): + if key == "data": + reads.append(key) + return super().get(key, default) + + tracer = provider.get_tracer("t") + with tracer.start_as_current_span("denied") as span: + traceai_mcp.capture_mcp_result(manager, span, {"content": [Part(type="image", data="AAAA", mimeType="image/png")]}, + trace_config=TraceConfig(hide_outputs=True)) + assert reads == [] + assert traceai_mcp.capture_mcp_result(manager, span, object()) == {} + assert traceai_mcp.capture_mcp_result(None, span, {"content": []}) == {} + manager.shutdown(5) + + +def test_error_results_follow_hide_error_details(f01, spans): + """An isError result's media is error detail: hide_error_details denies it, a success result still uploads.""" + manager = f01.manager() + provider, _ = spans + shot, ok = server.png(31), server.png(32) + + def result(data, error): + return {"isError": error, "content": [{"type": "image", "data": server.b64(data), "mimeType": "image/png"}]} + + config = TraceConfig(hide_error_details=True) + with provider.get_tracer("t").start_as_current_span("failed") as span: + assert traceai_mcp.capture_mcp_result(manager, span, result(shot, True), trace_config=config) == {} + with provider.get_tracer("t").start_as_current_span("succeeded") as span: + assert len(traceai_mcp.capture_mcp_result(manager, span, result(ok, False), trace_config=config)) == 1 + flush(manager) + assert f01.payloads == [ok] + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py new file mode 100644 index 000000000..458ceff2e --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py @@ -0,0 +1,102 @@ +"""Real mcp-use MCPAgent (scripted LangChain chat model) + stdio MCP media server -> loopback F01.""" + +import asyncio +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("mcp_use") +pytest.importorskip("traceai_mcp_use") +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "mcp-use" / "tests")) +import _mcp_use_support as support # noqa: E402 (also turns mcp-use telemetry off) + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_mcp_use import FutureAGICallback # noqa: E402 + +from _harness import flush, refs # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _mcp_media_server as server # noqa: E402 + +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" + + +def run(callback, tool, args=None): + from mcp_use import MCPAgent, MCPClient + + async def go(): + client = MCPClient.from_dict({"mcpServers": {"media": {"command": sys.executable, "args": [str(SERVER)]}}}) + try: + agent = MCPAgent(llm=support.ChatFake(script=[support.tool_call(tool, args or {}, "call_media_1"), + support.answer()]), + client=client, callbacks=[callback]) + return await agent.run(support.PROMPT) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +def tool_span(exporter, name): + return [s for s in exporter.get_finished_spans() if s.name == "execute_tool " + name][-1] + + +def test_tool_image_and_blob_upload_on_the_tool_span(f01, spans): + manager = f01.manager() + provider, exporter = spans + callback = FutureAGICallback(tracer_provider=provider, attachments=manager) + assert run(callback, "screenshot") == support.ANSWER + # LangChain turns an audio-only tool result into a string before callbacks see it: no typed block, no upload. + run(callback, "voice") + run(callback, "report") + flush(manager) + assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ + ("application/pdf", "output", "tool_result", "call_media_1"), + ("image/png", "output", "tool_result", "call_media_1")] + assert server.png(1) in f01.payloads + span = tool_span(exporter, "screenshot") + assert len(refs(span)) == 1 and refs(span)[0]["role"] == "tool" + assert server.b64(server.png(1))[:40] not in str(span.attributes) + manager.shutdown(5) + + +def test_provider_manager_is_the_default(f01, spans): + manager = f01.manager() + provider, _ = spans + provider.attachment_manager = manager + run(FutureAGICallback(tracer_provider=provider), "screenshot") + flush(manager) + assert len(f01.uploads) == 1 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "pii_redaction"]) +def test_local_restrictions_deny(f01, spans, flag): + manager = f01.manager() + provider, _ = spans + before = len(f01.after_bootstrap()) + run(FutureAGICallback(tracer_provider=provider, config=TraceConfig(**{flag: True}), attachments=manager), + "screenshot") + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_malformed_and_relabeled_media_upload_nothing(f01, spans): + manager = f01.manager() + provider, _ = spans + callback = FutureAGICallback(tracer_provider=provider, attachments=manager) + run(callback, "relabeled") + run(callback, "malformed") + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, spans): + provider, _ = spans + run(FutureAGICallback(tracer_provider=provider, attachments=create_attachment_manager(config=AttachmentConfig())), + "screenshot") + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py new file mode 100644 index 000000000..946697b24 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py @@ -0,0 +1,215 @@ +"""Real OpenAI Agents SDK Runner (scripted local Model, real ComputerTool and function tools) -> loopback F01.""" + +import asyncio +import json +from importlib.metadata import version + +import pytest + +agents = pytest.importorskip("agents") +pytest.importorskip("traceai_openai_agents") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from packaging.version import Version # noqa: E402 +from traceai_openai_agents import OpenAIAgentsInstrumentor # noqa: E402 + +from _harness import data_uri, flush, png, refs # noqa: E402 + +SDK = Version(version("openai-agents")) +COMPUTER_SPAN = SDK >= Version("0.10.0") # the SDK records computer calls on a function span from 0.10.0 +TYPED_IMAGE = SDK >= Version("0.4.0") # ToolOutputImage exists from 0.4.0 +SHOT = png(seed=41) +TOOL_IMAGE = png(seed=42) + + +class ScriptedModel(agents.Model): + """Replays model outputs; no network.""" + + def __init__(self, turns): + self.turns = list(turns) + self.inputs = [] + + async def get_response(self, *args, **kwargs): + from agents.usage import Usage + + self.inputs.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + output = self.turns.pop(0) + fields = dict(output=output, usage=Usage(), response_id=None) + try: + return agents.ModelResponse(**fields) + except TypeError: + return agents.ModelResponse(output=output, usage=Usage(), response_id=None, request_id=None) + + def stream_response(self, *args, **kwargs): + raise NotImplementedError + + +def message(text="done"): + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + return ResponseOutputMessage(id="m1", type="message", role="assistant", status="completed", + content=[ResponseOutputText(type="output_text", text=text, annotations=[])]) + + +def computer_call(call_id="call_shot"): + from openai.types.responses import ResponseComputerToolCall + + return ResponseComputerToolCall.model_validate({ + "id": "cc1", "type": "computer_call", "call_id": call_id, "status": "completed", + "pending_safety_checks": [], "action": {"type": "screenshot"}}) + + +def function_call(name, call_id="call_fn"): + from openai.types.responses import ResponseFunctionToolCall + + return ResponseFunctionToolCall(id="fc1", type="function_call", call_id=call_id, name=name, arguments="{}") + + +class Screen(agents.AsyncComputer): + shots = 0 + + @property + def environment(self): + return "browser" + + @property + def dimensions(self): + return (1024, 768) + + async def screenshot(self): + Screen.shots += 1 + return data_uri("image/png", SHOT).split(",", 1)[1] + + async def click(self, x, y, button): + return None + + async def double_click(self, x, y): + return None + + async def scroll(self, x, y, scroll_x, scroll_y): + return None + + async def type(self, text): + return None + + async def wait(self): + return None + + async def move(self, x, y): + return None + + async def keypress(self, keys): + return None + + async def drag(self, path): + return None + + +@pytest.fixture +def traced(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + state = {} + + def install(manager, config=None): + OpenAIAgentsInstrumentor().instrument(tracer_provider=provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + OpenAIAgentsInstrumentor().uninstrument() # lets the next test install a fresh processor + agents.set_trace_processors([]) + provider.shutdown() + + +def run_computer(manager, install, config=None): + exporter = install(manager, config) + model = ScriptedModel([[computer_call()], [message()]]) + agent = agents.Agent(name="operator", model=model, tools=[agents.ComputerTool(computer=Screen())]) + result = asyncio.run(agents.Runner.run(agent, "take a screenshot")) + return exporter, model, result + + +def function_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "TOOL"] + + +def test_computer_screenshot_uploads_on_the_computer_tool_span(f01, traced): + manager = f01.manager() + exporter, model, result = run_computer(manager, traced) + flush(manager) + assert result.final_output == "done" + if not COMPUTER_SPAN: + assert f01.uploads == [] # no function span records the computer call before 0.10.0 + return + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "computer_screenshot")] + assert f01.payloads == [SHOT] + tool = [s for s in function_spans(exporter) if refs(s)] + assert len(tool) == 1 + output = tool[0].attributes["output.value"] + assert output.startswith("fi-attachment:") and output[len("fi-attachment:"):] == refs(tool[0])[0]["occurrence_id"] + # The SDK still sent the original screenshot to the model. + sent = json.dumps(model.inputs[-1], default=str) + assert data_uri("image/png", SHOT) in sent + manager.shutdown(5) + + +@pytest.mark.skipif(not TYPED_IMAGE, reason="ToolOutputImage needs openai-agents 0.4.0") +def test_function_tool_typed_image_uploads_and_url_or_file_id_never_fetch(f01, traced): + manager = f01.manager() + exporter = traced(manager) + + @agents.function_tool + def chart() -> list: + """Draw a chart.""" + return [agents.ToolOutputImage(image_url=data_uri("image/png", TOOL_IMAGE)), + agents.ToolOutputImage(image_url="https://example.invalid/chart.png"), + agents.ToolOutputImage(file_id="file-123")] + + model = ScriptedModel([[function_call("chart")], [message()]]) + agent = agents.Agent(name="analyst", model=model, tools=[chart]) + asyncio.run(agents.Runner.run(agent, "chart it")) + flush(manager) + assert [(u["mime_type"], u["purpose"]) for u in f01.uploads] == [("image/png", "tool_result")] + assert f01.payloads == [TOOL_IMAGE] + span = [s for s in function_spans(exporter) if refs(s)][0] + assert "https://example.invalid/chart.png" in span.attributes["output.value"] + assert data_uri("image/png", TOOL_IMAGE) not in span.attributes["output.value"] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny_the_screenshot(f01, traced, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + run_computer(manager, traced, TraceConfig(**{flag: True})) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_keeps_the_old_attributes(f01, traced): + exporter, _, result = run_computer(create_attachment_manager(config=AttachmentConfig()), traced) + assert result.final_output == "done" and f01.uploads == [] + for span in function_spans(exporter): + assert not refs(span) + assert not str(span.attributes.get("output.value", "")).startswith("fi-attachment:") + + +def test_sensitive_data_off_records_and_uploads_nothing(f01, traced): + manager = f01.manager() + traced(manager) + model = ScriptedModel([[computer_call()], [message()]]) + agent = agents.Agent(name="operator", model=model, tools=[agents.ComputerTool(computer=Screen())]) + asyncio.run(agents.Runner.run(agent, "shot", run_config=agents.RunConfig(trace_include_sensitive_data=False))) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_smolagents_attachments.py b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py new file mode 100644 index 000000000..1762d5a49 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py @@ -0,0 +1,231 @@ +"""Real smolagents (OpenAIServerModel on a loopback OpenAI double, real Tool and agent loop) -> loopback F01.""" + +import io + +import pytest + +smolagents = pytest.importorskip("smolagents") +PIL_Image = pytest.importorskip("PIL.Image") +pytest.importorskip("openai") +pytest.importorskip("traceai_smolagents") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_smolagents import SmolagentsInstrumentor # noqa: E402 + +from _harness import flush, harness # noqa: E402 + +provider = harness.provider # the loopback OpenAI-compatible API (fixture) + + +def picture(seed=0, size=(8, 6)): + image = PIL_Image.new("RGB", size, (seed % 256, 10, 20)) + return image + + +def png_of(image): + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + SmolagentsInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + SmolagentsInstrumentor().uninstrument() + + +def model(provider): + return smolagents.OpenAIServerModel(model_id="m", api_base=provider.url + "/v1", api_key="SYNTHETIC_KEY") + + +def user_message(*images): + content = [{"type": "text", "text": "look"}] + [{"type": "image", "image": image} for image in images] + if not hasattr(smolagents.OpenAIServerModel, "generate"): + return {"role": "user", "content": content} # releases before generate() take plain dicts + from smolagents.models import ChatMessage, MessageRole + + return ChatMessage(role=MessageRole.USER, content=content) + + +def call(m, messages): + method = getattr(m, "generate", None) or m.__call__ + return method(messages) + + +def test_model_input_images_upload_once_on_the_model_span(f01, instrument, provider): + manager = f01.manager() + exporter = instrument(manager) + shot = picture(1) + m = model(provider) + call(m, [user_message(shot)]) + call(m, [user_message(shot)]) # memory replay of the same observation image + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "input", "model_input")] + assert f01.payloads == [png_of(shot)] + # The model still received the image. + assert any(b"data:image/png;base64," in raw for _, _, raw in provider.requests) + llm = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "LLM"] + assert llm and harness.refs(llm[0]) + manager.shutdown(5) + + +def test_tool_returned_images_upload_on_the_tool_span(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + raw, wrapped_image = picture(2), picture(3) + + class Snap(smolagents.Tool): + name = "snap" + description = "Take a picture." + inputs = {"which": {"type": "string", "description": "which"}} + output_type = "image" + + def forward(self, which): + if which == "raw": + return raw + return smolagents.AgentImage(wrapped_image) + + tool = Snap() + tool(which="raw", sanitize_inputs_outputs=False) + tool(which="agent", sanitize_inputs_outputs=False) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] * 2 + assert sorted(f01.payloads) == sorted([png_of(raw), png_of(wrapped_image)]) + tools = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "TOOL"] + assert sum(len(harness.refs(s)) for s in tools) == 2 + manager.shutdown(5) + + +def test_file_backed_and_audio_values_are_never_read(f01, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + picture(4).save(path) + + class Files(smolagents.Tool): + name = "files" + description = "Return files." + inputs = {"which": {"type": "string", "description": "which"}} + output_type = "image" + + def forward(self, which): + if which == "agent-path": + return smolagents.AgentImage(str(path)) + return PIL_Image.open(path) + + tool = Files() + tool(which="agent-path", sanitize_inputs_outputs=False) + tool(which="opened", sanitize_inputs_outputs=False) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny_model_images(f01, instrument, provider, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + call(model(provider), [user_message(picture(5))]) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_agent_run_with_task_images_uploads_each_image_once(f01, instrument, provider): + manager = f01.manager() + instrument(manager) + agent = smolagents.ToolCallingAgent(tools=[], model=model(provider), max_steps=1) + try: + agent.run("describe", images=[picture(6)]) + except Exception: + pass # the double never calls a tool; the loop's own errors are not under test + flush(manager) + assert [u["purpose"] for u in f01.uploads] == ["model_input"] + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, instrument, provider): + instrument(create_attachment_manager(config=AttachmentConfig())) + call(model(provider), [user_message(picture(7))]) + assert f01.uploads == [] + + +def test_every_model_call_carrying_a_captured_image_is_marked_owner(f01, instrument, provider, monkeypatch): + """A replayed memory image is already uploaded: the provider adapter below must not upload it again.""" + import openai.resources.chat.completions as completions + from fi_instrumentation.attachments import tool_media + from opentelemetry import context as otel_context + + marks = [] + real = completions.Completions.create + + def create(self, *args, **kwargs): + marks.append(otel_context.get_value(tool_media.OWNER_KEY)) + return real(self, *args, **kwargs) + + monkeypatch.setattr(completions.Completions, "create", create) + manager = f01.manager() + instrument(manager) + shot = picture(11) + m = model(provider) + call(m, [user_message(shot)]) + call(m, [user_message(shot)]) # memory replay: the same image object goes to the model again + call(m, [user_message()]) # no image: no mark + flush(manager) + assert marks == [True, True, None] + assert len(f01.uploads) == 1 + assert sum(raw.count(b"data:image/png;base64,") for _, _, raw in provider.requests) == 2 + manager.shutdown(5) + + +def test_seen_table_is_fresh_in_a_forked_child_even_if_its_lock_was_held(): + import os + import signal + import threading + + from traceai_smolagents import _attachments as sa + + if not hasattr(os, "fork"): + pytest.skip("no fork") + held, release = threading.Event(), threading.Event() + + def holder(): + with sa._SEEN._owned_lock(): + held.set() + release.wait(5) + + thread = threading.Thread(target=holder) + thread.start() + held.wait(5) + + class Manager: + pass + + keep = Manager() + pid = os.fork() + if pid == 0: # child: must not block on the lock the parent's thread held at fork + signal.alarm(5) + try: + image = picture(1) + sa._SEEN.add(keep, image) + os._exit(0 if (keep, image) in sa._SEEN else 4) + except BaseException: + os._exit(3) + release.set() + thread.join() + _, status = os.waitpid(pid, 0) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 diff --git a/python/tests/attachments_mcp_agents/test_tool_media.py b/python/tests/attachments_mcp_agents/test_tool_media.py new file mode 100644 index 000000000..c77c58fd4 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_tool_media.py @@ -0,0 +1,205 @@ +"""The shared tool-media helper: deny-wins folding, preflight before read, budgets, dedup, ownership.""" + +import struct + +import pytest + +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy, create_attachment_manager +from fi_instrumentation.attachments import tool_media +from opentelemetry import context as otel_context + +from _harness import b64, data_uri, flush, png, refs + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_messages", "hide_input_text", "hide_outputs", + "hide_output_messages", "hide_output_text", "pii_redaction"]) +def test_tool_policy_folds_input_and_output_restrictions(flag): + policy = tool_media.tool_capture_policy(TraceConfig(**{flag: True})) + assert policy.capture_content is False and policy.model_input and policy.contains_output + + +def test_tool_policy_images_and_errors(): + assert tool_media.tool_capture_policy(TraceConfig(hide_input_images=True), image=True).capture_content is False + assert tool_media.tool_capture_policy(TraceConfig(hide_input_images=True)).capture_content is True + assert tool_media.tool_capture_policy(TraceConfig(hide_error_details=True), error=True).capture_content is False + base = CapturePolicy(record_outputs=False) + assert tool_media.tool_capture_policy(None, base).record_outputs is False + + +def test_owner_context_disables_capture(f01, spans): + manager = f01.manager() + provider, _ = spans + token = otel_context.attach(tool_media.with_ownership()) + try: + with provider.get_tracer("t").start_as_current_span("owned") as span: + media = tool_media.ToolMedia(manager, span, "test") + assert not media.usable + assert tool_media.tool_capture_policy().requested is False + finally: + otel_context.detach(token) + manager.shutdown(5) + + +def test_value_is_read_only_after_admission_and_only_once(f01, spans): + manager = f01.manager() + provider, _ = spans + reads = [] + + def read(): + reads.append(1) + return b64(png(seed=1)) + + with provider.get_tracer("t").start_as_current_span("tool") as span: + denied = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_outputs=True)) + assert denied.base64("a", read, "image/png", purpose="tool_result", expect="image") is None + assert reads == [] + media = tool_media.ToolMedia(manager, span, "test") + first = media.base64("b", read, "image/png", purpose="tool_result", expect="image") + again = tool_media.ToolMedia(manager, span, "test").base64("b", read, "image/png", purpose="tool_result") + assert first is not None and again == first and reads == [1] + assert media.base64("c", read, "audio/wav", purpose="tool_result", expect="image") is None + assert media.base64("d", read, "image/gif", purpose="tool_result") is None + assert reads == [1] + flush(manager) + assert len(f01.uploads) == 1 and len(refs(span)) == 1 + manager.shutdown(5) + + +def test_budget_and_part_cap(f01, spans): + manager = f01.manager() + provider, _ = spans + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", budget=200, max_parts=2) + image = png(width=4, height=4, seed=2) + assert len(image) < 200 + assert media.base64("1", lambda: b64(image), "image/png", purpose="tool_result") is not None + assert media.remaining == 200 - len(image) + assert media.data_uri("2", lambda: data_uri("image/png", png(width=40, height=40, seed=3)), ("image/png",), + purpose="tool_result") is None # over the remaining budget + assert media.base64("3", lambda: b64(png(seed=4)), "image/png", purpose="tool_result") is None # part cap + flush(manager) + assert len(f01.uploads) == 1 + manager.shutdown(5) + + +def test_disabled_and_missing_managers_are_inert(spans): + provider, _ = spans + with provider.get_tracer("t").start_as_current_span("tool") as span: + for manager in (None, create_attachment_manager(config=AttachmentConfig())): + media = tool_media.ToolMedia(manager, span, "test") + assert not media.usable + assert media.base64("x", lambda: pytest.fail("read"), "image/png", purpose="tool_result") is None + + +def test_pil_images_are_encoded_only_when_in_memory(f01, spans, tmp_path): + Image = pytest.importorskip("PIL.Image") + manager = f01.manager() + provider, _ = spans + in_memory = Image.new("RGB", (4, 3), (1, 2, 3)) + path = tmp_path / "x.png" + in_memory.save(path) + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test") + assert media.image("file", Image.open(path), purpose="tool_result") is None + assert media.image("mem", in_memory, purpose="tool_result") is not None + assert media.image("huge", Image.new("L", (4096, 4096)), purpose="tool_result") is None + assert media.image("not-an-image", b"\x89PNG", purpose="tool_result") is None + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["image/png"] + manager.shutdown(5) + + +def test_signature_of_another_modality_is_denied_and_never_relabelled(f01, spans): + """A PNG declared as audio, a generic file or a PDF is not uploaded (hide_input_images cannot be escaped).""" + from _harness import PDF, wav + + manager = f01.manager() + provider, _ = spans + shot = png(seed=21) + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_input_images=True)) + assert media.base64("a", lambda: b64(shot), "audio/wav", purpose="tool_result", expect="audio") is None + assert media.base64("b", lambda: b64(shot), "application/octet-stream", purpose="tool_result") is None + assert media.base64("c", lambda: b64(shot), "application/pdf", purpose="tool_result") is None + assert media.data_uri("d", lambda: "data:audio/wav;base64," + b64(shot), ("audio/wav",), + purpose="tool_result") is None + # Matching signatures and signature-less bytes still upload. + assert media.base64("e", lambda: b64(wav()), "audio/wav", purpose="tool_result", expect="audio") + assert media.base64("f", lambda: b64(PDF), "application/pdf", purpose="tool_result") + flush(manager) + assert shot not in f01.payloads + assert sorted(u["mime_type"] for u in f01.uploads) == ["application/pdf", "audio/wav"] + manager.shutdown(5) + + +@pytest.mark.parametrize("head,kind", [ + (b"\x89PNG\r\n\x1a\n\x00\x00\x00\x00", "image"), (b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01", "image"), + (b"RIFF\x00\x00\x00\x00WEBP", "image"), (b"RIFF\x00\x00\x00\x00WAVE", "audio"), (b"OggS\x00\x02", "audio"), + (b"fLaC\x00\x00", "audio"), (b"ID3\x04\x00", "audio"), (b"%PDF-1.7\n", "document"), + (b"\x00\x00\x00\x18ftypmp42", None), (b"\x1aE\xdf\xa3\x9fB\x86", None), (b"RIFF\x00\x00\x00\x00AVI ", None), + (b"plain text", None), (b"BM6\x00\x00\x00\x00\x00\x00\x006\x00", "image"), (b"II*\x00\x08\x00\x00\x00", "image"), + (b"MM\x00*\x00\x00\x00\x08", "image"), (b"\x00\x00\x00\x18ftypavif", "image"), + (b"\x00\x00\x00\x18ftypheic", "image"), (b"\x00\x00\x00\x18ftypheix", "image"), + (b"\x00\x00\x00\x18ftypmif1", "image"), (b"\x00\x00\x00\x18ftypmsf1", "image"), + (b"\x00\x00\x00\x18ftypisom", None), (b"\x00\x00\x00\x18ftypM4A ", None), (b"\x00\x00\x00\x14ftypqt ", None), + (b"\x00\x00\x00\x18avifftyp", None)]) +def test_signature_kind(head, kind): + assert tool_media.signature_kind(head) == kind + + +def _bmp(): + pixels = b"\x00\x00\xff\x00" * 4 + header = b"BM" + struct.pack("I", 24) + b"ftyp" + brand + b"\x00\x00\x00\x00" + brand + b"mif1" + b"\x00" * 32 + + +_IMAGE_MISLABELS = [ + ("bmp-as-audio/wav", "audio/wav", _bmp), ("bmp-as-audio/L16", "audio/L16", _bmp), + ("bmp-as-octet-stream", "application/octet-stream", _bmp), ("tiff-le-as-audio/L16", "audio/L16", _tiff), + ("tiff-be-as-audio/wav", "audio/wav", lambda: _tiff(b"MM\x00*")), + ("avif-as-video/mp4", "video/mp4", lambda: _heif(b"avif")), ("heic-as-audio/mp4", "audio/mp4", lambda: _heif(b"heic")), + ("heix-as-octet-stream", "application/octet-stream", lambda: _heif(b"heix")), + ("mif1-as-video/mp4", "video/mp4", lambda: _heif(b"mif1")), ("msf1-as-video/mp4", "video/mp4", lambda: _heif(b"msf1"))] + + +@pytest.mark.parametrize("label,mime,make", _IMAGE_MISLABELS, ids=[case[0] for case in _IMAGE_MISLABELS]) +def test_bmp_tiff_and_heif_images_cannot_escape_hide_input_images(f01, spans, label, mime, make): + """Image bytes declared as audio, video or a generic file are not uploaded under hide_input_images.""" + manager = f01.manager() + provider, _ = spans + data = make() + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_input_images=True)) + assert media.base64("a", lambda: b64(data), mime, purpose="tool_result") is None + assert media.data_uri("b", lambda: "data:%s;base64,%s" % (mime, b64(data)), (mime,), + purpose="tool_result") is None + flush(manager) + assert f01.uploads == [] and data not in f01.payloads + manager.shutdown(5) + + +def test_octet_stream_with_a_recognised_signature_is_skipped(f01, spans): + """application/octet-stream is the file modality: image, audio or PDF bytes in it are skipped, never relabelled.""" + from _harness import PDF, wav + + manager = f01.manager() + provider, _ = spans + plain = b"opaque tool output, no known signature" * 4 + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test") + for position, data in (("png", png(seed=31)), ("wav", wav()), ("pdf", PDF), ("bmp", _bmp())): + assert media.base64(position, lambda d=data: b64(d), "application/octet-stream", + purpose="tool_result") is None, position + assert media.base64("plain", lambda: b64(plain), "application/octet-stream", purpose="tool_result") + flush(manager) + assert [(u["mime_type"], p) for u, p in zip(f01.uploads, f01.payloads)] == [("application/octet-stream", plain)] + manager.shutdown(5) diff --git a/python/tests/attachments_providers/test_auto_framework_media.py b/python/tests/attachments_providers/test_auto_framework_media.py new file mode 100644 index 000000000..43699d53c --- /dev/null +++ b/python/tests/attachments_providers/test_auto_framework_media.py @@ -0,0 +1,347 @@ +"""Auto with the Agno and Pydantic AI media adapters (#691): real frameworks, loopback F01. + +traceai.init(auto_instrument=["agno"] / ["pydantic_ai"]) must hand each framework the ONE +shared manager in the keyword arguments it accepts, carry auto's hide flags, and stay a strict +no-op with attachments OFF. Skips when the installed framework version has no media adapter, +unless TRACEAI_EXPECT_FRAMEWORK_ADAPTERS=1, which turns the skip into a failure. + +With the OpenAI instrumentor also enabled, each model call is uploaded ONCE, on the framework span +(the framework runs the nested SDK call under the owner mark). When auto's trace_config sets +pii_redaction or hide_error_details in code, Agno (which reads them only from the environment) +gets no media manager and its row is config_unsupported. +""" + +import importlib +import json +import os + +import pytest + +traceai = pytest.importorskip("traceai") + +from conftest import flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +EXPECT = os.environ.get("TRACEAI_EXPECT_FRAMEWORK_ADAPTERS") == "1" +IMAGE = png(seed=81) +PROMPT = "describe this synthetic picture" + + +def _adapter(path, name): + try: + return list(getattr(importlib.import_module(path), name)["operations"]) + except Exception: + if EXPECT: + raise + pytest.skip("%s has no media adapter in this version" % path) + + +@pytest.fixture +def auto_state(monkeypatch): + for name, value in (("_tracer_provider", None), ("_init_options", None), ("_instrumented", set()), + ("_adapter_status", {}), ("_attachment_manager_borrowed", False)): + monkeypatch.setattr(traceai, name, value) + monkeypatch.setenv("AGNO_TELEMETRY", "false") + for key in ("OPENINFERENCE_HIDE_INPUTS", "OPENINFERENCE_HIDE_INPUT_IMAGES", "OPENINFERENCE_HIDE_OUTPUTS"): + monkeypatch.delenv(key, raising=False) + yield + try: + from traceai_agno import AgnoInstrumentorWrapper + + AgnoInstrumentorWrapper().uninstrument() + except ImportError: + pass + for module, name in (("traceai_pydantic_ai", "PydanticAIInstrumentor"), ("traceai_openai", "OpenAIInstrumentor"), + ("traceai_langchain", "LangChainInstrumentor")): + try: + instrumentor = getattr(importlib.import_module(module), name)() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + except ImportError: + pass + + +def _init(f01, monkeypatch, libraries, enabled=True, **options): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + if enabled: + options["attachment_config"] = AttachmentConfig(enabled=True, endpoint=f01.endpoint) + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=libraries, + verbose=False, set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + return tracer_provider, exporter + + +def _exported(exporter): + return json.dumps([dict(span.attributes or {}) for span in exporter.get_finished_spans()], default=str) + + +# ------------------------------------------------------------------------------------- Agno +def _agno_run(provider): + pytest.importorskip("agno") + pytest.importorskip("openinference.instrumentation.agno") + from agno.agent import Agent + from agno.media import Image + from agno.models.openai import OpenAIChat + + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + try: + agent = Agent(model=model, telemetry=False) + except TypeError: # older Agno has no telemetry switch; AGNO_TELEMETRY=false covers it + agent = Agent(model=model) + agent.run(PROMPT, images=[Image(content=IMAGE, format="png")]) + + +def test_auto_agno_uploads_the_image_on_the_model_span(f01, provider, auto_state, monkeypatch): + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + row = {r["name"]: r for r in traceai.report()["integrations"]}["agno"] + assert (row["status"], row["media"]) == ("registered", "provider_adapter") + _agno_run(provider) + flush(manager) + assert f01.payloads == [IMAGE] + owners = [span.name for span in exporter.get_finished_spans() for _ in refs(span)] + assert len(owners) == 1 and owners[0].endswith((".invoke", ".ainvoke")), owners + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_agno_off_is_a_no_op(f01, provider, auto_state, monkeypatch): + pytest.importorskip("agno") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], enabled=False) + try: + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + _agno_run(provider) + assert f01.state["requests"] == [] + assert exporter.get_finished_spans() and all(not refs(s) for s in exporter.get_finished_spans()) + assert PROMPT in _exported(exporter) # tracing itself is unchanged + finally: + tracer_provider.shutdown() + + +def test_auto_agno_honours_auto_hide_flags(f01, provider, auto_state, monkeypatch): + """Auto's fi TraceConfig reaches Agno's OpenInference config: input text AND image stay out.""" + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], trace_config=TraceConfig(hide_inputs=True)) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + _agno_run(provider) + flush(manager) + assert f01.payloads == [] + assert exporter.get_finished_spans() + assert PROMPT not in _exported(exporter) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +# ------------------------------------------------------------------------------ Pydantic AI +def _pydantic_ai_run(): + pytest.importorskip("pydantic_ai") + from pydantic_ai import Agent, BinaryContent + from pydantic_ai.messages import ModelResponse, TextPart + from pydantic_ai.models.function import FunctionModel + + def echo(messages, info): + return ModelResponse(parts=[TextPart("ok")]) + + Agent(FunctionModel(echo)).run_sync([PROMPT, BinaryContent(data=IMAGE, media_type="image/png")]) + + +def _pydantic_ai_installed_by_name(): + """BaseInstrumentor requires the "pydantic-ai" distribution; pydantic-ai-slim alone fails its check.""" + from importlib.metadata import PackageNotFoundError, version + + try: + version("pydantic-ai") + return True + except PackageNotFoundError: + return False + + +def test_auto_pydantic_ai_uploads_the_image_or_reports_the_dependency_gap(f01, auto_state, monkeypatch): + pytest.importorskip("pydantic_ai") + _adapter("traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["pydantic-ai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + row = {r["name"]: r for r in traceai.report()["integrations"]}["pydantic_ai"] + if not _pydantic_ai_installed_by_name(): + # Accurate, not "registered": nothing was instrumented. + assert row["status"] == "dependency_unavailable" + _pydantic_ai_run() + flush(manager) + assert f01.payloads == [] and not exporter.get_finished_spans() + return + assert (row["status"], row["media"]) == ("registered", "provider_adapter") + _pydantic_ai_run() + flush(manager) + assert f01.payloads == [IMAGE] + assert len([s for s in exporter.get_finished_spans() if refs(s)]) == 1 + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_pydantic_ai_off_is_a_no_op(f01, auto_state, monkeypatch): + pytest.importorskip("pydantic_ai") + if not _pydantic_ai_installed_by_name(): + pytest.skip("pydantic-ai-slim only: auto cannot instrument it (dependency check)") + tracer_provider, exporter = _init(f01, monkeypatch, ["pydantic-ai"], enabled=False) + try: + _pydantic_ai_run() + assert f01.state["requests"] == [] + assert exporter.get_finished_spans() and all(not refs(s) for s in exporter.get_finished_spans()) + finally: + tracer_provider.shutdown() + + +# --------------------------------------------- framework + OpenAI through auto: one upload +def _is_openai_span(span): + return getattr(span.instrumentation_scope, "name", "") == "traceai_openai" + + +def _owners(exporter): + return [span for span in exporter.get_finished_spans() for _ in refs(span)] + + +def _sent(provider, image): + from conftest import b64 + + return [path for path, body, raw in provider.requests if b64(image).encode() in raw] + + +def _assert_one_upload_on_the_framework_span(f01, provider, manager, exporter, image): + flush(manager) + assert len(_sent(provider, image)) == 1, "the nested OpenAI SDK call did not send the image" + assert f01.payloads == [image], len(f01.payloads) + (owner,) = _owners(exporter) + assert not _is_openai_span(owner), owner.name + assert any(_is_openai_span(s) for s in exporter.get_finished_spans()), "no OpenAI span: not a 2-layer call" + + +def _openai_adapter(): + pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + + +@pytest.mark.parametrize("libraries", [["langchain", "openai"], ["openai", "langchain"]]) +def test_auto_langchain_over_openai_uploads_once(f01, provider, auto_state, monkeypatch, libraries): + _openai_adapter() + pytest.importorskip("langchain_openai") + _adapter("traceai_langchain._attachments", "langchain_attachment_capabilities") + from conftest import b64 + from langchain_core.messages import HumanMessage + from langchain_openai import ChatOpenAI + + tracer_provider, exporter = _init(f01, monkeypatch, libraries) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + rows = {r["name"]: r for r in traceai.report()["integrations"]} + assert [(rows[n]["status"], rows[n]["media"]) for n in ("langchain", "openai")] == [ + ("registered", "provider_adapter")] * 2 + image = png(seed=82) + model = ChatOpenAI(model="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + model.invoke([HumanMessage(content=[{"type": "text", "text": PROMPT}, { + "type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}])]) + _assert_one_upload_on_the_framework_span(f01, provider, manager, exporter, image) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +@pytest.mark.parametrize("libraries", [["agno", "openai"], ["openai", "agno"]]) +def test_auto_agno_over_openai_uploads_once(f01, provider, auto_state, monkeypatch, libraries): + _openai_adapter() + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, libraries) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + _agno_run(provider) + _assert_one_upload_on_the_framework_span(f01, provider, manager, exporter, IMAGE) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +# ------------------------------------------- Agno and a trace_config flag it cannot apply +@pytest.mark.parametrize("flag", ["pii_redaction", "hide_error_details"]) +def test_auto_agno_skips_media_for_a_flag_it_cannot_apply(f01, provider, auto_state, monkeypatch, flag): + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], trace_config=TraceConfig(**{flag: True})) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + row = {r["name"]: r for r in traceai.report()["integrations"]}["agno"] + assert (row["status"], row["media"]) == ("config_unsupported", "provider_adapter") + assert traceai.attachment_status()["adapters"]["agno"] == "config_unsupported" + _agno_run(provider) + flush(manager) + assert f01.payloads == [] and not _owners(exporter) + assert exporter.get_finished_spans() # Agno is still traced + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +@pytest.mark.parametrize("flag,env", [("pii_redaction", "FI_PII_REDACTION"), + ("hide_error_details", "FI_HIDE_ERROR_DETAILS")]) +def test_auto_agno_keeps_media_when_the_environment_sets_the_flag(f01, provider, auto_state, monkeypatch, flag, + env): + """Control: the same flag from the environment reaches Agno, so its media stays on.""" + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + monkeypatch.setenv(env, "true") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno"], trace_config=TraceConfig(**{flag: True})) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + _agno_run(provider) + flush(manager) + if flag == "pii_redaction": + assert f01.payloads == [] # the shared manager's PII policy denies media anyway + else: + assert f01.payloads == [IMAGE] and len(_owners(exporter)) == 1 + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_agno_unapplied_error_flag_leaves_the_call_to_openai(f01, provider, auto_state, monkeypatch): + """Agno without media marks nothing, so the OpenAI instrumentor (which applies the flag) owns the call.""" + _openai_adapter() + pytest.importorskip("agno") + _adapter("traceai_agno._attachments", "agno_attachment_capabilities") + tracer_provider, exporter = _init(f01, monkeypatch, ["agno", "openai"], + trace_config=TraceConfig(hide_error_details=True)) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + rows = {r["name"]: r for r in traceai.report()["integrations"]} + assert (rows["agno"]["status"], rows["openai"]["status"]) == ("config_unsupported", "registered") + _agno_run(provider) + flush(manager) + assert f01.payloads == [IMAGE] + (owner,) = _owners(exporter) + assert _is_openai_span(owner), owner.name + finally: + manager.shutdown(5) + tracer_provider.shutdown() diff --git a/python/tests/attachments_providers/test_auto_init.py b/python/tests/attachments_providers/test_auto_init.py index a9ba1d425..973537ae2 100644 --- a/python/tests/attachments_providers/test_auto_init.py +++ b/python/tests/attachments_providers/test_auto_init.py @@ -6,8 +6,10 @@ openai = pytest.importorskip("openai") pytest.importorskip("traceai_openai") -from conftest import b64, flush, png # noqa: E402 -from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from conftest import b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy # noqa: E402 +from opentelemetry import context as otel_context # noqa: E402 from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 @@ -18,8 +20,10 @@ def auto_state(monkeypatch): monkeypatch.setattr(traceai, "_init_options", None) monkeypatch.setattr(traceai, "_instrumented", set()) monkeypatch.setattr(traceai, "_adapter_status", {}) + monkeypatch.setattr(traceai, "_attachment_manager_borrowed", False) yield - for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_anthropic", "AnthropicInstrumentor")): + for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_anthropic", "AnthropicInstrumentor"), + ("traceai_groq", "GroqInstrumentor")): try: instrumentor = getattr(__import__(module), name)() if instrumentor.is_instrumented_by_opentelemetry: @@ -66,3 +70,176 @@ def test_auto_init_with_attachments_off_is_unchanged(provider, auto_state): assert OpenAIInstrumentor().is_instrumented_by_opentelemetry finally: tracer_provider.shutdown() + + +def _enabled_init(f01, monkeypatch, libraries, **options): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + if "attachment_manager" not in options: + options["attachment_config"] = AttachmentConfig(enabled=True, endpoint=f01.endpoint) + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=libraries, + verbose=False, set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + return tracer_provider, exporter + + +def _chat_image(provider, image): + openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0).chat.completions.create( + model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}]}]) + + +def _shot(data): + return {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": b64(data)}} + + +def test_auto_init_captures_openai_image_and_anthropic_computer_screenshot(f01, provider, auto_state, monkeypatch): + """A01 end to end: ONE manager from traceai.init reaches both instrumentors and the loopback F01.""" + anthropic = pytest.importorskip("anthropic") + pytest.importorskip("traceai_anthropic") + tracer_provider, exporter = _enabled_init(f01, monkeypatch, ["openai", "anthropic"]) + manager = traceai.get_attachment_manager() + try: + assert manager is tracer_provider.attachment_manager + assert manager.ready(5), manager.status() + image, screenshot = png(seed=51), png(seed=52) + _chat_image(provider, image) + anthropic.Anthropic(api_key="sk-synthetic", base_url=provider.url, max_retries=0).messages.create( + model="claude-test", max_tokens=16, + tools=[{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, + "display_height_px": 768}], + messages=[{"role": "user", "content": [{"type": "text", "text": "open the page"}]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_9", "name": "computer", "input": {"action": "screenshot"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_9", "content": [_shot(screenshot)]}]}]) + flush(manager) + assert sorted(f01.payloads) == sorted([image, screenshot]) + uploads = {u["purpose"]: u for u in f01.uploads} + assert uploads["computer_screenshot"].get("tool_call_id") == "toolu_9" + assert uploads["model_input"]["direction"] == "input" + span_refs = [ref for span in exporter.get_finished_spans() for ref in refs(span)] + assert sorted(ref["purpose"] for ref in span_refs) == ["computer_screenshot", "model_input"] + shot = [ref for ref in span_refs if ref["purpose"] == "computer_screenshot"][0] + assert (shot["message_index"], shot["part_index"], shot["role"]) == (2, 0, "tool") + exported = str([dict(span.attributes) for span in exporter.get_finished_spans()]) + assert b64(image) not in exported and b64(screenshot) not in exported + report = traceai.report() + assert report["attachments"]["state"] == "ready" + rows = {row["name"]: row for row in report["integrations"]} + for name in ("openai", "anthropic"): + assert (rows[name]["status"], rows[name]["media"]) == ("registered", "provider_adapter") + assert "tool_result.input" in rows["anthropic"]["media_operations"] + assert "SYNTHETIC" not in str(report) and str(f01.server.server_port) not in str(report) + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_report_rows_for_installed_media_adapters(auto_state): + pytest.importorskip("traceai_groq") + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert {name: rows[name]["media"] for name in ("openai", "anthropic", "groq")} == dict.fromkeys( + ("openai", "anthropic", "groq"), "provider_adapter") + # traceai_litellm is a media adapter only in a version that declares the descriptor, on a core with + # provider_media; otherwise (or when it is not installed) its row is unqualified. + try: + import traceai_litellm + + declared = list(traceai_litellm.litellm_attachment_capabilities["operations"]) + except Exception: + declared = [] + assert rows["litellm"]["media"] == ("provider_adapter" if declared else "unqualified") + + +def test_auto_borrowed_manager_is_shared_and_tightened(f01, provider, auto_state, monkeypatch): + from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider + + borrowed = f01.manager() + sdk = SdkTracerProvider() + try: + with sdk.get_tracer("synthetic").start_as_current_span("probe") as span: + ticket = borrowed.preflight(span, mime_type="image/png", direction="input", max_bytes=16) + assert ticket.admitted, ticket.reason + ticket.release() + tracer_provider, _ = _enabled_init(f01, monkeypatch, ["openai"], attachment_manager=borrowed, + trace_config=TraceConfig(pii_redaction=True)) + assert traceai.get_attachment_manager() is borrowed is tracer_provider.attachment_manager + assert openai.OpenAI.request._self_wrapper._attachments is borrowed + # Auto's PII flag now restricts the application's manager itself, for every caller: + # raw media under PII redaction needs a sanitizer, and none exists. + with sdk.get_tracer("synthetic").start_as_current_span("probe") as span: + assert borrowed.preflight(span, mime_type="image/png", direction="input", + max_bytes=16).reason == "redaction_unavailable" + _chat_image(provider, png(seed=53)) + flush(borrowed) + assert f01.payloads == [] + tracer_provider.shutdown() + # Auto never shuts a borrowed manager down. + assert borrowed.status().state == "ready" + finally: + borrowed.shutdown(5) + sdk.shutdown() + + +def test_auto_kill_switch_after_init_denies_every_admission(f01, provider, auto_state, monkeypatch): + tracer_provider, _ = _enabled_init(f01, monkeypatch, ["openai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5) + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + _chat_image(provider, png(seed=54)) + flush(manager) + assert f01.payloads == [] + assert traceai.report()["attachments"]["state"] == "disabled" + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED") + image = png(seed=55) + _chat_image(provider, image) + flush(manager) + assert f01.payloads == [image] + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_kill_switch_at_init_keeps_the_old_code_path(f01, provider, auto_state, monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + tracer_provider, _ = _enabled_init(f01, monkeypatch, ["openai"]) + try: + assert tracer_provider.attachment_manager.status().state == "disabled" + assert openai.OpenAI.request._self_wrapper._attachments is None + _chat_image(provider, png(seed=56)) + assert f01.state["requests"] == [] # not even a session bootstrap + finally: + tracer_provider.shutdown() + + +def test_auto_per_call_policy_and_ownership(f01, provider, auto_state, monkeypatch): + tracer_provider, exporter = _enabled_init(f01, monkeypatch, ["openai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5) + for context in (traceai.with_attachment_capture_policy(CapturePolicy(capture_content=False)), + traceai.with_attachment_capture_policy(CapturePolicy(muted=True)), + traceai.with_attachment_ownership()): + token = otel_context.attach(context) + try: + _chat_image(provider, png(seed=57)) + finally: + otel_context.detach(token) + flush(manager) + assert f01.payloads == [] + assert not any(refs(span) for span in exporter.get_finished_spans()) + image = png(seed=58) + token = otel_context.attach(traceai.with_attachment_capture_policy(CapturePolicy())) + try: + _chat_image(provider, image) + finally: + otel_context.detach(token) + flush(manager) + assert f01.payloads == [image] + finally: + manager.shutdown(5) + tracer_provider.shutdown() diff --git a/python/tests/attachments_providers/test_auto_provider_media.py b/python/tests/attachments_providers/test_auto_provider_media.py new file mode 100644 index 000000000..666844150 --- /dev/null +++ b/python/tests/attachments_providers/test_auto_provider_media.py @@ -0,0 +1,227 @@ +"""Auto with the provider-media (#697) and framework (#691) adapters: real SDKs, loopback F01. + +- report() rows equal each INSTALLED integration's own descriptor, and every integration + whose installed version declares none is unqualified; +- LiteLLM over the OpenAI instrumentor, both enabled through traceai.init, uploads the + call's image ONCE, on the LiteLLM span (LiteLLM's owner mark governs the nested call); +- with attachments OFF, the same call never reaches F01. + +Rows that need a core with ``provider_media`` skip on an older core unless +TRACEAI_EXPECT_PROVIDER_ADAPTERS=1, which turns the skip into a failure. +""" + +import importlib +import importlib.util +import os + +import pytest + +traceai = pytest.importorskip("traceai") + +from conftest import b64, flush, png, refs # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +EXPECT = os.environ.get("TRACEAI_EXPECT_PROVIDER_ADAPTERS") == "1" + +# report row name -> (instrumentor package, descriptor module, descriptor name). Independent of +# traceai._MEDIA_ADAPTERS on purpose: a wrong module or name there fails these rows. +PROVIDERS = { + "google_genai": ("traceai_google_genai", "traceai_google_genai", "google_genai_attachment_capabilities"), + "vertexai": ("traceai_vertexai", "traceai_vertexai", "vertexai_attachment_capabilities"), + "bedrock": ("traceai_bedrock", "traceai_bedrock", "bedrock_attachment_capabilities"), + "litellm": ("traceai_litellm", "traceai_litellm", "litellm_attachment_capabilities"), + "groq": ("traceai_groq", "traceai_groq._audio", "groq_attachment_capabilities"), + "fireworks": ("traceai_fireworks", "traceai_fireworks", "fireworks_attachment_capabilities"), + "xai": ("traceai_xai", "traceai_xai", "xai_attachment_capabilities"), + "mistralai": ("traceai_mistralai", "traceai_mistralai", "mistralai_attachment_capabilities"), + "openrouter": ("traceai_openrouter", "traceai_openrouter", "openrouter_attachment_capabilities"), + "portkey": ("traceai_portkey", "traceai_portkey", "portkey_attachment_capabilities"), + "ollama": ("traceai_ollama", "traceai_ollama", "ollama_attachment_capabilities"), + "vllm": ("traceai_vllm", "traceai_vllm", "vllm_attachment_capabilities"), + "minimax": ("traceai_minimax", "traceai_minimax", "minimax_attachment_capabilities"), +} +FRAMEWORKS = { + "agno": ("traceai_agno", "traceai_agno._attachments", "agno_attachment_capabilities"), + "pydantic_ai": ("traceai_pydantic_ai", "traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities"), + "langchain": ("traceai_langchain", "traceai_langchain._attachments", "langchain_attachment_capabilities"), + "llamaindex": ("traceai_llamaindex", "traceai_llamaindex._attachments", "llamaindex_attachment_capabilities"), +} +UNCLAIMED = {"together": "traceai_together", "deepseek": "traceai_deepseek", "cerebras": "traceai_cerebras"} + + +def _installed(module): + try: + return importlib.util.find_spec(module) is not None + except (ImportError, ValueError): + return False + + +def _declared(path, name): + """The installed version's own descriptor operations, or None when it declares none.""" + try: + return list(getattr(importlib.import_module(path), name)["operations"]) + except Exception: + return None + + +@pytest.fixture +def provider_media(): + try: + return importlib.import_module("fi_instrumentation.attachments.provider_media") + except ImportError: + if EXPECT: + raise + pytest.skip("this core has no provider_media") + + +@pytest.fixture +def auto_state(monkeypatch): + for name, value in (("_tracer_provider", None), ("_init_options", None), ("_instrumented", set()), + ("_adapter_status", {}), ("_attachment_manager_borrowed", False)): + monkeypatch.setattr(traceai, name, value) + yield + for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_litellm", "LiteLLMInstrumentor")): + try: + instrumentor = getattr(importlib.import_module(module), name)() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + except ImportError: + pass + + +def _rows(): + return {row["name"]: row for row in traceai.report()["integrations"]} + + +def test_report_rows_equal_each_installed_descriptor(provider_media, auto_state): + rows, checked = _rows(), [] + for name, (package, path, attr) in {**PROVIDERS, **FRAMEWORKS}.items(): + row = rows[name] + assert row["package"] == package + if not _installed(package): + assert (row["status"], row["media"]) == ("not_installed", "unqualified"), name + continue + declared = _declared(path, attr) + if not declared: + assert row["media"] == "unqualified" and "media_operations" not in row, name + continue + assert (row["media"], row["media_operations"]) == ("provider_adapter", declared), name + if name in PROVIDERS and name != "groq": + # The package's lazy descriptor IS the core's matrix row for it. + assert declared == list(provider_media.capabilities(package)["operations"]), name + checked.append(name) + for name, package in UNCLAIMED.items(): + assert rows[name]["media"] == "unqualified", name + assert list(provider_media.capabilities(package)["operations"]) == [], name + # A VM run names the rows it installed, so a silently missing package cannot pass. + expected = [n for n in os.environ.get("TRACEAI_EXPECT_MEDIA_ROWS", "").split(",") if n] + assert sorted(set(expected) - set(checked)) == [], checked + + +def test_rows_without_a_declared_descriptor_are_unqualified(auto_state): + """Runs on every core: an installed version that declares no descriptor is unqualified.""" + rows = _rows() + for name, (package, path, attr) in {**PROVIDERS, **FRAMEWORKS}.items(): + if _installed(package) and not _declared(path, attr): + assert rows[name]["media"] == "unqualified", name + for name in UNCLAIMED: + assert rows[name]["media"] == "unqualified", name + + +def _init(f01, monkeypatch, libraries, enabled=True): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + options = {"attachment_config": AttachmentConfig(enabled=True, endpoint=f01.endpoint)} if enabled else {} + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=libraries, + verbose=False, set_global_tracer_provider=False, **options) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + return tracer_provider, exporter + + +def _litellm_image(litellm, provider, image): + litellm.completion(model="openai/gpt-synthetic", api_base=provider.url + "/v1", api_key="sk-synthetic", + messages=[{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}]}]) + + +def _litellm(provider_media): + litellm = pytest.importorskip("litellm") + pytest.importorskip("traceai_openai") + traceai_litellm = pytest.importorskip("traceai_litellm") + if not _declared("traceai_litellm", "litellm_attachment_capabilities"): + if EXPECT: + pytest.fail("traceai_litellm declares no media descriptor") + pytest.skip("this traceai_litellm has no media adapter") + return litellm, traceai_litellm + + +def _sent_image(provider, image): + return [path for path, body, raw in provider.requests if b64(image).encode() in raw] + + +@pytest.mark.parametrize("libraries", [["openai", "litellm"], ["litellm", "openai"]]) +def test_auto_litellm_over_openai_uploads_once(f01, provider, provider_media, auto_state, monkeypatch, libraries): + litellm, _ = _litellm(provider_media) + tracer_provider, exporter = _init(f01, monkeypatch, libraries) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + rows = _rows() + assert (rows["litellm"]["status"], rows["litellm"]["media"]) == ("registered", "provider_adapter") + assert (rows["openai"]["status"], rows["openai"]["media"]) == ("registered", "provider_adapter") + image = png(seed=71) + _litellm_image(litellm, provider, image) + flush(manager) + assert len(_sent_image(provider, image)) == 1 # the nested OpenAI SDK call sent the image + assert f01.payloads == [image] + owners = [span.name for span in exporter.get_finished_spans() for _ in refs(span)] + assert owners == ["completion"], owners + # The OpenAI instrumentor traced the nested call, and captured nothing for it. + nested = [span for span in exporter.get_finished_spans() if span.name != "completion"] + assert nested and all(not refs(span) for span in nested), [s.name for s in exporter.get_finished_spans()] + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_control_openai_alone_captures_the_litellm_call(f01, provider, provider_media, auto_state, monkeypatch): + """Without the LiteLLM instrumentor, the OpenAI one owns the same bytes: the claim above matters.""" + litellm, _ = _litellm(provider_media) + tracer_provider, exporter = _init(f01, monkeypatch, ["openai"]) + manager = traceai.get_attachment_manager() + try: + assert manager.ready(5), manager.status() + image = png(seed=72) + _litellm_image(litellm, provider, image) + flush(manager) + assert f01.payloads == [image] + owners = [span.name for span in exporter.get_finished_spans() for _ in refs(span)] + assert len(owners) == 1 and owners != ["completion"], owners + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +@pytest.mark.parametrize("off", ["default", "kill_switch"]) +def test_auto_litellm_over_openai_off_never_reaches_f01(f01, provider, provider_media, auto_state, monkeypatch, + off): + litellm, _ = _litellm(provider_media) + if off == "kill_switch": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + tracer_provider, exporter = _init(f01, monkeypatch, ["openai", "litellm"], enabled=(off == "kill_switch")) + try: + assert traceai.report()["attachments"]["state"] == "disabled" + image = png(seed=73) + _litellm_image(litellm, provider, image) + assert len(_sent_image(provider, image)) == 1 + assert f01.state["requests"] == [] # not even a session bootstrap + assert all(not refs(span) for span in exporter.get_finished_spans()) + assert {span.name for span in exporter.get_finished_spans()} >= {"completion"} + finally: + tracer_provider.shutdown() diff --git a/python/traceai/README.md b/python/traceai/README.md index 500b77469..2e8c2d8f8 100644 --- a/python/traceai/README.md +++ b/python/traceai/README.md @@ -39,6 +39,56 @@ The helper does not install instrumentors or supply provider credentials. See [OpenAI setup](https://docs.futureagi.com/docs/integrations/traceai/openai) for a provider-specific example. +## Attachments (images, files, audio) + +Attachments are **off** by default; upgrading captures no new media. They need a +`fi-instrumentation-otel` release that includes `fi_instrumentation.attachments`. + +| Setting | Behavior | +|---|---| +| `attachment_config=AttachmentConfig(enabled=True)` or `FI_ATTACHMENTS_ENABLED=true` | The provider creates one attachment manager | +| `FI_ATTACHMENTS_ENABLED=false` | Kill switch: wins over code and denies every capture, even for a manager that is already running | +| `attachment_manager=` | Borrow your own manager instead. `traceai` never shuts it down and only adds the hide and PII flags of `trace_config` to it | +| `trace_config=TraceConfig(...)` | Hide flags and PII redaction also turn media capture off; nothing can relax them | + +`traceai` passes the same manager to every instrumentor it enables (as `attachments=`), +and `traceai.get_attachment_manager()` returns it. Use `with_attachment_capture_policy(policy)` +to add a `CapturePolicy` restriction for some calls, and `with_attachment_ownership()` to +mark calls whose media your own code captures, so instrumentors do not upload it again. +Both return an OpenTelemetry context to attach around those calls. + +MCP and agent tool media (screenshots, images, audio, files) uses the same manager: + +| Integration | How to enable it | +|---|---| +| `openai_agents`, `claude_agent_sdk`, `smolagents`, `firecrawl` | `auto_instrument=[...]` with the name | +| `mcp` | `session = traceai.wrap_mcp_client(session)` on your `mcp.ClientSession` (the `mcp` instrumentor only propagates context) | +| `mcp_use` | `MCPAgent(..., callbacks=[traceai.mcp_use_callback()])` (mcp-use has a callback, not an instrumentor) | + +Both helpers use the provider and hide flags from `traceai.init`, and pass the shared +manager only while capture is on. + +`traceai.report()` shows the attachment state and, per integration, `media`: +`provider_adapter` when the installed instrumentor version captures media, with its +`media_operations`; `session_binding` for voice integrations that capture only through an +explicit `bind_*` call; otherwise `unqualified`. It contains no endpoint or credential and +is not a playback claim. A row can be `provider_adapter` for: + +| Kind | Packages | +|---|---| +| Providers | `openai`, `anthropic`, `groq`, `google_genai`, `vertexai`, `bedrock`, `litellm`, `fireworks`, `xai`, `mistralai`, `openrouter`, `portkey`, `ollama`, `vllm`, `minimax` | +| Frameworks | `langchain`, `llamaindex`, `agno`, `pydantic_ai` | +| MCP and agents | the six packages above | + +An installed version without the adapter reports `unqualified`. `together`, `deepseek` and +`cerebras` capture no media. + +Agno reads `pii_redaction` and `hide_error_details` only from `FI_PII_REDACTION` and +`FI_HIDE_ERROR_DETAILS`. When `trace_config` sets either one in code and the environment does +not, `traceai` still traces Agno but captures no Agno media, and its status is +`config_unsupported`. Agno span text then does not follow that flag; set the environment +variable to apply it. + ## License Apache-2.0. diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 4888b539d..c94119c14 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -11,7 +11,9 @@ response = client.chat.completions.create(...) """ +import dataclasses import importlib +import importlib.util import threading from typing import Any, Dict, List, Optional @@ -29,15 +31,63 @@ # core, importing traceai and plain init() keep working; asking for attachments # raises a clear error instead. try: - from fi_instrumentation.attachments import AttachmentConfig + from fi_instrumentation.attachments import AttachmentConfig, DisabledAttachmentManager + from fi_instrumentation.attachments.policy import env_disabled as _env_disabled _ATTACHMENTS_SUPPORTED = True except ImportError: # pragma: no cover - exercised in a subprocess test AttachmentConfig = None + DisabledAttachmentManager = None _ATTACHMENTS_SUPPORTED = False -_CORE_TOO_OLD = ("attachment_config and trace_config need a fi-instrumentation-otel release that " - "includes fi_instrumentation.attachments; upgrade fi-instrumentation-otel") + def _env_disabled() -> bool: + return True + +# Context keys read by the shared provider-media helper (and any adapter) BY NAME, so +# setting them never needs the attachments package. They match +# fi_instrumentation.attachments.media_capture.CAPTURE_POLICY_KEY / OWNER_KEY and the +# TypeScript `traceai_auto` keys (`createContextKey` names) exactly. +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" +_OWNER_KEY = "traceai.auto.attachment_owner.v1" + +_CORE_TOO_OLD = ("attachment_config, attachment_manager and trace_config need a fi-instrumentation-otel " + "release that includes fi_instrumentation.attachments; upgrade fi-instrumentation-otel") + +# Instrumentors that capture media through the shared manager they receive as +# ``attachments``: (module, descriptor). A descriptor is only present in an installed +# instrumentor version that has the adapter, so the report is version aware. +_MEDIA_ADAPTERS = { + "traceai_openai": ("traceai_openai", "openai_attachment_capabilities"), + "traceai_anthropic": ("traceai_anthropic", "anthropic_attachment_capabilities"), + "traceai_groq": ("traceai_groq._audio", "groq_attachment_capabilities"), + # MCP and agent tool media. MCP and mcp-use capture only through their explicit entry + # points (``wrap_mcp_client`` and ``mcp_use_callback`` below): the MCP instrumentor only + # propagates context, and mcp-use has a callback, not an instrumentor. + "traceai_mcp": ("traceai_mcp", "mcp_attachment_capabilities"), + "traceai_mcp_use": ("traceai_mcp_use._attachments", "mcp_use_attachment_capabilities"), + "traceai_openai_agents": ("traceai_openai_agents._attachments", "openai_agents_attachment_capabilities"), + "traceai_claude_agent_sdk": ("traceai_claude_agent_sdk._attachments", "claude_agent_attachment_capabilities"), + "traceai_smolagents": ("traceai_smolagents._attachments", "smolagents_attachment_capabilities"), + "traceai_firecrawl": ("traceai_firecrawl._attachments", "firecrawl_attachment_capabilities"), + # Provider media (shared provider_media walkers). Each package resolves its descriptor lazily from + # fi_instrumentation.attachments.provider_media, so an installed version without the binding, or a core + # without provider_media, reports unqualified. together, deepseek and cerebras claim no media operation + # and are deliberately absent (their rows stay unqualified). + **{module: (module, module[len("traceai_"):] + "_attachment_capabilities") for module in ( + "traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_litellm", "traceai_fireworks", + "traceai_xai", "traceai_mistralai", "traceai_openrouter", "traceai_portkey", "traceai_ollama", + "traceai_vllm", "traceai_minimax")}, + # Framework media. + "traceai_langchain": ("traceai_langchain._attachments", "langchain_attachment_capabilities"), + "traceai_llamaindex": ("traceai_llamaindex._attachments", "llamaindex_attachment_capabilities"), + "traceai_agno": ("traceai_agno._attachments", "agno_attachment_capabilities"), + "traceai_pydantic_ai": ("traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities"), +} +# Voice integrations capture only through an explicit session binding (bind_* below). +_SESSION_BINDINGS = {"traceai_pipecat": "bind_audio_processor", "traceai_livekit": "bind_session"} +# Integrations with no Instrumentor: enabling one only records it, and the application +# connects it explicitly (a pipeline processor, a live session, a LangChain callback). +_EXPLICIT_ONLY = frozenset(("traceai_pipecat", "traceai_livekit", "traceai_mcp_use")) # Registry of available instrumentors _INSTRUMENTOR_REGISTRY: Dict[str, str] = { @@ -60,6 +110,10 @@ "ollama": "traceai_ollama", "google_genai": "traceai_google_genai", "google-genai": "traceai_google_genai", + "openrouter": "traceai_openrouter", + "portkey": "traceai_portkey", + "vllm": "traceai_vllm", + "minimax": "traceai_minimax", # Framework instrumentors "langchain": "traceai_langchain", "llamaindex": "traceai_llamaindex", @@ -72,6 +126,18 @@ "instructor": "traceai_instructor", "guardrails": "traceai_guardrails", "mcp": "traceai_mcp", + "mcp_use": "traceai_mcp_use", + "mcp-use": "traceai_mcp_use", + "agno": "traceai_agno", + "pydantic_ai": "traceai_pydantic_ai", + "pydantic-ai": "traceai_pydantic_ai", + # Agent frameworks and tools + "openai_agents": "traceai_openai_agents", + "openai-agents": "traceai_openai_agents", + "claude_agent_sdk": "traceai_claude_agent_sdk", + "claude-agent-sdk": "traceai_claude_agent_sdk", + "smolagents": "traceai_smolagents", + "firecrawl": "traceai_firecrawl", "pipecat": "traceai_pipecat", "livekit": "traceai_livekit", } @@ -90,6 +156,13 @@ "traceai_autogen": "AutogenInstrumentor", "traceai_dspy": "DSPyInstrumentor", "traceai_haystack": "HaystackInstrumentor", "traceai_instructor": "InstructorInstrumentor", "traceai_guardrails": "GuardrailsInstrumentor", "traceai_mcp": "MCPInstrumentor", + "traceai_openai_agents": "OpenAIAgentsInstrumentor", "traceai_claude_agent_sdk": "ClaudeAgentInstrumentor", + "traceai_smolagents": "SmolagentsInstrumentor", "traceai_firecrawl": "FirecrawlInstrumentor", + "traceai_openrouter": "OpenRouterInstrumentor", "traceai_portkey": "PortkeyInstrumentor", + "traceai_vllm": "VLLMInstrumentor", "traceai_minimax": "MiniMaxInstrumentor", + "traceai_pydantic_ai": "PydanticAIInstrumentor", + # A process-wide wrapper over OpenInference Agno, with its own keyword arguments (_agno_kwargs). + "traceai_agno": "AgnoInstrumentorWrapper", } _tracer_provider: Optional[TracerProvider] = None @@ -97,6 +170,9 @@ _init_lock = threading.RLock() _instrumented = set() _adapter_status = {} +# Whether the application lent the provider's attachment manager (auto never shuts a +# borrowed manager down). +_attachment_manager_borrowed = False def init( @@ -112,6 +188,7 @@ def init( set_global_tracer_provider: bool = True, attachment_config: Optional["AttachmentConfig"] = None, trace_config: Optional[TraceConfig] = None, + attachment_manager: Any = None, ) -> TracerProvider: """ Initialize TraceAI with a simple one-liner. @@ -129,6 +206,15 @@ def init( batch: Use batch span processor instead of simple (better for production) verbose: Print configuration details set_global_tracer_provider: Set as global OpenTelemetry tracer provider + attachment_config: Attachment options for the manager the provider creates. + Attachments are OFF unless enabled here or with FI_ATTACHMENTS_ENABLED=true. + trace_config: Hide and PII flags; they also restrict attachment capture. + attachment_manager: Borrow the application's AttachmentManager instead. Auto + never shuts it down and only adds trace_config's hide/PII restrictions to it. + Cannot be combined with attachment_config. + + Every instrumentor auto enables receives the same manager (``attachments=``); + ``get_attachment_manager()`` returns it and ``report()`` describes it. Returns: TracerProvider: The configured tracer provider @@ -161,7 +247,7 @@ def init( verbose=False, ) """ - global _tracer_provider, _init_options + global _tracer_provider, _init_options, _attachment_manager_borrowed # Convert string enums to enum types project_type_enum = ( @@ -171,13 +257,21 @@ def init( semantic_convention_enum = SemanticConvention(semantic_convention.lower()) transport_enum = Transport(transport.lower()) - if not _ATTACHMENTS_SUPPORTED and (attachment_config is not None or trace_config is not None): + if not _ATTACHMENTS_SUPPORTED and (attachment_config is not None or trace_config is not None + or attachment_manager is not None): raise RuntimeError(_CORE_TOO_OLD) + if attachment_manager is not None and attachment_config is not None: + raise ValueError("pass either attachment_config or attachment_manager, not both") # An older core's register() accepts neither keyword: pass them only when supported. attachment_kwargs = (dict(attachment_config=attachment_config, trace_config=trace_config) if _ATTACHMENTS_SUPPORTED else {}) + if attachment_manager is not None: + # The provider must not create a second manager (FI_ATTACHMENTS_ENABLED=true + # would otherwise start one with its own threads). + attachment_kwargs["attachment_config"] = AttachmentConfig(enabled=False) options = (project_name, project_type, project_version_name, semantic_convention, - transport, batch, set_global_tracer_provider, attachment_config, trace_config) + transport, batch, set_global_tracer_provider, attachment_config, trace_config, + id(attachment_manager) if attachment_manager is not None else None) with _init_lock: if _tracer_provider is not None and _init_options != options: raise ValueError("traceai is already initialized with different configuration") @@ -195,8 +289,14 @@ def init( semantic_convention=semantic_convention_enum, **attachment_kwargs, ) + if attachment_manager is not None: + _restrict_borrowed(attachment_manager, getattr(provider, "trace_config", None)) + # The provider holds the ONE manager, so get_attachment_manager(), the + # instrumentors and the bind_* helpers all use the borrowed one. + provider.attachment_manager = attachment_manager _tracer_provider = provider _init_options = options + _attachment_manager_borrowed = attachment_manager is not None _instrumented.clear() _adapter_status.clear() @@ -232,9 +332,10 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: # Import the instrumentor package module = importlib.import_module(instrumentor_module) - if lib_name_lower in ("pipecat", "livekit"): - # These use native/session integrations, not an Instrumentor. Auto - # cannot insert a pipeline processor or guess a live job/session. + if instrumentor_module in _EXPLICIT_ONLY: + # These use native/session/callback integrations, not an Instrumentor. Auto + # cannot insert a pipeline processor, guess a live job/session or attach a + # LangChain callback (``mcp_use_callback()`` builds one on the shared manager). _adapter_status[lib_name_lower] = "registered" return True with _init_lock: @@ -244,12 +345,35 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: instrumentor = instrumentor_class() provider = _tracer_provider kwargs = dict(tracer_provider=provider, config=getattr(provider, "trace_config", None)) - if _ATTACHMENTS_SUPPORTED: - manager = getattr(provider, "attachment_manager", None) - kwargs.update(attachment_manager=manager, attachment_policy=getattr(manager, "policy", None)) + status = "registered" + if instrumentor_module == "traceai_agno": + kwargs, unreachable = _agno_kwargs(provider) + if unreachable: + # Agno is traced, but cannot apply these flags and captures no media. + status = "config_unsupported" + if verbose: + print(f"Warning: {lib_name} cannot apply trace_config {', '.join(unreachable)} " + "set in code (it reads them only from FI_* environment variables); " + "its media capture is off and its span text does not follow them") + elif _ATTACHMENTS_SUPPORTED: + manager = get_attachment_manager() + if _capturing(manager): + # The ONE shared manager, as ``attachments`` (``attachment_manager`` is the + # alias the Python instrumentors also accept). + kwargs.update(attachments=manager, attachment_manager=manager) + else: + # Attachments OFF: exactly the keyword arguments auto passed before. + kwargs.update(attachment_manager=manager, attachment_policy=getattr(manager, "policy", None)) instrumentor.instrument(**kwargs) + if getattr(instrumentor, "is_instrumented_by_opentelemetry", True) is False: + # BaseInstrumentor returns without instrumenting on a dependency conflict (for + # example pydantic-ai-slim for a "pydantic-ai" requirement): never report it registered. + _adapter_status[lib_name_lower] = "dependency_unavailable" + if verbose: + print(f"Warning: {lib_name} was not instrumented (dependency check failed)") + return False _instrumented.add(instrumentor_module) - _adapter_status[lib_name_lower] = "registered" + _adapter_status[lib_name_lower] = status if verbose: print(f"| Auto-instrumented: {lib_name}") @@ -268,11 +392,132 @@ def _instrument_library(lib_name: str, verbose: bool = True) -> bool: return False +# FI-only TraceConfig flags. Agno's privacy bridge builds its FI config from the environment and +# takes only the hide_* flags from the OpenInference config auto passes, so these reach Agno only +# through FI_PII_REDACTION / FI_HIDE_ERROR_DETAILS. +_AGNO_ENV_ONLY = ("pii_redaction", "hide_error_details") + + +def _agno_privacy(trace_config): + """``(hide flags for Agno's OpenInference config or None, FI-only flags set on auto that Agno cannot apply)``. + + What Agno applies is its environment policy plus the forwarded hide flags (they also derive + ``hide_error_details``), so a flag counts as unreachable only when that would not set it. + """ + hidden = [field.name for field in dataclasses.fields(TraceConfig) + if field.name.startswith("hide_") and getattr(trace_config, field.name, False) is True] + forwarded: Optional[Dict[str, bool]] = None + if hidden: + from openinference.instrumentation import TraceConfig as OpenInferenceTraceConfig + + names = {field.name for field in dataclasses.fields(OpenInferenceTraceConfig)} + forwarded = {name: True for name in hidden if name in names} + wanted = [name for name in _AGNO_ENV_ONLY if getattr(trace_config, name, False) is True] + if not wanted: + return forwarded, [] + try: + reach = TraceConfig(**(forwarded or {})) + except Exception: + reach = None # Unknown: treat every wanted flag as unreachable. + return forwarded, [name for name in wanted if getattr(reach, name, False) is not True] + + +def _agno_kwargs(provider): + """``(keyword arguments for AgnoInstrumentorWrapper.instrument, unreachable FI-only flags)``. + + It takes OpenInference's TraceConfig (not fi's) and no ``attachment_*`` alias. Auto's + hide flags are carried over, and only when one is set, so an unrestricted setup keeps + Agno's own default and environment policy. ``attachments`` is passed only while capturing, + only to a version that has the media adapter, and never while auto's trace_config sets + ``pii_redaction`` or ``hide_error_details`` that Agno cannot apply (fail closed). + """ + kwargs: Dict[str, Any] = {"tracer_provider": provider} + forwarded, unreachable = _agno_privacy(getattr(provider, "trace_config", None)) + if forwarded is not None: + from openinference.instrumentation import TraceConfig as OpenInferenceTraceConfig + + kwargs["config"] = OpenInferenceTraceConfig(**forwarded) + if _ATTACHMENTS_SUPPORTED and not unreachable: + manager = get_attachment_manager() + # Only a traceai-agno version with the media adapter (it declares the descriptor) accepts + # ``attachments``; an older one keeps tracing with the OFF arguments. + if _capturing(manager) and _media_support("traceai_agno")["media"] == "provider_adapter": + kwargs["attachments"] = manager + return kwargs, unreachable + + def get_tracer_provider() -> Optional[TracerProvider]: """Get the current tracer provider, if initialized.""" return _tracer_provider +def get_attachment_manager(): + """The one attachment manager auto passes to instrumentors, or None before init(). + + It is the borrowed ``attachment_manager``, else the one the provider created + (the disabled null object while attachments are OFF). Call ``ready()`` on it + to wait for the attachment session at startup. + """ + return getattr(_tracer_provider, "attachment_manager", None) + + +def _capturing(manager) -> bool: + """A manager that can admit media now: enabled, and not killed by FI_ATTACHMENTS_ENABLED.""" + return (manager is not None and DisabledAttachmentManager is not None + and not isinstance(manager, DisabledAttachmentManager) + and callable(getattr(manager, "preflight", None)) and not _env_disabled()) + + +_RESTRICTIONS = ("hide_inputs", "hide_outputs", "hide_input_messages", "hide_output_messages", + "hide_input_images", "hide_input_text", "hide_output_text", "pii_redaction") + + +def _restrict_borrowed(manager, trace_config) -> None: + """Add auto's hide/PII flags to a borrowed manager's policy. Never removes one.""" + policy = getattr(manager, "policy", None) + current = getattr(policy, "trace_config", None) + if policy is None or current is None or trace_config is None: + return + added = {name: True for name in _RESTRICTIONS + if getattr(trace_config, name, False) and not getattr(current, name, False)} + if getattr(trace_config, "hide_error_details", False) and not getattr(current, "hide_error_details", False): + added["hide_error_details"] = True + if added: + # TraceConfig is frozen: replace it whole; every flag already set stays set. + policy.trace_config = dataclasses.replace(current, **added) + + +def with_attachment_capture_policy(policy, context=None): + """Return a context whose calls capture media under ``policy`` as well. + + ``policy`` is a ``fi_instrumentation.attachments.CapturePolicy``. It can only + restrict: it is intersected with any policy already in the context and with every + other layer (server, credential, hide/PII flags, modality). Anything that is not a + ``CapturePolicy`` denies capture. Attach the result with + ``opentelemetry.context.attach`` around the calls it covers. + """ + from opentelemetry import context as otel_context + + try: + from fi_instrumentation.attachments.media_capture import merge_capture_policies + except ImportError: + merged = policy + else: + merged = merge_capture_policies(otel_context.get_value(_CAPTURE_POLICY_KEY, context), policy) + return otel_context.set_value(_CAPTURE_POLICY_KEY, merged, context) + + +def with_attachment_ownership(context=None): + """Return a context marking its calls' media as captured by your own adapter. + + Instrumentors called inside it capture no media for those calls, so the same + media is not uploaded twice. Mark only the downstream call, not your own capture. + """ + from opentelemetry import context as otel_context + + return otel_context.set_value(_OWNER_KEY, True, context) + + def instrument(*libraries: str, verbose: bool = True) -> None: """ Instrument additional libraries after init(). @@ -290,13 +535,121 @@ def instrument(*libraries: str, verbose: bool = True) -> None: _instrument_library(lib_name, verbose=verbose) +_AUTO_SOURCES = ( + "inline base64 and data-URI images, PDFs and files in OpenAI chat and Responses requests, " + "including computer_call_output screenshots (traceai_openai)", + "OpenAI images.generate b64_json and completed Responses image_generation_call output (traceai_openai)", + "OpenAI and Groq speech the SDK buffered, and OpenAI speech the application consumed", + "OpenAI and Groq transcription/translation audio the SDK holds as bytes", + "Anthropic Messages base64 images and PDFs, and computer-use tool_result screenshots (traceai_anthropic)", + "Pipecat and LiveKit recordings, only through an explicit bind_* call", + "for every other integration, exactly the media_operations of its provider_adapter row below " + "(the installed instrumentor version decides)", +) +_AUTO_UNSUPPORTED = ( + "integrations whose media row is 'unqualified'", + "remote URLs and provider file ids (never fetched)", + "open file objects and file paths the SDK does not read itself", + "partial or streamed images, OpenAI images.edit and variations", + "Anthropic beta.messages and generated output media", + "HTTP wire capture (the Python auto module has no wire layer)", + "new recordings without an explicit voice binding", + "product playback qualification", +) + + +def _attachment_report() -> dict: + manager = get_attachment_manager() + if not _ATTACHMENTS_SUPPORTED: + state = "unsupported_version" + elif manager is None: + state = "not_initialized" + elif _env_disabled(): + # The kill switch denies at every admission even for an enabled manager, + # whose own status() does not consult it. + state = "disabled" + else: + try: + status = manager.status() + state = status.state + except Exception: + status, state = None, "unavailable" + report = {"state": state, "transport": "shared_manager", "borrowed": _attachment_manager_borrowed, + "sources": list(_AUTO_SOURCES), "unsupported": list(_AUTO_UNSUPPORTED)} + if state not in ("unsupported_version", "not_initialized", "disabled", "unavailable"): + report["status"] = {name: getattr(status, name) for name in + ("queued", "active", "tickets", "streams", "recordings", "owned_bytes", "dropped") + if isinstance(getattr(status, name, None), int)} + return report + + +def _installed(module: str) -> bool: + try: + return importlib.util.find_spec(module) is not None + except (ImportError, ValueError): + return False + + +def _media_support(module: str) -> Dict[str, Any]: + """Whether the INSTALLED instrumentor version captures media; imports, never instruments.""" + if not _ATTACHMENTS_SUPPORTED or not _installed(module): + return {"media": "unqualified"} + try: + if module in _MEDIA_ADAPTERS: + path, name = _MEDIA_ADAPTERS[module] + descriptor = getattr(importlib.import_module(path), name) + operations = list(descriptor["operations"]) + if operations: + return {"media": "provider_adapter", "media_operations": operations} + # A descriptor that claims no operation (tracing only) is not a media adapter. + return {"media": "unqualified"} + if module in _SESSION_BINDINGS and callable(getattr(importlib.import_module(module), + _SESSION_BINDINGS[module], None)): + return {"media": "session_binding"} + except Exception: + # An older instrumentor or core: no descriptor means no media capture. + pass + return {"media": "unqualified"} + + +def report() -> dict: + """Content-free setup report: attachment state and per-integration media support. + + ``integrations[].media`` is ``provider_adapter`` when the installed instrumentor + captures media through the shared manager, ``session_binding`` for voice + integrations that capture only through an explicit ``bind_*`` call, otherwise + ``unqualified``. It describes capability, not that any media was captured, and + never a playback claim. No endpoint, header or credential is included. + + ``integrations[].status`` is ``registered`` once enabled, ``config_unsupported`` for an + Agno traced without media because auto's trace_config sets ``pii_redaction`` or + ``hide_error_details`` it cannot apply, ``dependency_unavailable``, ``unsupported_version``, + else ``not_enabled`` or ``not_installed``. + """ + names = {} + for name, module in _INSTRUMENTOR_REGISTRY.items(): + names.setdefault(module, name) + integrations = [] + for module, name in names.items(): + aliases = [alias for alias, target in _INSTRUMENTOR_REGISTRY.items() if target == module] + status = next((_adapter_status[a] for a in aliases if a in _adapter_status), None) + if status is None: + status = "not_enabled" if _installed(module) else "not_installed" + integrations.append({"name": name, "package": module, "status": status, **_media_support(module)}) + return {"initialized": _tracer_provider is not None, "attachments": _attachment_report(), + "integrations": integrations} + + def attachment_status() -> dict: """Content-free setup status, never an integrated recording-ready claim.""" if not _ATTACHMENTS_SUPPORTED: return {"runtime": "unsupported_version", "adapters": dict(_adapter_status)} - manager = getattr(_tracer_provider, "attachment_manager", None) - return {"runtime": manager.status().state if manager is not None else "uninitialized", - "adapters": dict(_adapter_status)} + manager = get_attachment_manager() + if manager is None: + runtime = "uninitialized" + else: + runtime = "disabled" if _env_disabled() else manager.status().state + return {"runtime": runtime, "adapters": dict(_adapter_status)} def _bind(library: str, entry_point: str, *args, **kwargs): @@ -323,6 +676,56 @@ def _bind(library: str, entry_point: str, *args, **kwargs): return binding +def _explicit_kwargs(library: str, kwargs: dict, provider_key: str, config_key: str) -> dict: + """Defaults for an explicit entry point: auto's provider, its trace config and, only + while capture is on and the installed package has the media adapter, the ONE manager.""" + if _tracer_provider is None: + raise RuntimeError("initialize traceai before connecting " + library) + kwargs.setdefault(provider_key, _tracer_provider) + config = getattr(_tracer_provider, "trace_config", None) + if config is not None: + # Auto's hide and PII flags reach the integration; the caller can pass its own. + kwargs.setdefault(config_key, config) + if _ATTACHMENTS_SUPPORTED: + manager = get_attachment_manager() + module = _INSTRUMENTOR_REGISTRY[library] + if _capturing(manager) and _media_support(module)["media"] == "provider_adapter": + kwargs.setdefault("attachments", manager) + return kwargs + + +def mcp_use_callback(**kwargs): + """A ``traceai_mcp_use.FutureAGICallback`` on auto's provider and shared manager. + + Pass it to the agent: ``MCPAgent(..., callbacks=[traceai.mcp_use_callback()])``. + With attachments off, or an installed ``traceAI-mcp-use`` without the media adapter, + it is the same callback as before (no ``attachments=`` argument). Keyword arguments + are passed through and win over auto's defaults. + """ + module = importlib.import_module(_INSTRUMENTOR_REGISTRY["mcp_use"]) + callback = module.FutureAGICallback(**_explicit_kwargs("mcp_use", kwargs, "tracer_provider", "config")) + _adapter_status["mcp_use"] = "registered" + return callback + + +def wrap_mcp_client(session, **kwargs): + """Trace an ``mcp.ClientSession`` with ``traceai_mcp.wrap_mcp_client`` on auto's provider + and shared manager, so tool and resource media reach Future AGI. + + The MCP instrumentor only propagates context; this is the entry point that captures + media. An installed ``traceAI-mcp`` without ``wrap_mcp_client`` returns the session + unchanged and reports ``unsupported_version``. + """ + module = importlib.import_module(_INSTRUMENTOR_REGISTRY["mcp"]) + wrap = getattr(module, "wrap_mcp_client", None) + if wrap is None: + _adapter_status["mcp"] = "unsupported_version" + return session + wrapped = wrap(session, **_explicit_kwargs("mcp", kwargs, "tracer_provider", "trace_config")) + _adapter_status.setdefault("mcp", "registered") + return wrapped + + def bind_pipecat_audio_processor(processor, session_context): """Observe an existing processor; never start a new recording implicitly.""" return _bind("pipecat", "bind_audio_processor", processor, session_context) @@ -340,8 +743,14 @@ def bind_livekit_session(session, job_context, *, owned_recording_files=False, "init", "AttachmentConfig", "attachment_status", + "get_attachment_manager", + "report", + "with_attachment_capture_policy", + "with_attachment_ownership", "bind_pipecat_audio_processor", "bind_livekit_session", + "mcp_use_callback", + "wrap_mcp_client", "instrument", "get_tracer_provider", "SemanticConvention", diff --git a/python/traceai/tests/conftest.py b/python/traceai/tests/conftest.py index b53fd44e1..93d76b235 100644 --- a/python/traceai/tests/conftest.py +++ b/python/traceai/tests/conftest.py @@ -9,3 +9,4 @@ def isolated_auto_state(monkeypatch): monkeypatch.setattr(traceai, "_init_options", None) monkeypatch.setattr(traceai, "_instrumented", set()) monkeypatch.setattr(traceai, "_adapter_status", {}) + monkeypatch.setattr(traceai, "_attachment_manager_borrowed", False) diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index b4e18e306..a6369840c 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -94,7 +94,19 @@ def old_register(*, project_name=None, project_type=None, project_version_name=N assert calls == ["synthetic"], calls assert traceai.bind_livekit_session(object(), object()) is None assert traceai.attachment_status()["adapters"]["livekit"] == "unsupported_version" +assert traceai.get_attachment_manager() is None +assert traceai.report()["attachments"]["state"] == "unsupported_version" +assert all(row["media"] == "unqualified" for row in traceai.report()["integrations"]) +from opentelemetry import context as otel_context +marked = traceai.with_attachment_ownership() +assert otel_context.get_value("traceai.auto.attachment_owner.v1", marked) is True traceai._tracer_provider = None +try: + traceai.init(project_name="other", verbose=False, attachment_manager=object()) +except RuntimeError as error: + assert "fi-instrumentation-otel" in str(error) +else: + raise AssertionError("attachment_manager accepted on a core without attachments") try: traceai.init(project_name="other", verbose=False, trace_config=object()) except RuntimeError as error: @@ -117,3 +129,616 @@ def test_import_and_init_on_core_without_attachments(): timeout=120, env=env) assert result.returncode == 0, result.stderr[-2000:] assert "OLD_CORE_OK" in result.stdout + + +# ----------------------------------------------------------- shared manager (A01) +import dataclasses # noqa: E402 + +import pytest # noqa: E402 +from fi_instrumentation.attachments import AttachmentManager, CapturePolicy, DisabledAttachmentManager # noqa: E402 +from fi_instrumentation.attachments import media_capture # noqa: E402 +from opentelemetry import context as otel_context # noqa: E402 + + +def _enabled_like_manager(): + """Stands in for an enabled manager: not the null object, and it can preflight.""" + return SimpleNamespace(policy=object(), preflight=Mock(), status=lambda: SimpleNamespace(state="ready")) + + +def _install(monkeypatch, manager, config=None): + provider = SimpleNamespace(attachment_manager=manager, trace_config=config or TraceConfig()) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + constructor = Mock() + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(OpenAIInstrumentor=constructor))) + return provider, constructor + + +def test_enabled_manager_reaches_every_instrumentor_as_attachments(monkeypatch): + manager = _enabled_like_manager() + provider, constructor = _install(monkeypatch, manager) + assert traceai.get_attachment_manager() is None + traceai.init(project_name="synthetic", auto_instrument=["openai"], verbose=False) + assert traceai.get_attachment_manager() is manager + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachments=manager, attachment_manager=manager) + + +def test_kill_switch_keeps_the_off_path_and_reports_disabled(monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = _enabled_like_manager() + provider, constructor = _install(monkeypatch, manager) + traceai.init(project_name="synthetic", auto_instrument=["openai"], verbose=False) + # Exactly the keyword arguments auto passed before this change, with attachments OFF. + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachment_manager=manager, + attachment_policy=manager.policy) + assert traceai.report()["attachments"]["state"] == "disabled" + assert "status" not in traceai.report()["attachments"] + assert traceai.attachment_status()["runtime"] == "disabled" + + +def test_borrowed_manager_gains_only_restrictions(monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "true") + app_config = TraceConfig(hide_inputs=True, pii_redaction=False) + borrowed = AttachmentManager(config=AttachmentConfig(enabled=True), trace_config=app_config) # no scope: no thread + register = Mock(return_value=SimpleNamespace(attachment_manager=DisabledAttachmentManager(), + trace_config=TraceConfig(hide_inputs=False, pii_redaction=True))) + monkeypatch.setattr(traceai, "register", register) + try: + traceai.init(project_name="synthetic", verbose=False, attachment_manager=borrowed, + trace_config=TraceConfig(hide_inputs=False, pii_redaction=True)) + # The provider does not build a second (env-enabled) manager. + assert register.call_args.kwargs["attachment_config"].enabled is False + assert traceai.get_attachment_manager() is borrowed + assert traceai.report()["attachments"]["borrowed"] is True + restricted = borrowed.policy.trace_config + assert restricted.pii_redaction is True # auto's restriction reached the borrowed manager + assert restricted.hide_inputs is True # the application's own flag survived auto's False + assert app_config.pii_redaction is False # the application's object is not mutated + finally: + borrowed.shutdown(0) + + +def test_borrowed_manager_and_attachment_config_are_exclusive(monkeypatch): + monkeypatch.setattr(traceai, "register", Mock(side_effect=AssertionError("registered"))) + with pytest.raises(ValueError): + traceai.init(project_name="synthetic", verbose=False, attachment_manager=_enabled_like_manager(), + attachment_config=AttachmentConfig(enabled=False)) + + +def test_context_keys_match_the_shared_helper(): + assert traceai._CAPTURE_POLICY_KEY == media_capture.CAPTURE_POLICY_KEY + assert traceai._OWNER_KEY == media_capture.OWNER_KEY + + +def test_context_keys_use_the_typescript_names(): + # The TypeScript auto module names its keys createContextKey("traceai.auto.attachment_policy.v1") + # and createContextKey("traceai.auto.attachment_owner.v1"); the Python names are the same strings. + assert traceai._CAPTURE_POLICY_KEY == "traceai.auto.attachment_policy.v1" + assert traceai._OWNER_KEY == "traceai.auto.attachment_owner.v1" + assert media_capture.CAPTURE_POLICY_KEY == "traceai.auto.attachment_policy.v1" + assert media_capture.OWNER_KEY == "traceai.auto.attachment_owner.v1" + + +def test_policy_set_under_the_shared_name_is_honoured(): + # An adapter that sets the per-call policy by the shared NAME (no traceai import) restricts capture. + ctx = otel_context.set_value("traceai.auto.attachment_policy.v1", CapturePolicy(capture_content=False)) + token = otel_context.attach(ctx) + try: + assert media_capture.local_capture_policy(TraceConfig(), direction="input").capture_content is False + finally: + otel_context.detach(token) + assert media_capture.local_capture_policy(TraceConfig(), direction="input").capture_content is True + assert otel_context.get_value("traceai.auto.attachment_policy.v1", + traceai.with_attachment_capture_policy(CapturePolicy(muted=True))).muted is True + + +def test_capture_policy_helper_only_restricts(): + denied = traceai.with_attachment_capture_policy(CapturePolicy(capture_content=False, muted=True)) + # A later, permissive policy in the same context cannot relax the earlier denial. + relaxed = traceai.with_attachment_capture_policy(CapturePolicy(), denied) + token = otel_context.attach(relaxed) + try: + folded = media_capture.local_capture_policy(TraceConfig(), direction="input") + assert folded.capture_content is False and folded.muted is True + finally: + otel_context.detach(token) + assert media_capture.local_capture_policy(TraceConfig(), direction="input").capture_content is True + token = otel_context.attach(traceai.with_attachment_capture_policy("not a policy")) + try: + folded = media_capture.local_capture_policy(TraceConfig(), direction="output") + assert folded.capture_content is False and folded.requested is False + finally: + otel_context.detach(token) + + +def test_ownership_helper_turns_the_shared_helper_off(): + from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider + + manager = AttachmentManager(config=AttachmentConfig(enabled=True)) + sdk = SdkTracerProvider() + try: + with sdk.get_tracer("synthetic").start_as_current_span("call") as span: + assert media_capture.Observation(manager, span, "openai").usable is True + token = otel_context.attach(traceai.with_attachment_ownership()) + try: + assert media_capture.local_capture_policy(None, direction="input").requested is False + assert media_capture.Observation(manager, span, "openai").usable is False + finally: + otel_context.detach(token) + assert media_capture.local_capture_policy(None, direction="input").requested is True + assert media_capture.Observation(manager, span, "openai").usable is True + finally: + manager.shutdown(0) + sdk.shutdown() + + +def test_merge_is_deny_wins_for_every_fact(): + for item in dataclasses.fields(CapturePolicy): + grant = item.name in ("capture_content", "record_inputs", "record_outputs", "requested") + tight = CapturePolicy(**{item.name: not grant}) + assert getattr(media_capture.merge_capture_policies(CapturePolicy(), tight), item.name) is (not grant) + assert getattr(media_capture.merge_capture_policies(tight, CapturePolicy()), item.name) is (not grant) + + +def test_report_media_rows_are_version_aware_and_content_free(monkeypatch): + descriptor = {"operations": ("messages.create.input", "tool_result.input")} + modules = {"traceai_anthropic": SimpleNamespace(anthropic_attachment_capabilities=descriptor), + "traceai_openai": SimpleNamespace(), # an older instrumentor without the adapter + "traceai_livekit": SimpleNamespace(bind_session=lambda *a, **k: None)} + + def import_module(name): + if name in modules: + return modules[name] + raise ImportError(name) + + monkeypatch.setattr(traceai, "_installed", lambda module: module in modules) + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows["anthropic"]["media"] == "provider_adapter" + assert rows["anthropic"]["media_operations"] == ["messages.create.input", "tool_result.input"] + assert rows["openai"]["media"] == "unqualified" + assert rows["livekit"]["media"] == "session_binding" + assert rows["groq"] == {"name": "groq", "package": "traceai_groq", "status": "not_installed", + "media": "unqualified"} + assert rows["anthropic"]["status"] == "not_enabled" + assert "mistral" not in rows # aliases share one row + assert traceai.report()["attachments"]["state"] == "not_initialized" + + +# ------------------------------------------------- MCP and agent media adapters (#693) +_AGENT_INSTRUMENTORS = { # registry name -> (module, entry point) + "openai_agents": ("traceai_openai_agents", "OpenAIAgentsInstrumentor"), + "claude_agent_sdk": ("traceai_claude_agent_sdk", "ClaudeAgentInstrumentor"), + "smolagents": ("traceai_smolagents", "SmolagentsInstrumentor"), + "firecrawl": ("traceai_firecrawl", "FirecrawlInstrumentor"), +} +_MCP_AGENT_DESCRIPTORS = { # report row name -> (descriptor module, descriptor name) + "mcp": ("traceai_mcp", "mcp_attachment_capabilities"), + "mcp_use": ("traceai_mcp_use._attachments", "mcp_use_attachment_capabilities"), + "openai_agents": ("traceai_openai_agents._attachments", "openai_agents_attachment_capabilities"), + "claude_agent_sdk": ("traceai_claude_agent_sdk._attachments", "claude_agent_attachment_capabilities"), + "smolagents": ("traceai_smolagents._attachments", "smolagents_attachment_capabilities"), + "firecrawl": ("traceai_firecrawl._attachments", "firecrawl_attachment_capabilities"), +} + + +def test_mcp_and_agent_packages_are_registered_with_explicit_entry_points(): + for name, (module, entry) in _AGENT_INSTRUMENTORS.items(): + assert traceai._INSTRUMENTOR_REGISTRY[name] == module + assert traceai._ENTRY_POINTS[module] == entry + for alias, name in (("openai-agents", "openai_agents"), ("claude-agent-sdk", "claude_agent_sdk"), + ("mcp-use", "mcp_use")): + assert traceai._INSTRUMENTOR_REGISTRY[alias] == traceai._INSTRUMENTOR_REGISTRY[name] + assert traceai._INSTRUMENTOR_REGISTRY["mcp_use"] == "traceai_mcp_use" + # mcp-use has a LangChain callback, not an instrumentor: auto must never guess a class for it. + assert "traceai_mcp_use" not in traceai._ENTRY_POINTS + assert "traceai_mcp_use" in traceai._EXPLICIT_ONLY + + +# Provider and framework instrumentors newly reachable from auto (#697 and #691). They take +# the generic keyword arguments; Agno has its own (see test_agno_*). +_PROVIDER_FRAMEWORK_INSTRUMENTORS = { + "openrouter": ("traceai_openrouter", "OpenRouterInstrumentor"), + "portkey": ("traceai_portkey", "PortkeyInstrumentor"), + "vllm": ("traceai_vllm", "VLLMInstrumentor"), + "minimax": ("traceai_minimax", "MiniMaxInstrumentor"), + "pydantic_ai": ("traceai_pydantic_ai", "PydanticAIInstrumentor"), +} + + +@pytest.mark.parametrize("name", sorted({**_AGENT_INSTRUMENTORS, **_PROVIDER_FRAMEWORK_INSTRUMENTORS})) +def test_agent_instrumentors_get_the_shared_manager_and_the_off_kwargs_when_off(monkeypatch, name): + module_name, entry = {**_AGENT_INSTRUMENTORS, **_PROVIDER_FRAMEWORK_INSTRUMENTORS}[name] + for env, expected in ((None, "on"), ("false", "off")): + if env is None: + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + else: + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", env) + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai, "_init_options", None) + manager = _enabled_like_manager() + provider = SimpleNamespace(attachment_manager=manager, trace_config=TraceConfig()) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + constructor = Mock() + imported = [] + + def import_module(requested, constructor=constructor, imported=imported): + imported.append(requested) + return SimpleNamespace(**{entry: constructor}) + + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + traceai.init(project_name="synthetic", auto_instrument=[name], verbose=False) + assert imported == [module_name] + if expected == "on": + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachments=manager, attachment_manager=manager) + else: # attachments OFF: exactly the keyword arguments every other instrumentor gets + constructor.return_value.instrument.assert_called_once_with( + tracer_provider=provider, config=provider.trace_config, attachment_manager=manager, + attachment_policy=manager.policy) + assert traceai.attachment_status()["adapters"][name] == "registered" + + +def test_enabling_mcp_use_records_it_and_constructs_nothing(monkeypatch): + module = SimpleNamespace(FutureAGICallback=Mock(side_effect=AssertionError("constructed a callback")), + SomethingInstrumentor=Mock(side_effect=AssertionError("guessed a class"))) + monkeypatch.setattr(traceai.importlib, "import_module", Mock(return_value=module)) + assert traceai._instrument_library("mcp-use", verbose=False) + assert traceai.attachment_status()["adapters"]["mcp-use"] == "registered" + + +def test_mcp_and_agent_report_rows_read_each_installed_descriptor(monkeypatch): + modules = {} + for name, (path, attr) in _MCP_AGENT_DESCRIPTORS.items(): + modules[path] = SimpleNamespace(**{attr: {"operations": (name + ".output",)}}) + modules.setdefault(traceai._INSTRUMENTOR_REGISTRY[name], SimpleNamespace()) + + def import_module(requested): + if requested in modules: + return modules[requested] + raise ImportError(requested) + + monkeypatch.setattr(traceai, "_installed", lambda module: module in modules) + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + for name in _MCP_AGENT_DESCRIPTORS: + assert rows[name]["media"] == "provider_adapter", name + assert rows[name]["media_operations"] == [name + ".output"], name + assert rows[name]["status"] == "not_enabled", name + for alias in ("mcp-use", "openai-agents", "claude-agent-sdk"): + assert alias not in rows # aliases share one row + # An installed package version without the media adapter (no descriptor) is unqualified. + for name, (path, _attr) in _MCP_AGENT_DESCRIPTORS.items(): + saved = modules[path] + if path == traceai._INSTRUMENTOR_REGISTRY[name]: + modules[path] = SimpleNamespace() # the package is installed, its descriptor is not + else: + del modules[path] # this package version has no _attachments module + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows[name] == {"name": name, "package": traceai._INSTRUMENTOR_REGISTRY[name], + "status": "not_enabled", "media": "unqualified"}, name + modules[path] = saved + + +def _explicit_setup(monkeypatch, manager, config=None, module=None, adapter=True): + provider = SimpleNamespace(attachment_manager=manager, trace_config=config or TraceConfig(hide_outputs=True)) + monkeypatch.setattr(traceai, "_tracer_provider", provider) + monkeypatch.setattr(traceai, "_media_support", lambda m: {"media": "provider_adapter" if adapter else "unqualified"}) + monkeypatch.setattr(traceai.importlib, "import_module", Mock(return_value=module)) + return provider + + +def test_mcp_use_callback_uses_the_shared_manager_only_while_capturing(monkeypatch): + callback_class = Mock() + module = SimpleNamespace(FutureAGICallback=callback_class) + manager = _enabled_like_manager() + provider = _explicit_setup(monkeypatch, manager, module=module) + assert traceai.mcp_use_callback() is callback_class.return_value + callback_class.assert_called_once_with(tracer_provider=provider, config=provider.trace_config, attachments=manager) + assert traceai.attachment_status()["adapters"]["mcp_use"] == "registered" + # Caller arguments win over auto's defaults. + callback_class.reset_mock() + own = TraceConfig() + traceai.mcp_use_callback(config=own, redact=["x"]) + assert callback_class.call_args.kwargs["config"] is own and callback_class.call_args.kwargs["redact"] == ["x"] + # Attachments OFF: the callback auto built before (no attachments argument at all). + for off in ("env", "disabled", "no_adapter"): + callback_class.reset_mock() + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + if off == "env": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + provider = _explicit_setup(monkeypatch, manager, module=module) + elif off == "disabled": + provider = _explicit_setup(monkeypatch, DisabledAttachmentManager(), module=module) + else: + provider = _explicit_setup(monkeypatch, manager, module=module, adapter=False) + traceai.mcp_use_callback() + callback_class.assert_called_once_with(tracer_provider=provider, config=provider.trace_config) + + +def test_wrap_mcp_client_uses_the_shared_manager_and_tolerates_old_packages(monkeypatch): + wrap = Mock() + manager = _enabled_like_manager() + provider = _explicit_setup(monkeypatch, manager, module=SimpleNamespace(wrap_mcp_client=wrap)) + session = object() + assert traceai.wrap_mcp_client(session) is wrap.return_value + wrap.assert_called_once_with(session, tracer_provider=provider, trace_config=provider.trace_config, + attachments=manager) + wrap.reset_mock() + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + traceai.wrap_mcp_client(session) + wrap.assert_called_once_with(session, tracer_provider=provider, trace_config=provider.trace_config) + # An installed traceAI-mcp without wrap_mcp_client: the session comes back untouched. + _explicit_setup(monkeypatch, manager, module=SimpleNamespace()) + assert traceai.wrap_mcp_client(session) is session + assert traceai.attachment_status()["adapters"]["mcp"] == "unsupported_version" + + +def test_explicit_entry_points_need_init(monkeypatch): + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(FutureAGICallback=Mock(), wrap_mcp_client=Mock()))) + with pytest.raises(RuntimeError): + traceai.mcp_use_callback() + with pytest.raises(RuntimeError): + traceai.wrap_mcp_client(object()) + + +# ------------------------------------- provider (#697) and framework (#691) media adapters +# report row name -> (descriptor module, descriptor name). The provider packages define the +# descriptor as a lazy module attribute; the framework packages in their _attachments module. +_PROVIDER_FRAMEWORK_DESCRIPTORS = { + "google_genai": ("traceai_google_genai", "google_genai_attachment_capabilities"), + "vertexai": ("traceai_vertexai", "vertexai_attachment_capabilities"), + "bedrock": ("traceai_bedrock", "bedrock_attachment_capabilities"), + "litellm": ("traceai_litellm", "litellm_attachment_capabilities"), + "groq": ("traceai_groq._audio", "groq_attachment_capabilities"), + "fireworks": ("traceai_fireworks", "fireworks_attachment_capabilities"), + "xai": ("traceai_xai", "xai_attachment_capabilities"), + "mistralai": ("traceai_mistralai", "mistralai_attachment_capabilities"), + "openrouter": ("traceai_openrouter", "openrouter_attachment_capabilities"), + "portkey": ("traceai_portkey", "portkey_attachment_capabilities"), + "ollama": ("traceai_ollama", "ollama_attachment_capabilities"), + "vllm": ("traceai_vllm", "vllm_attachment_capabilities"), + "minimax": ("traceai_minimax", "minimax_attachment_capabilities"), + "langchain": ("traceai_langchain._attachments", "langchain_attachment_capabilities"), + "llamaindex": ("traceai_llamaindex._attachments", "llamaindex_attachment_capabilities"), + "agno": ("traceai_agno._attachments", "agno_attachment_capabilities"), + "pydantic_ai": ("traceai_pydantic_ai._attachments", "pydantic_ai_attachment_capabilities"), +} +# Instrumented, but their descriptors claim no media operation: never a media adapter. +_UNCLAIMED = {"together": "traceai_together", "deepseek": "traceai_deepseek", "cerebras": "traceai_cerebras"} + + +def test_provider_and_framework_packages_are_registered_with_explicit_entry_points(): + for name, (module, entry) in _PROVIDER_FRAMEWORK_INSTRUMENTORS.items(): + assert traceai._INSTRUMENTOR_REGISTRY[name] == module + assert traceai._ENTRY_POINTS[module] == entry + assert traceai._INSTRUMENTOR_REGISTRY["pydantic-ai"] == "traceai_pydantic_ai" + assert traceai._INSTRUMENTOR_REGISTRY["agno"] == "traceai_agno" + assert traceai._ENTRY_POINTS["traceai_agno"] == "AgnoInstrumentorWrapper" + for name, (path, attr) in _PROVIDER_FRAMEWORK_DESCRIPTORS.items(): + module = traceai._INSTRUMENTOR_REGISTRY[name] + assert traceai._MEDIA_ADAPTERS[module] == (path, attr), name + for module in _UNCLAIMED.values(): + assert module not in traceai._MEDIA_ADAPTERS + assert module in traceai._ENTRY_POINTS # still traced + + +def test_provider_and_framework_report_rows_read_each_installed_descriptor(monkeypatch): + modules = {} + for name, (path, attr) in _PROVIDER_FRAMEWORK_DESCRIPTORS.items(): + modules[path] = SimpleNamespace(**{attr: {"operations": (name + ".input",)}}) + modules.setdefault(traceai._INSTRUMENTOR_REGISTRY[name], SimpleNamespace()) + for name, module in _UNCLAIMED.items(): # installed, with a descriptor that claims nothing + modules[module] = SimpleNamespace(**{name + "_attachment_capabilities": {"operations": ()}}) + + def import_module(requested): + if requested in modules: + return modules[requested] + raise ImportError(requested) + + monkeypatch.setattr(traceai, "_installed", lambda module: module in modules) + monkeypatch.setattr(traceai.importlib, "import_module", import_module) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + for name in _PROVIDER_FRAMEWORK_DESCRIPTORS: + assert rows[name]["media"] == "provider_adapter", name + assert rows[name]["media_operations"] == [name + ".input"], name + assert rows[name]["status"] == "not_enabled", name + for name, module in _UNCLAIMED.items(): + assert rows[name] == {"name": name, "package": module, "status": "not_enabled", "media": "unqualified"} + for alias in ("pydantic-ai", "google-genai", "vertex", "mistral"): + assert alias not in rows # aliases share one row + # An installed version without the descriptor: the lazy attribute raises AttributeError (old + # package) or ImportError (a core without provider_media); both are unqualified. + for name, (path, attr) in _PROVIDER_FRAMEWORK_DESCRIPTORS.items(): + saved = modules[path] + package = traceai._INSTRUMENTOR_REGISTRY[name] + for broken in ("attribute", "import"): + if path != package: + del modules[path] + elif broken == "attribute": + modules[path] = SimpleNamespace() + else: + modules[path] = _RaisingModule(ImportError("fi_instrumentation.attachments.provider_media")) + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows[name] == {"name": name, "package": package, "status": "not_enabled", + "media": "unqualified"}, (name, broken) + modules[path] = saved + # A descriptor that lists no operation is never reported as a media adapter. + modules["traceai_litellm"] = SimpleNamespace(litellm_attachment_capabilities={"operations": []}) + assert {row["name"]: row for row in traceai.report()["integrations"]}["litellm"]["media"] == "unqualified" + + +class _RaisingModule: + def __init__(self, error): + self._error = error + + def __getattr__(self, name): + raise self._error + + +def _stub_openinference(monkeypatch): + """OpenInference's TraceConfig shape (hide_* flags plus an image cap), without the package.""" + import dataclasses + import sys + from types import ModuleType + + @dataclasses.dataclass(frozen=True) + class OpenInferenceTraceConfig: + hide_llm_invocation_parameters: bool = False + hide_inputs: bool = False + hide_outputs: bool = False + hide_input_messages: bool = False + hide_output_messages: bool = False + hide_input_images: bool = False + hide_input_text: bool = False + hide_output_text: bool = False + hide_embedding_vectors: bool = False + base64_image_max_length: int = 32000 + + package, module = ModuleType("openinference"), ModuleType("openinference.instrumentation") + module.TraceConfig = OpenInferenceTraceConfig + package.instrumentation = module + monkeypatch.setitem(sys.modules, "openinference", package) + monkeypatch.setitem(sys.modules, "openinference.instrumentation", module) + return OpenInferenceTraceConfig + + +def _agno_init(monkeypatch, manager, config): + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai, "_init_options", None) + monkeypatch.setattr(traceai, "_instrumented", set()) + provider = SimpleNamespace(attachment_manager=manager, trace_config=config) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + wrapper = Mock() + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(AgnoInstrumentorWrapper=wrapper))) + traceai.init(project_name="synthetic", auto_instrument=["agno"], verbose=False) + wrapper.assert_called_once_with() + wrapper.return_value.instrument.assert_called_once() + return provider, wrapper.return_value.instrument.call_args + + +def test_agno_gets_its_own_keyword_arguments_on_and_off(monkeypatch): + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + monkeypatch.setattr(traceai, "_media_support", lambda module: {"media": "provider_adapter"}) + manager = _enabled_like_manager() + provider, call = _agno_init(monkeypatch, manager, TraceConfig()) + # No fi TraceConfig and no attachment_* alias: Agno's wrapper accepts neither. + assert call.args == () and call.kwargs == {"tracer_provider": provider, "attachments": manager} + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + for off in ("env", "disabled"): + if off == "env": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = _enabled_like_manager() + else: + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + manager = traceai.DisabledAttachmentManager() + provider, call = _agno_init(monkeypatch, manager, TraceConfig()) + # Attachments OFF: exactly what configure_agno_tracing(tracer_provider=...) gets. + assert call.kwargs == {"tracer_provider": provider}, off + + +def test_agno_carries_auto_hide_flags_as_an_openinference_config(monkeypatch): + oi_config = _stub_openinference(monkeypatch) + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + config = TraceConfig(hide_inputs=True, hide_output_text=True, hide_input_images=True) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), config) + passed = call.kwargs["config"] + assert isinstance(passed, oi_config) + assert (passed.hide_inputs, passed.hide_output_text, passed.hide_input_images) == (True, True, True) + assert (passed.hide_outputs, passed.hide_input_text, passed.base64_image_max_length) == (False, False, 32000) + assert set(call.kwargs) == {"tracer_provider", "config"} + + +def test_a_dependency_conflict_is_never_reported_registered(monkeypatch): + provider = SimpleNamespace(attachment_manager=traceai.DisabledAttachmentManager(), trace_config=TraceConfig()) + monkeypatch.setattr(traceai, "register", Mock(return_value=provider)) + instrumentor = Mock(is_instrumented_by_opentelemetry=False) # BaseInstrumentor skipped _instrument + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(PydanticAIInstrumentor=Mock(return_value=instrumentor)))) + traceai.init(project_name="synthetic", auto_instrument=["pydantic-ai"], verbose=False) + instrumentor.instrument.assert_called_once() + assert traceai.attachment_status()["adapters"]["pydantic-ai"] == "dependency_unavailable" + assert "traceai_pydantic_ai" not in traceai._instrumented + rows = {row["name"]: row for row in traceai.report()["integrations"]} + assert rows["pydantic_ai"]["status"] == "dependency_unavailable" + + +def test_agno_without_the_media_adapter_keeps_the_off_arguments(monkeypatch): + """A traceai-agno release before the adapter has no ``attachments`` parameter at all.""" + monkeypatch.delenv("FI_ATTACHMENTS_ENABLED", raising=False) + monkeypatch.setattr(traceai, "_media_support", lambda module: {"media": "unqualified"}) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), TraceConfig()) + assert call.kwargs == {"tracer_provider": provider} + assert traceai.attachment_status()["adapters"]["agno"] == "registered" + + +def _agno_on(monkeypatch): + for name in ("FI_ATTACHMENTS_ENABLED", "FI_PII_REDACTION", "FI_HIDE_ERROR_DETAILS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(traceai, "_media_support", lambda module: {"media": "provider_adapter"}) + + +def _agno_row(): + return {row["name"]: row for row in traceai.report()["integrations"]}["agno"]["status"] + + +def test_agno_fails_closed_on_a_pii_or_error_flag_it_cannot_apply(monkeypatch): + """Agno reads pii_redaction / hide_error_details only from the environment: no media, own status.""" + _stub_openinference(monkeypatch) + for config in (TraceConfig(pii_redaction=True), TraceConfig(hide_error_details=True), + TraceConfig(pii_redaction=True, hide_error_details=True)): + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), config) + assert call.kwargs == {"tracer_provider": provider}, config + assert traceai.attachment_status()["adapters"]["agno"] == "config_unsupported" + assert _agno_row() == "config_unsupported" + # A hide flag Agno does receive keeps its config, and the error flag it derives is not lost. + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + provider, call = _agno_init(monkeypatch, _enabled_like_manager(), + TraceConfig(hide_inputs=True, pii_redaction=True)) + assert set(call.kwargs) == {"tracer_provider", "config"} and call.kwargs["config"].hide_inputs is True + assert _agno_row() == "config_unsupported" + + +def test_agno_keeps_media_when_the_environment_carries_the_flag(monkeypatch): + _stub_openinference(monkeypatch) + for env, config in (("FI_PII_REDACTION", TraceConfig(pii_redaction=True)), + ("FI_HIDE_ERROR_DETAILS", TraceConfig(hide_error_details=True))): + _agno_on(monkeypatch) + monkeypatch.setenv(env, "true") + monkeypatch.setattr(traceai, "_adapter_status", {}) + manager = _enabled_like_manager() + provider, call = _agno_init(monkeypatch, manager, config) + assert call.kwargs == {"tracer_provider": provider, "attachments": manager}, env + assert _agno_row() == "registered" + # hide_error_details derived from a forwarded content hide flag is applied by Agno too. + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + manager = _enabled_like_manager() + config = TraceConfig(hide_inputs=True) + assert config.hide_error_details is True + provider, call = _agno_init(monkeypatch, manager, config) + assert set(call.kwargs) == {"tracer_provider", "config", "attachments"} + assert _agno_row() == "registered" + + +def test_agno_off_arguments_are_unchanged_by_an_unreachable_flag(monkeypatch): + _stub_openinference(monkeypatch) + for off in ("env", "disabled"): + _agno_on(monkeypatch) + monkeypatch.setattr(traceai, "_adapter_status", {}) + if off == "env": + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = _enabled_like_manager() + else: + manager = traceai.DisabledAttachmentManager() + provider, call = _agno_init(monkeypatch, manager, TraceConfig(pii_redaction=True)) + assert call.kwargs == {"tracer_provider": provider}, off + assert _agno_row() == "config_unsupported", off