diff --git a/.github/workflows/attachments-python-mcp-agents-tests.yml b/.github/workflows/attachments-python-mcp-agents-tests.yml new file mode 100644 index 000000000..89bf05ee6 --- /dev/null +++ b/.github/workflows/attachments-python-mcp-agents-tests.yml @@ -0,0 +1,106 @@ +name: Python MCP and agent attachments + +# Real MCP, mcp-use, OpenAI Agents SDK, Claude Agent SDK, smolagents and +# firecrawl-py releases (declared floor and latest) through the Python +# instrumentors to a TLS loopback F01 attachment server. Outbound network is +# disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/mcp/**' + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/openai-agents/**' + - 'python/frameworks/claude-agent-sdk/**' + - 'python/frameworks/smolagents/**' + - 'python/frameworks/firecrawl/**' + - 'python/tests/attachments_mcp_agents/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-mcp-agents-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/mcp/**' + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/openai-agents/**' + - 'python/frameworks/claude-agent-sdk/**' + - 'python/frameworks/smolagents/**' + - 'python/frameworks/firecrawl/**' + - 'python/tests/attachments_mcp_agents/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-mcp-agents-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +env: + MCP_USE_ANONYMIZED_TELEMETRY: 'false' + MCP_USE_LANGFUSE: 'false' + MCP_USE_LAMINAR: 'false' + +jobs: + attachments: + name: ${{ matrix.package }} attachments (${{ matrix.tier }}) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - {package: core, tier: latest, python: '3.12', path: '', suite: test_tool_media.py, sdks: 'pillow'} + - {package: mcp, tier: floor, python: '3.11', path: mcp, suite: test_mcp_attachments.py, sdks: 'mcp==1.30.0'} + - {package: mcp, tier: latest, python: '3.12', path: mcp, suite: test_mcp_attachments.py, sdks: 'mcp==2.3.0'} + - {package: mcp-use, tier: floor, python: '3.11', path: mcp-use, suite: test_mcp_use_attachments.py, + sdks: 'mcp-use==1.7.1 langchain-core==1.6.12'} + - {package: mcp-use, tier: latest, python: '3.12', path: mcp-use, suite: test_mcp_use_attachments.py, + sdks: 'mcp-use==1.7.1'} + # 0.2.6 is the declared floor; computer calls get a function span from 0.10.0. + - {package: openai-agents, tier: floor, python: '3.11', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.2.6'} + - {package: openai-agents, tier: computer-span, python: '3.12', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.10.0 openai==2.19.0'} + - {package: openai-agents, tier: latest, python: '3.12', path: openai-agents, + suite: test_openai_agents_attachments.py, sdks: 'openai-agents==0.24.0'} + - {package: claude-agent-sdk, tier: floor, python: '3.11', path: claude-agent-sdk, + suite: test_claude_agent_sdk_attachments.py, sdks: 'claude-agent-sdk==0.1.0'} + - {package: claude-agent-sdk, tier: latest, python: '3.12', path: claude-agent-sdk, + suite: test_claude_agent_sdk_attachments.py, sdks: 'claude-agent-sdk==0.2.180'} + - {package: smolagents, tier: floor, python: '3.11', path: smolagents, suite: test_smolagents_attachments.py, + sdks: 'smolagents==1.12.0 openai'} + - {package: smolagents, tier: latest, python: '3.12', path: smolagents, suite: test_smolagents_attachments.py, + sdks: 'smolagents==1.26.0 openai'} + - {package: firecrawl, tier: floor, python: '3.11', path: firecrawl, suite: test_firecrawl_attachments.py, + sdks: 'firecrawl-py==4.46.2'} + - {package: firecrawl, tier: latest, python: '3.12', path: firecrawl, suite: test_firecrawl_attachments.py, + sdks: 'firecrawl-py==4.55.0'} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install the source integration and the selected SDK + env: + SDKS: ${{ matrix.sdks }} + PKG_PATH: ${{ matrix.path }} + run: | + python -m venv .venv + extra="" + if [ -n "$PKG_PATH" ]; then extra="-e python/frameworks/$PKG_PATH"; fi + # shellcheck disable=SC2086 + .venv/bin/pip install -e python $extra opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' packaging $SDKS + .venv/bin/pip freeze | grep -iE '^(mcp|mcp-use|openai-agents|openai|claude-agent-sdk|smolagents|firecrawl-py|pillow|langchain-core)==' || true + - name: Run with outbound network disabled + env: + SUITE: ${{ matrix.suite }} + run: | + sudo --preserve-env=MCP_USE_ANONYMIZED_TELEMETRY,MCP_USE_LANGFUSE,MCP_USE_LAMINAR unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + PYTHONPATH=. "$1" -m pytest "tests/attachments_mcp_agents/$3" -q -p no:cacheprovider -o addopts="" \ + -o faulthandler_timeout=120 + ' _ "$PWD/.venv/bin/python" "$PWD" "$SUITE" diff --git a/python/fi_instrumentation/attachments/tool_media.py b/python/fi_instrumentation/attachments/tool_media.py new file mode 100644 index 000000000..1ef0925e7 --- /dev/null +++ b/python/fi_instrumentation/attachments/tool_media.py @@ -0,0 +1,305 @@ +"""Shared tool-media helpers for MCP and agent adapters, built on ``media_capture``. + +MCP clients, ``mcp-use``, the OpenAI Agents SDK, the Claude Agent SDK, +smolagents and Firecrawl all observe media that a TOOL returned. These helpers +give them one set of rules (the TypeScript ``@traceai/mcp`` and +``@traceai/claude-agent-sdk`` semantics): + +- Tool media is tool output AND potential model input, so input and output + hide flags, ``hide_input_images`` for images and ``pii_redaction`` all deny + (deny wins; the core cannot relax a local restriction). +- Only typed inline values are read: base64 fields, data URIs, and in-memory + images. Nothing here fetches a URL, dereferences a resource URI or path, + opens a file, or reads a stream. A URL or file-backed value is skipped. +- Every value is preflighted before it is read or decoded. Each observation + has a bounded decode budget and part count; parts beyond them are skipped. +- A declared MIME is never trusted over the bytes: when the first bytes are a + known signature (``_SIGNATURES``) of ANOTHER modality (a PNG labelled + ``audio/wav``), the part is denied, so a mislabel cannot escape + ``hide_input_images`` or a modality restriction. Never relabelled. The + signatures checked are PNG, JPEG, GIF, WebP, BMP, TIFF, ISO-BMFF image + brands (``ftyp`` avif/heic/heix/mif1/msf1), WAV, Ogg, FLAC, ID3-tagged MP3 + and PDF; other bytes (MP4/WebM containers, MP3 without an ID3 tag) are + judged by their declared MIME only. ``application/octet-stream`` is the + ``file`` modality, so an octet-stream blob whose bytes carry a recognised + image, audio or PDF signature is skipped (fail-closed, never relabelled). +- A context marked by another adapter as the owner of this call's media + (``OWNER_KEY``, shared by name with ``traceai``) captures nothing here. + +Not part of ``fi_instrumentation.attachments.__all__``. Every function is +non-throwing and returns ``None``/empty on denial. +""" + +import base64 +import binascii +import dataclasses +import io +import sys +from typing import Any, Callable, Mapping, Optional + +from opentelemetry import context as otel_context + +from . import media_capture as mc +from .policy import CapturePolicy +from .types import MIME_MODALITIES, Bytes + +KiB = 1024 +MiB = 1024 * KiB +#: Decoded bytes one tool result may contribute (TypeScript MCP_MEDIA_DECODE_BUDGET). +TOOL_RESULT_BUDGET = 256 * KiB +#: Typed parts inspected per tool result. +MAX_TOOL_PARTS = 64 +#: In-memory images are PNG-encoded only up to this many pixels (no huge synchronous encodes). +MAX_IMAGE_PIXELS = 2048 * 2048 +#: The cross-adapter owner mark, by name (``media_capture.OWNER_KEY`` where the core defines it). +OWNER_KEY = getattr(mc, "OWNER_KEY", "traceai.auto.attachment_owner.v1") +URI_PREFIX = "fi-attachment:" +Occurrence = mc.Occurrence + + +def owned_elsewhere() -> bool: + """True when another adapter marked this call's media as its own (by context key name).""" + try: + return otel_context.get_value(OWNER_KEY) is not None + except Exception: + return True + + +def with_ownership(context=None): + """A context that tells downstream observers this call's media is captured already.""" + return otel_context.set_value(OWNER_KEY, True, context) + + +def uri(occurrence: Occurrence) -> str: + return URI_PREFIX + occurrence.occurrence_id + + +def tool_capture_policy(trace_config=None, capture: Optional[CapturePolicy] = None, *, image: bool = False, + error: bool = False) -> CapturePolicy: + """Deny-only policy for media a tool returned: input AND output restrictions both apply.""" + base = capture if isinstance(capture, CapturePolicy) else CapturePolicy() + seen_input = mc.local_capture_policy(trace_config, direction="input", image=image, error_detail=error, + base=base) + seen_output = mc.local_capture_policy(trace_config, direction="output", image=image, error_detail=error, + base=base) + return dataclasses.replace( + seen_input, + capture_content=seen_input.capture_content and seen_output.capture_content, + requested=seen_input.requested and seen_output.requested and not owned_elsewhere(), + contains_output=True, + model_input=True, + ) + + +def mime_kind(mime: Any) -> Optional[str]: + """The F01 modality of an exact MIME; None when the MIME is not in the F01 enum.""" + return MIME_MODALITIES.get(mime) if isinstance(mime, str) else None + + +#: Leading-byte signatures with an unambiguous F01 modality. ISO-BMFF ``ftyp`` boxes name an +#: image only for the HEIF/AVIF still-image brands listed here; other brands (mp42, isom, +#: ``M4A ``, ``qt ``) and WebM/Matroska are shared by audio and video, so those bytes are +#: judged by their declared MIME only. ``consistent`` decodes the first 12 bytes. +_SIGNATURES = ( + (0, b"\x89PNG\r\n\x1a\n", "image"), (0, b"\xff\xd8\xff", "image"), (0, b"GIF87a", "image"), + (0, b"GIF89a", "image"), (8, b"WEBP", "image"), (0, b"BM", "image"), (0, b"II*\x00", "image"), + (0, b"MM\x00*", "image"), (4, b"ftypavif", "image"), (4, b"ftypheic", "image"), (4, b"ftypheix", "image"), + (4, b"ftypmif1", "image"), (4, b"ftypmsf1", "image"), (8, b"WAVE", "audio"), + (0, b"OggS", "audio"), (0, b"fLaC", "audio"), (0, b"ID3", "audio"), (0, b"%PDF-", "document"), +) + + +def signature_kind(head: bytes) -> Optional[str]: + """The modality the first bytes name, or None when they name none (or are ambiguous).""" + for offset, magic, kind in _SIGNATURES: + if head[offset:offset + len(magic)] == magic: + if offset == 8 and head[:4] != b"RIFF": + continue + return kind + return None + + +def _declared_kind(value: str, mimes, raw: bool) -> Optional[str]: + if raw: + return mime_kind(mimes[0]) + end = value.find(";", 5, 128) + return mime_kind(value[5:end]) if value.startswith("data:") and end > 5 else None + + +def consistent(value: Any, mimes, raw: bool) -> bool: + """False when the base64 value's first bytes are a signature of another modality than declared.""" + if not isinstance(value, str): + return True # Not inline base64: the structural parse rejects it. + start = 0 + if not raw: + comma = value.find(",", 0, 128) + if comma < 0: + return True + start = comma + 1 + try: + head = base64.b64decode(value[start:start + 16], validate=True) + except (binascii.Error, ValueError): + return True # Malformed base64: strict decoding rejects it later. + found = signature_kind(head) + return found is None or found == _declared_kind(value, mimes, raw) + + +def _pil_image(value: Any) -> bool: + module = sys.modules.get("PIL.Image") + image_type = getattr(module, "Image", None) if module is not None else None + return isinstance(image_type, type) and isinstance(value, image_type) + + +def model_input_policy(trace_config=None, capture: Optional[CapturePolicy] = None, *, + image: bool = False) -> CapturePolicy: + """Deny-only policy for media an agent framework hands to a model as input.""" + policy = mc.local_capture_policy(trace_config, direction="input", image=image, base=capture) + return dataclasses.replace(policy, requested=policy.requested and not owned_elsewhere()) + + +class ToolMedia: + """One adapter's observation of one span, with a decode budget and registry dedup. + + ``namespace`` keeps positions of different adapters apart in the shared + per-manager registry; a repeated observation of the same span and + position returns the first occurrence and uploads nothing. + ``direction`` is ``"output"`` for media a tool returned (tool policy) or + ``"input"`` for media a framework sends to a model (model-input policy). + """ + + def __init__(self, manager, span, namespace: str, *, trace_config=None, + capture: Optional[CapturePolicy] = None, budget: int = TOOL_RESULT_BUDGET, + max_parts: int = MAX_TOOL_PARTS, direction: str = "output"): + self.manager, self.span = manager, span + self.direction = "input" if direction == "input" else "output" + self._trace_config, self._capture = trace_config, capture + self.remaining = budget + self._parts = 0 + self._max_parts = max_parts + self._observation = mc.Observation(manager, span, namespace) + if owned_elsewhere(): + self._observation.key = None + + def _policy(self, image: bool, error: bool) -> CapturePolicy: + if self.direction == "input": + return model_input_policy(self._trace_config, self._capture, image=image) + return tool_capture_policy(self._trace_config, self._capture, image=image, error=error) + + @property + def usable(self) -> bool: + return self._observation.usable and self.remaining > 0 + + @property + def captured(self) -> Mapping[str, Occurrence]: + return dict(self._observation.captured) + + def _visit(self, position: str, capture: Callable[[], Optional[Occurrence]]) -> Optional[Occurrence]: + if not self.usable or self._parts >= self._max_parts: + return None + self._parts += 1 + self._observation.visit(position, capture) + return self._observation.captured.get(position) + + def base64(self, position: str, read: Callable[[], Any], mime: Any, *, purpose: str, + expect: Optional[str] = None, role: str = "tool", tool_call_id: Optional[str] = None, + message_index: Optional[int] = None, part_index: Optional[int] = None, + error: bool = False) -> Optional[Occurrence]: + """A raw base64 value with a declared MIME (MCP ImageContent/AudioContent/blob, Anthropic source). + + ``expect`` ("image", "audio", ...) rejects a MIME of another modality, so a + misleading MIME cannot relabel an image to escape ``hide_input_images``. + ``read`` is called at most once, and only after a preflight admitted the part. + """ + kind = mime_kind(mime) + if kind is None or (expect is not None and kind != expect): + return None + return self._inline(position, read, (mime,), True, purpose, kind == "image", role, tool_call_id, + message_index, part_index, error) + + def data_uri(self, position: str, read: Callable[[], Any], mimes, *, purpose: str, role: str = "tool", + tool_call_id: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, error: bool = False) -> Optional[Occurrence]: + """A ``data:;base64,`` string whose MIME must be one of ``mimes`` (first is preflighted).""" + mimes = tuple(mime for mime in mimes if mime_kind(mime)) + if not mimes: + return None + image = any(mime_kind(mime) == "image" for mime in mimes) + return self._inline(position, read, mimes, False, purpose, image, role, tool_call_id, message_index, + part_index, error) + + def _inline(self, position, read, mimes, raw, purpose, image, role, tool_call_id, message_index, part_index, + error) -> Optional[Occurrence]: + seen = {} + + def once(): + if "value" not in seen: + value = read() + seen["value"] = value if consistent(value, mimes, raw) else None + return seen["value"] + + def capture(): + policy = self._policy(image, error) + return mc.attach_inline(self.manager, self.span, once, mimes, raw=raw, direction=self.direction, + purpose=purpose, capture=policy, message_index=message_index, + part_index=part_index, role=role, tool_call_id=tool_call_id, + max_bytes=self.remaining) + + before = self._observation.captured.get(position) + occurrence = self._visit(position, capture) + if occurrence is not None and before is None and "value" in seen: + value = seen["value"] + if not raw and isinstance(value, str): + value = value[value.find(",", 0, 128) + 1:] + self.remaining -= mc.encoded_length(value, self.remaining) or self.remaining + return occurrence + + def image(self, position: str, image: Any, *, purpose: str, role: str = "tool", + tool_call_id: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, error: bool = False) -> Optional[Occurrence]: + """An in-memory ``PIL.Image.Image``, PNG-encoded only after a pixel-bounded preflight. + + A file-backed image that is not loaded yet is skipped: encoding it would + read the file. PIL is never imported here; a value is an image only when + the application already imported PIL. + """ + if not _pil_image(image): + return None + + def capture(): + try: + fp = getattr(image, "fp", None) + if fp is not None and not isinstance(fp, io.BytesIO): + return None + width, height = image.size + bands = len(image.getbands()) + if not (0 < width and 0 < height and width * height <= MAX_IMAGE_PIXELS): + return None + # PNG never exceeds the raw pixel bytes plus per-row filter bytes and chunk framing. + bound = min(width * height * max(bands, 1) + height + 4 * KiB, self.remaining) + policy = self._policy(True, error) + options = dict(mime_type="image/png", direction=self.direction, purpose=purpose, + capture_policy=policy) + ticket = self.manager.preflight(self.span, max_bytes=bound, **options) + if not ticket.admitted: + return None + except Exception: + return None + try: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + data = buffer.getvalue() + except Exception: + data = b"" + try: + if not data or len(data) > ticket.max_bytes: + ticket.release() + return None + handle = ticket.attach(Bytes(data), **mc._fields(message_index, part_index, role, tool_call_id)) + if handle.admission != "queued": + return None + self.remaining -= len(data) + return Occurrence(handle.occurrence_id, "image/png") + except Exception: + return None + + return self._visit(position, capture) diff --git a/python/frameworks/claude-agent-sdk/README.md b/python/frameworks/claude-agent-sdk/README.md index 33a9fc14f..71fb0e01e 100644 --- a/python/frameworks/claude-agent-sdk/README.md +++ b/python/frameworks/claude-agent-sdk/README.md @@ -183,3 +183,37 @@ pytest tests/ -v ## License MIT + +## Tool-result media (attachments) + +Pass the shared manager: `ClaudeAgentInstrumentor().instrument(tracer_provider=..., attachments=manager)`. +The base64 `image` and PDF `document` blocks of a `ToolResultBlock` are +uploaded from the original SDK message, before any text flattening or +truncation, as `tool_result` occurrences with the `tool_use_id`. They belong +to the tool's span and are projected on it (`fi.attachment.refs`). The CLI +usually runs the PostToolUse hook before it streams the tool result, so with +attachments on, while that client's `receive_response()` conversation is +running, the tool span stays recording until its result has been captured (at +most 30 seconds, until that conversation ends, or until `disconnect()`) and +then ends at the time the hook recorded; its duration is unchanged. Each +client keeps its own waiting spans, so one conversation ending never ends +another client's. Tool media is captured from `receive_response()` only: +with `receive_messages()`, or for a hook that fires before +`receive_response()` starts, the hook ends the span immediately, as it +always does with attachments off. +Media bytes never reach `tool.output` or the conversation output text. The +SDK drops top-level user image blocks while parsing, so prompt images are not +captured. URL and file sources are never read. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py new file mode 100644 index 000000000..3a42f9096 --- /dev/null +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attachments.py @@ -0,0 +1,151 @@ +"""Tool-result media of a Claude Agent SDK conversation, through the shared attachment manager. + +The SDK parses each CLI ``user`` message into a ``UserMessage`` whose +``ToolResultBlock.content`` keeps the original Anthropic content blocks. The +base64 ``image`` and PDF ``document`` blocks there are uploaded BEFORE any text +flattening or truncation, at the tool's span (with ``tool_call_id``): + +- on the tool span, while it is still recording: before the PostToolUse hook + (result-first) or after it, because with capture on and the client's + ``receive_response()`` conversation live, the hook defers the span's end + until its result has been captured (hooks-first, the CLI's usual order); + the span then ends at the time the hook recorded. Each client has its own + deferral tables (``_hooks.ToolDeferrals``); +- for a result with no traced or recording tool span (for example a hook that + ran before ``receive_response()`` started), on the conversation span. + +Every occurrence is projected (``fi.attachment.refs``) on the live span that +is its upload parent. + +URL and file sources, paths and transcripts are never read. The SDK drops +top-level user image blocks during parsing, so prompt images are not +observable here (``source_unavailable``). Model calls run inside the Claude +Code process, so no in-process provider adapter sees the same bytes. +""" + +from types import MappingProxyType +from typing import Any, List, Mapping + +__all__ = ("capture_tool_results", "claude_agent_attachment_capabilities", "media_safe", "tool_result_ids") + +claude_agent_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tool_result.output",), + "sources": ("inline_base64",), + "content_types": ("image", "document"), + "unsupported": ("url_source", "file_source", "prompt_images", "transcript", "local_path"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_MEDIA_TYPES = ("image", "document", "audio") +_MAX_BLOCKS = 64 + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def tool_result_ids(message: Any) -> List[str]: + """The tool_use_ids of the ``ToolResultBlock``s in one ``UserMessage`` (bounded). Never raises.""" + try: + content = _get(message, "content") + if not isinstance(content, list): + return [] + return [block.tool_use_id for block in content[:_MAX_BLOCKS] + if type(block).__name__ == "ToolResultBlock" and isinstance(_get(block, "tool_use_id"), str)] + except Exception: + return [] + + +def capture_tool_results(manager, message: Any, conversation_span: Any, active_spans: Mapping[str, Any], + deferred_spans: Mapping[str, Any] = _EMPTY, *, message_index: int, trace_config=None, + capture=None) -> int: + """Capture the media blocks of every ``ToolResultBlock`` in one ``UserMessage``. Never raises. + + ``active_spans`` maps a tool_use_id to its open span (PostToolUse not run yet) and + ``deferred_spans`` to a span PostToolUse completed whose end waits for this result; + both are the hooks' tables and both spans are still recording. Any other result is + captured on the (recording) conversation span. Returns the number of admitted blocks. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return 0 + admitted = 0 + try: + if not active(manager): + return 0 + content = _get(message, "content") + if not isinstance(content, list): + return 0 + for block in content[:MAX_TOOL_PARTS]: + if type(block).__name__ != "ToolResultBlock": + continue + blocks = _get(block, "content") + if not isinstance(blocks, list): + continue + tool_use_id = _get(block, "tool_use_id") + entry = None + if isinstance(tool_use_id, str): + entry = active_spans.get(tool_use_id) or deferred_spans.get(tool_use_id) + target = entry[0] if entry is not None else conversation_span + if not _recording(target): + target = conversation_span + error = _get(block, "is_error") is True + media = ToolMedia(manager, target, "claude_agent_sdk", trace_config=trace_config, capture=capture) + for part_index, part in enumerate(blocks[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(part, "type") + if kind not in ("image", "document"): + continue + source = _get(part, "source") + if _get(source, "type") != "base64": + continue # url / file sources are never fetched + mime = _get(source, "media_type") + expect = "image" if kind == "image" else "document" + found = media.base64("tool_result/%s/%d" % (tool_use_id, part_index), + lambda s=source: _get(s, "data"), mime, purpose="tool_result", + expect=expect, role="tool", tool_call_id=tool_use_id, + message_index=min(message_index, 65535), part_index=part_index, + error=error) + admitted += found is not None + except Exception: + return admitted + return admitted + + +def _recording(span: Any) -> bool: + try: + return bool(span.is_recording()) + except Exception: + return False + + +def media_safe(value: Any, depth: int = 0) -> Any: + """A telemetry copy without inline media bytes: base64 fields of media blocks become ``"[omitted]"``. + + Used before tool responses are JSON-serialized and truncated, so bytes never + reach span attributes as text. Non-media values are returned unchanged. + """ + if depth > 6: + return value + if isinstance(value, list): + return [media_safe(item, depth + 1) for item in value] + if not isinstance(value, Mapping): + return value + media = value.get("type") in _MEDIA_TYPES + result = {} + for key, item in value.items(): + if media and key in ("data", "base64", "blob") and isinstance(item, str): + result[key] = "[omitted]" + elif media and key in ("source", "file") and isinstance(item, Mapping): + result[key] = {k: ("[omitted]" if k in ("data", "base64") and isinstance(v, str) else v) + for k, v in item.items()} + else: + result[key] = media_safe(item, depth + 1) + return result diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py index 5cdb1cc89..296812e27 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_client_wrapper.py @@ -14,8 +14,11 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from ._attachments import capture_tool_results, tool_result_ids from ._attributes import ClaudeAgentAttributes, ClaudeAgentSpanKind from ._hooks import ( + ToolDeferrals, + _active_tool_spans, create_pre_tool_use_hook, create_subagent_start_hook, create_post_tool_use_hook, @@ -23,6 +26,8 @@ clear_parent_span, set_mcp_servers, clear_active_tool_spans, + end_deferred_tool, + end_deferred_tool_spans, safe_json_serialize, ) @@ -166,12 +171,16 @@ def _flatten_content(self, content: Any) -> str: return str(content) -def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: +def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer, + deferrals: Optional[ToolDeferrals] = None) -> None: """Inject OpenTelemetry tracing hooks into ClaudeAgentOptions. Args: options: ClaudeAgentOptions instance tracer: OpenTelemetry tracer + deferrals: the client's deferral tables when tool-result media capture is on: while + its receive_response() conversation is live, a completed tool span stays + recording until its result is captured (it keeps the hook's end time) """ if not hasattr(options, "hooks"): return @@ -192,7 +201,8 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: from claude_agent_sdk import HookMatcher pre_hook = create_pre_tool_use_hook(tracer) - post_hook = create_post_tool_use_hook(tracer) + post_hook = (create_post_tool_use_hook(tracer, deferrals=deferrals) if deferrals is not None + else create_post_tool_use_hook(tracer)) start_hook = create_subagent_start_hook(tracer) # Insert at beginning to ensure we trace before user hooks @@ -213,13 +223,20 @@ def inject_tracing_hooks(options: Any, tracer: trace_api.Tracer) -> None: def wrap_claude_sdk_client( - original_class: Any, tracer: trace_api.Tracer + original_class: Any, + tracer: trace_api.Tracer, + attachments: Any = None, + trace_config: Any = None, + attachment_capture: Any = None, ) -> Any: """Wrap ClaudeSDKClient with tracing. Args: original_class: The original ClaudeSDKClient class tracer: OpenTelemetry tracer + attachments: the shared attachment manager for tool-result media (None: off) + trace_config: TraceConfig whose hide flags and PII redaction deny media capture + attachment_capture: CapturePolicy that can only add restrictions Returns: Wrapped class with tracing @@ -231,8 +248,13 @@ class TracedClaudeSDKClient(original_class): def __init__(self, *args: Any, **kwargs: Any): # Extract options for hook injection options = kwargs.get("options") or (args[0] if args else None) + # This client's deferred tool ends (tool-result media capture only). + deferrals = ToolDeferrals() if attachments is not None else None if options: - inject_tracing_hooks(options, tracer) + if deferrals is not None: + inject_tracing_hooks(options, tracer, deferrals=deferrals) + else: + inject_tracing_hooks(options, tracer) # Store MCP server configs for tool attribution mcp_servers = getattr(options, "mcp_servers", None) @@ -246,6 +268,7 @@ def __init__(self, *args: Any, **kwargs: Any): self._prompt: Optional[str] = None self._start_time: Optional[float] = None self._options = options + self._tool_deferrals = deferrals async def query(self, *args: Any, **kwargs: Any) -> Any: """Capture prompt and timestamp when query starts.""" @@ -267,6 +290,9 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: attributes=span_attributes, ) as conversation_span: set_parent_span(conversation_span) + deferrals = getattr(self, "_tool_deferrals", None) + if deferrals is not None: + deferrals.live += 1 # Initialize turn tracker turn_tracker = TurnTracker( @@ -277,10 +303,12 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: collected_messages: List[Dict[str, Any]] = [] result_usage: Optional[Dict[str, Any]] = None + message_index = 0 try: async for msg in messages: msg_type = type(msg).__name__ + message_index += 1 if msg_type == "AssistantMessage": # Start new turn @@ -295,6 +323,25 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: }) elif msg_type == "UserMessage": + # Tool-result media first, from the original typed blocks, + # before any text flattening or truncation. + # A tool span the PostToolUse hook completed is still recording + # (deferred end), so the media refs are projected on it. + if attachments is not None: + try: + capture_tool_results( + attachments, + msg, + conversation_span, + _active_tool_spans, + deferrals.ends if deferrals is not None else {}, + message_index=message_index, + trace_config=trace_config, + capture=attachment_capture, + ) + finally: + for tool_use_id in tool_result_ids(msg): + end_deferred_tool(deferrals, tool_use_id) # End current turn, mark next start turn_tracker.end_current_turn() turn_tracker.mark_next_start() @@ -349,6 +396,10 @@ async def receive_response(self) -> AsyncGenerator[Any, None]: finally: turn_tracker.end_current_turn() clear_parent_span() + if deferrals is not None: + # Only this conversation's deferred tool spans end here, at their hook's time. + deferrals.live -= 1 + end_deferred_tool_spans(deferrals) clear_active_tool_spans() def _build_span_attributes(self) -> Dict[str, Any]: @@ -414,6 +465,15 @@ def _extract_content(self, msg: Any) -> Optional[str]: parts.append(block.text) elif type(block).__name__ == "ToolResultBlock": result = getattr(block, "content", "") + if isinstance(result, list): + # Only text blocks: media bytes never become span text. + result = " ".join( + item.get("text", "") + for item in result + if isinstance(item, dict) and isinstance(item.get("text"), str) + ) + elif not isinstance(result, str): + result = "" parts.append(f"[Tool Result: {result[:100]}...]") return " ".join(parts) if parts else None @@ -488,4 +548,15 @@ async def __aenter__(self) -> "TracedClaudeSDKClient": async def __aexit__(self, *args: Any) -> None: await super().__aexit__(*args) + if attachments is not None and hasattr(original_class, "disconnect"): + + async def disconnect(self: Any, *args: Any, **kwargs: Any) -> Any: + """Disconnect; this client's tool spans still waiting for their result end now.""" + try: + return await super(TracedClaudeSDKClient, self).disconnect(*args, **kwargs) + finally: + end_deferred_tool_spans(getattr(self, "_tool_deferrals", None)) + + TracedClaudeSDKClient.disconnect = disconnect + return TracedClaudeSDKClient diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py index fa2c876b1..89a4f08f4 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_hooks.py @@ -7,12 +7,14 @@ import json import logging import time +from collections import OrderedDict from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional, Tuple from opentelemetry import trace as trace_api from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from ._attachments import media_safe from ._attributes import ( ClaudeAgentAttributes, ClaudeAgentSpanKind, @@ -26,6 +28,33 @@ # Key: tool_use_id, Value: (span, start_time, tool_name) _active_tool_spans: Dict[str, Tuple[Span, float, str]] = {} +_MAX_DEFERRED_ENDS = 256 +#: A deferred tool span ends at most this long after its hook, even if its result never arrives. +DEFERRED_END_TIMEOUT_S = 30.0 + + +class ToolDeferrals: + """One client's tool spans whose end waits for their result (tool-result media capture on). + + The SDK streams a tool result AFTER the PostToolUse hook, so while that client's + ``receive_response()`` conversation span is live, a completed tool span stays recording + until its ToolResultBlock has been captured (its media refs are projected on it), then + ends at the time the hook recorded. Outside a live conversation (``receive_messages()``, + or a hook that fires before ``receive_response()`` starts) the hook ends the span as + before. Each client has its own tables, so one conversation's end flushes only its own. + + ``ends``: tool_use_id -> (span, end_time_ns, timer handle or None), bounded and + time-limited. ``results_seen``: results streamed while their span was still open + (result-first), so PostToolUse ends those now. ``live``: open conversation spans. + """ + + __slots__ = ("ends", "results_seen", "live") + + def __init__(self) -> None: + self.ends: "OrderedDict[str, Tuple[Span, int, Any]]" = OrderedDict() + self.results_seen: "OrderedDict[str, bool]" = OrderedDict() + self.live = 0 + # Tools that launch a subagent: "Task" in older Claude Code releases, "Agent" in current ones SUBAGENT_TOOLS = ("Task", "Agent") @@ -262,11 +291,86 @@ async def pre_tool_use_hook( return pre_tool_use_hook -def create_post_tool_use_hook(tracer: trace_api.Tracer) -> Callable: +def _end_deferred(entry: Tuple[Span, int, Any]) -> None: + span, end_ns, handle = entry + try: + if handle is not None: + handle.cancel() + except Exception: + pass + try: + span.end(end_time=end_ns) + except Exception as e: + logger.debug(f"Failed to end deferred tool span: {e}") + + +def _expire_deferred(deferrals: ToolDeferrals, tool_use_id: str, entry: Tuple[Span, int, Any]) -> None: + if deferrals.ends.get(tool_use_id) is entry: + del deferrals.ends[tool_use_id] + _end_deferred(entry) + + +def _defer_tool_end(deferrals: ToolDeferrals, tool_use_id: str, span: Span, end_ns: int) -> bool: + """Keep a completed tool span recording until its result is captured. False: end it now.""" + if deferrals.results_seen.pop(tool_use_id, None): + return False # Its result was already captured on the open span. + if deferrals.live <= 0: + return False # No receive_response() conversation will capture its result: end now. + try: + import asyncio + + loop = asyncio.get_running_loop() + except Exception: + return False # No asyncio loop to bound the wait (another async backend): end now. + try: + previous = deferrals.ends.pop(tool_use_id, None) + if previous is not None: + _end_deferred(previous) + holder: List[Any] = [] + handle = loop.call_later(DEFERRED_END_TIMEOUT_S, + lambda: _expire_deferred(deferrals, tool_use_id, holder[0])) + entry = (span, end_ns, handle) + holder.append(entry) + deferrals.ends[tool_use_id] = entry + while len(deferrals.ends) > _MAX_DEFERRED_ENDS: + _, oldest = deferrals.ends.popitem(last=False) + _end_deferred(oldest) + return True + except Exception: + return False + + +def end_deferred_tool(deferrals: Optional[ToolDeferrals], tool_use_id: Any) -> None: + """End one of this client's deferred tool spans (its result was processed), at its hook's time.""" + if deferrals is None or not isinstance(tool_use_id, str): + return + entry = deferrals.ends.pop(tool_use_id, None) + if entry is not None: + _end_deferred(entry) + elif tool_use_id in _active_tool_spans: + deferrals.results_seen[tool_use_id] = True + while len(deferrals.results_seen) > _MAX_DEFERRED_ENDS: + deferrals.results_seen.popitem(last=False) + + +def end_deferred_tool_spans(deferrals: Optional[ToolDeferrals]) -> None: + """End every deferred tool span of ONE client at its recorded time (its conversation ended).""" + if deferrals is None: + return + deferrals.results_seen.clear() + while deferrals.ends: + _, entry = deferrals.ends.popitem(last=False) + _end_deferred(entry) + + +def create_post_tool_use_hook(tracer: trace_api.Tracer, deferrals: Optional[ToolDeferrals] = None) -> Callable: """Create a PostToolUse hook for completing tool traces. Args: tracer: OpenTelemetry tracer instance + deferrals: the client's deferral tables when tool-result media capture is on: while + its receive_response() conversation is live, the completed span stays recording + until its tool result is captured (it keeps the hook's end time) Returns: Async hook function compatible with Claude Agent SDK @@ -302,12 +406,14 @@ async def post_tool_use_hook( return {} span, start_time, _ = span_info + end_ns = time.time_ns() duration_ms = (time.time() - start_time) * 1000 - # Set output and duration + # Set output and duration. Inline media bytes never go through text serialization + # or truncation; the attachment path uploads them from the typed tool result. span.set_attribute( ClaudeAgentAttributes.TOOL_OUTPUT, - safe_json_serialize(tool_response), + safe_json_serialize(media_safe(tool_response)), ) span.set_attribute(ClaudeAgentAttributes.TOOL_DURATION_MS, duration_ms) @@ -341,8 +447,11 @@ async def post_tool_use_hook( }, ) - # End the span - span.end() + # End the span. With media capture on and a live conversation, the tool result + # (streamed after this hook) is captured on the still-recording span first; it + # ends at end_ns. + if not (deferrals is not None and _defer_tool_end(deferrals, tool_use_id, span, end_ns)): + span.end() # Clean up subagent span if this launched a subagent if tool_name in SUBAGENT_TOOLS: @@ -438,6 +547,9 @@ def clear_active_tool_spans() -> None: """ global _active_tool_spans, _subagent_spans + # Deferred tool spans belong to one client; its conversation flushes its own + # (end_deferred_tool_spans) before calling this. + # End any orphaned spans for tool_use_id, (span, start_time, tool_name) in _active_tool_spans.items(): try: diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py index 8322da2b7..74b3bff25 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_instrumentor.py @@ -20,6 +20,23 @@ _instruments = ("claude-agent-sdk >= 0.1.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class ClaudeAgentInstrumentor(BaseInstrumentor): """OpenTelemetry Instrumentor for Claude Agent SDK. @@ -97,6 +114,12 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider, ) + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + self._attachments = _attachment_manager(kwargs) + self._trace_config = kwargs.get("config") + self._attachment_capture = kwargs.get("attachment_capture") + # Try to import and patch Claude Agent SDK try: self._patch_claude_agent_sdk() @@ -126,9 +149,16 @@ def _patch_claude_agent_sdk(self) -> None: self._original_client_class = claude_agent_sdk.ClaudeSDKClient # Create wrapped class + attachments = getattr(self, "_attachments", None) + media = {} if attachments is None else dict( + attachments=attachments, + trace_config=getattr(self, "_trace_config", None), + attachment_capture=getattr(self, "_attachment_capture", None), + ) wrapped_class = wrap_claude_sdk_client( self._original_client_class, self._tracer, + **media, ) # Replace in module diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index 03d710941..f68184784 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -114,3 +114,26 @@ Direct `FirecrawlClient` and `AsyncFirecrawlClient` instances look methods up on ## Watchers `Firecrawl.watcher()` polls `get_crawl_status` from its own thread. Each poll produces a separate root span; a swallowed poll error still appears as ERROR. `AsyncFirecrawl.watcher()` polls in the iterating task, so its spans inherit that task's current parent. + +## Screenshots and media (attachments) + +Pass the shared manager: `FirecrawlInstrumentor().instrument(tracer_provider=..., attachments=manager)`. +For `scrape`, `crawl`, `batch_scrape` and their status calls, a returned +Document's `screenshot`, `actions.screenshots`, `audio` and `video` are +uploaded on the operation span when they are inline `data:` URIs (up to 16 +documents and 8 MiB per result). Firecrawl Cloud returns hosted URLs for +these: they are never fetched, so nothing is uploaded for them. No extra +request, crawl or browser session is started. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py index 389ff8edf..748c1a1b4 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -56,8 +56,30 @@ } +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class FirecrawlInstrumentor(BaseInstrumentor): - """Instrument Firecrawl v2 scraping, extraction, agent and browser operations.""" + """Instrument Firecrawl v2 scraping, extraction, agent and browser operations. + + With an enabled shared attachment manager (``attachments=``), inline + ``data:`` screenshots and media of returned Documents are uploaded. + Hosted URLs are never fetched. + """ __slots__ = ("_original_methods",) @@ -73,7 +95,9 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() tracer = FITracer(trace_api.get_tracer(__name__, __version__, tracer_provider), config=config) self._original_methods: Dict[Tuple[str, str, str], Any] = {} - self._state = _State() + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + self._state = _State(_attachment_manager(kwargs), kwargs.get("attachment_capture")) for module_name, class_name, is_async in ( (_SYNC_MODULE, _SYNC_CLASS, False), diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py b/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py new file mode 100644 index 000000000..8925ffd3a --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/_attachments.py @@ -0,0 +1,84 @@ +"""Firecrawl returned screenshots and media, through the shared attachment manager. + +A firecrawl-py v2 ``Document`` carries ``screenshot``, ``audio`` and ``video`` +strings, and browser ``actions`` may carry ``screenshots``. Only values that +are inline ``data:;base64,`` URIs are uploaded, at the operation span, +as tool results. Firecrawl Cloud returns hosted URLs for these: they are never +fetched (the SDK core has no approved remote retrieval yet), so they are +``source_unavailable``. Nothing here starts a crawl, a browser or a request. +""" + +from types import MappingProxyType +from typing import Any, Mapping + +__all__ = ("capture_firecrawl_media", "firecrawl_attachment_capabilities") + +#: Operations whose results carry Documents. +MEDIA_METHODS = frozenset(( + "scrape", "crawl", "batch_scrape", "wait_batch_scrape", "get_crawl_status", "get_batch_scrape_status", +)) +MAX_DOCUMENTS = 16 +MAX_RESULT_BYTES = 8 * 1024 * 1024 +_IMAGE = ("image/png", "image/jpeg", "image/webp") +_AUDIO = ("audio/mpeg", "audio/wav", "audio/ogg", "audio/webm", "audio/mp4", "audio/flac") +_VIDEO = ("video/mp4", "video/webm") +_EMPTY: Mapping[str, Any] = MappingProxyType({}) + +firecrawl_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": tuple(sorted(MEDIA_METHODS)), + "sources": ("inline_data_uri",), + "fields": ("screenshot", "actions.screenshots", "audio", "video"), + "unsupported": ("hosted_url", "remote_url", "images_links", "pdf_url"), + "max_documents": MAX_DOCUMENTS, + "max_decoded_bytes_per_result": MAX_RESULT_BYTES, +}) + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def _documents(method: str, result: Any) -> list: + if method == "scrape": + return [result] + data = _get(result, "data") + return list(data[:MAX_DOCUMENTS]) if isinstance(data, (list, tuple)) else [] + + +def capture_firecrawl_media(manager, span, method: str, result: Any, *, trace_config=None, + capture=None) -> Mapping[str, Any]: + """Capture inline media of one operation's returned Documents. Never raises.""" + if method not in MEDIA_METHODS: + return _EMPTY + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import ToolMedia + except ImportError: + return _EMPTY + try: + if not active(manager): + return _EMPTY + media = ToolMedia(manager, span, "firecrawl", trace_config=trace_config, capture=capture, + budget=MAX_RESULT_BYTES) + for index, document in enumerate(_documents(method, result)[:MAX_DOCUMENTS]): + values = [("screenshot", _get(document, "screenshot"), _IMAGE), + ("audio", _get(document, "audio"), _AUDIO), + ("video", _get(document, "video"), _VIDEO)] + shots = _get(_get(document, "actions"), "screenshots") + if isinstance(shots, (list, tuple)): + values.extend(("actions.screenshots.%d" % n, shot, _IMAGE) for n, shot in enumerate(shots[:16])) + for field, value, mimes in values: + if not media.usable: + break + if not isinstance(value, str) or not value.startswith("data:"): + continue # hosted URLs are never fetched + media.data_uri("document/%d/%s" % (index, field), lambda v=value: v, mimes, + purpose="tool_result", role="tool", message_index=index) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 4e4f6b75e..34e132a2e 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -107,8 +107,11 @@ class _State: instrumentor is removed. """ - def __init__(self) -> None: + def __init__(self, attachments: Any = None, attachment_capture: Any = None) -> None: self.enabled = True + # The shared attachment manager (None unless an enabled one was passed). + self.attachments = attachments + self.attachment_capture = attachment_capture def _span_kind(method_name: str) -> str: @@ -406,6 +409,13 @@ def _finish_ok( ) -> None: try: self._record_result(span, instance, result, kwargs) + if self._state.attachments is not None: + from traceai_firecrawl._attachments import capture_firecrawl_media + + capture_firecrawl_media( + self._state.attachments, span, self._method_name, result, + trace_config=self._config, capture=self._state.attachment_capture, + ) except Exception: logger.debug("Could not read %s result", self._span_name, exc_info=True) finally: diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 6c0c1b30a..450fede1e 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -369,3 +369,27 @@ path is exported either way. The example test runs `examples/agent_with_tools.py` as its own process, once alone and once with `traceai_mcp` instrumented. The suite turns off mcp-use's usage telemetry. + +## Tool media (attachments) + +`FutureAGICallback(attachments=manager)` (default: the tracer provider's +`attachment_manager`) uploads the typed base64 blocks mcp-use 1.7 makes from +an MCP tool result: `image` and `file` (inline resource blob), on the tool +span as `tool_result` occurrences with the tool call id (an `audio` block is +captured when it reaches the callback as a typed block; LangChain 1.x turns +audio tool content into a string first, so nothing is uploaded for it). At +most 64 blocks and 256 KiB of decoded bytes per result. +`attachment_capture=` only adds restrictions. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py new file mode 100644 index 000000000..00c864136 --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_attachments.py @@ -0,0 +1,69 @@ +"""Tool media of an mcp-use agent run, through the shared attachment manager. + +mcp-use 1.7 converts an MCP ``CallToolResult`` into LangChain content blocks +(``agents/adapters/langchain_adapter.py``): ``ImageContent`` becomes +``{"type": "image", "source_type": "base64", "data", "mime_type"}``, +``AudioContent`` an ``"audio"`` block and an ``EmbeddedResource`` blob a +``"file"`` block. Only those typed inline blocks are observed. Text, URLs, +resource URIs and resource links are never fetched or scraped. Each block is +preflighted before its base64 is read; at most 64 blocks and 256 KiB of +decoded bytes per tool result. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Optional + +__all__ = ("capture_tool_blocks", "mcp_use_attachment_capabilities") + +mcp_use_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tool.output",), + "sources": ("inline_base64",), + "content_types": ("image", "audio", "file"), + # LangChain stringifies tool content it does not know as a block (audio), so no typed block reaches callbacks. + "unsupported": ("url", "resource_uri", "resource_link", "local_path", "audio_stringified_by_langchain"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_EXPECT = {"image": "image", "audio": "audio", "file": None} + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else None + except Exception: + return None + + +def capture_tool_blocks(manager, span, content: Any, *, trace_config=None, capture=None, + tool_call_id: Optional[str] = None, error: bool = False) -> Mapping[str, Any]: + """Capture the typed base64 blocks of one tool result. Never raises; returns admitted positions.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return _EMPTY + try: + if not active(manager) or not isinstance(content, (list, tuple)): + return _EMPTY + media = ToolMedia(manager, span, "mcp_use", trace_config=trace_config, capture=capture) + for index, block in enumerate(content[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(block, "type") + if kind not in _EXPECT: + continue + # LangChain's v0 shape (source_type + data) and its v1 standard shape (base64). + if _get(block, "source_type") == "base64": + field = "data" + elif _get(block, "base64") is not None: + field = "base64" + else: + continue # url / id sources are never fetched + media.base64("tool/%d" % index, lambda b=block, f=field: _get(b, f), _get(block, "mime_type"), + purpose="tool_result", expect=_EXPECT[kind], role="tool", tool_call_id=tool_call_id, + part_index=index, error=error) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index 08b7e6348..a603ba26a 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -140,6 +140,8 @@ class _Run: chunks: int = 0 # A tool run's name as LangChain reported it (not capped or redacted). tool_name: Optional[str] = None + # The tool call id LangChain reported, when it is a safe token. + tool_call_id: Optional[str] = None def _describe(error: BaseException) -> str: @@ -329,6 +331,13 @@ class FutureAGICallback(BaseCallbackHandler): redact: secret strings to remove from everything written (for example an LLM key or an MCP server header the environment does not hold). + attachments: the shared attachment manager for tool media (images, + audio and inline resource blobs an MCP tool returned). Defaults + to ``tracer_provider.attachment_manager``. Capture is off unless + that manager is enabled; ``config`` hide flags and PII redaction + deny it. + attachment_capture: a ``fi_instrumentation.attachments.CapturePolicy`` + that can only add restrictions. """ raise_error = False @@ -345,6 +354,8 @@ def __init__( tracer_provider: Optional[trace_api.TracerProvider] = None, config: Optional[TraceConfig] = None, redact: Iterable[str] = (), + attachments: Any = None, + attachment_capture: Any = None, ) -> None: super().__init__() if config is None: @@ -373,6 +384,11 @@ def __init__( self._hide_outputs = bool(config.hide_outputs) self._pii = bool(config.pii_redaction) self._secrets = Secrets(secrets) + self._config = config + self._attachments = ( + attachments if attachments is not None else getattr(provider, "attachment_manager", None) + ) + self._attachment_capture = attachment_capture self._lock = threading.RLock() self._runs: Dict[UUID, _Run] = {} self._roots: "OrderedDict[UUID, _Root]" = OrderedDict() @@ -1042,6 +1058,9 @@ def on_tool_start( return if isinstance(raw_name, str): run.tool_name = raw_name + raw_call_id = kwargs.get("tool_call_id") + if isinstance(raw_call_id, str): + run.tool_call_id = raw_call_id if run.span is None: return arguments = _json(inputs) if isinstance(inputs, Mapping) else input_str @@ -1092,6 +1111,8 @@ def on_tool_end( ) except Exception: logger.debug("Could not classify the tool result", exc_info=True) + if run.span is not None and self._attachments is not None: + self._capture_tool_media(run, output, failure is not None) text = _tool_output_text(output) self._remember(root, [text], output=True) if failure is not None and root is not None: @@ -1115,6 +1136,16 @@ def on_tool_end( except Exception: logger.debug("mcp-use callback on_tool_end failed", exc_info=True) + def _capture_tool_media(self, run: _Run, output: Any, failed: bool) -> None: + """Upload the typed inline media blocks a tool returned (never URLs or paths).""" + try: + from traceai_mcp_use._attachments import capture_tool_blocks + except ImportError: + return + content = getattr(output, "content", output) + capture_tool_blocks(self._attachments, run.span, content, trace_config=self._config, + capture=self._attachment_capture, tool_call_id=run.tool_call_id, error=failed) + def on_tool_error( self, error: BaseException, diff --git a/python/frameworks/mcp/README.md b/python/frameworks/mcp/README.md index 8f1a00c87..5b344ade3 100644 --- a/python/frameworks/mcp/README.md +++ b/python/frameworks/mcp/README.md @@ -45,3 +45,49 @@ enabled is attached only if the reader is enabled when it consumes the item. An already-running request retains the context under which it started. Native MCP SDK tracing remains independent and can continue when this instrumentor is off, particularly with MCP 2.x. + +## Tool and resource media (attachments) + +`MCPInstrumentor` propagates context only and never captures media. To upload +the images, audio and inline resource blobs a server returns, wrap the client +session: + +```python +from traceai_mcp import wrap_mcp_client + +session = wrap_mcp_client(session, tracer_provider=trace_provider) +result = await session.call_tool("screenshot", {}) # unchanged SDK result +``` + +Each `call_tool` / `read_resource` gets an `mcp.tools/call` / +`mcp.resources/read` CLIENT span. `ImageContent.data`, `AudioContent.data` +and `BlobResourceContents.blob` (in an `EmbeddedResource` or a +`resources/read` result) are uploaded as `tool_result` occurrences: at most 64 +parts and 256 KiB of decoded bytes per result. `resource_link` blocks, +resource URIs and `structuredContent` are never read. A MIME of another +modality than the block type, or outside the attachment MIME list, uploads +nothing. So do bytes whose leading signature names another modality than the +declared MIME, for example a PNG labelled `audio/wav`. The signatures checked +are PNG, JPEG, GIF, WebP, BMP, TIFF, the HEIF/AVIF image brands of an ISO-BMFF +`ftyp` box (`avif`, `heic`, `heix`, `mif1`, `msf1`), WAV, Ogg, FLAC, MP3 with +an ID3 tag, and PDF. Other bytes, including MP4 and WebM containers and MP3 +without an ID3 tag, are judged by the declared MIME only. +`application/octet-stream` is the generic file modality, so an octet-stream +blob whose bytes carry one of those image, audio or PDF signatures is skipped +rather than relabelled. `attachments=` overrides the provider's manager; `trace_config=` and +`attachment_capture=` only add restrictions. Wrapping a wrapped session +returns it unchanged. `capture_mcp_result(manager, span, result, operation)` +is the same capture for an existing span. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/mcp/traceai_mcp/__init__.py b/python/frameworks/mcp/traceai_mcp/__init__.py index 1a8e685e3..d2b3bf458 100644 --- a/python/frameworks/mcp/traceai_mcp/__init__.py +++ b/python/frameworks/mcp/traceai_mcp/__init__.py @@ -8,12 +8,19 @@ from opentelemetry import context, propagate from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from traceai_mcp.package import _instruments +from traceai_mcp._attachments import capture_mcp_result, mcp_attachment_capabilities, wrap_mcp_client from wrapt import ObjectProxy, register_post_import_hook, wrap_function_wrapper +__all__ = ["MCPInstrumentor", "capture_mcp_result", "mcp_attachment_capabilities", "wrap_mcp_client"] + + class MCPInstrumentor(BaseInstrumentor): # type: ignore """ An instrumenter for MCP. + + It propagates trace context over MCP transports only and never captures + media. Use ``wrap_mcp_client(session)`` for tool and resource media. """ def instrumentation_dependencies(self) -> Collection[str]: diff --git a/python/frameworks/mcp/traceai_mcp/_attachments.py b/python/frameworks/mcp/traceai_mcp/_attachments.py new file mode 100644 index 000000000..1851e2fee --- /dev/null +++ b/python/frameworks/mcp/traceai_mcp/_attachments.py @@ -0,0 +1,199 @@ +"""MCP tool and resource media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/mcp`` (``wrapMCPClient`` / ``captureMCPResult``). Only typed +inline bytes are observed: ``ImageContent.data``, ``AudioContent.data`` and an +``EmbeddedResource`` (or ``resources/read``) ``BlobResourceContents.blob``. +Resource URIs, ``resource_link`` blocks, URLs, local paths and +``structuredContent`` are never fetched, opened or scraped. Each part is +preflighted before its base64 is read or decoded; at most 64 parts and +256 KiB of decoded bytes per result. + +This module imports ``fi_instrumentation.attachments`` lazily, so the +transport-only ``MCPInstrumentor`` keeps working on a core without it. +""" + +import functools +import inspect +from types import MappingProxyType +from typing import Any, Mapping, Optional + +from opentelemetry import context as otel_context +from opentelemetry import trace as trace_api +from opentelemetry.trace import SpanKind, Status, StatusCode +from wrapt import ObjectProxy + +from traceai_mcp.version import __version__ + +__all__ = ("capture_mcp_result", "wrap_mcp_client", "mcp_attachment_capabilities") + +#: Declarative handoff for auto and reconciliation. +mcp_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("tools/call.output", "resources/read.output"), + "sources": ("inline_base64",), + "content_types": ("image", "audio", "resource.blob"), + "unsupported": ("resource_link", "resource_uri", "remote_url", "local_path", "structured_content", + "transport_only_instrumentor"), + "max_parts": 64, + "max_decoded_bytes_per_result": 256 * 1024, +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_OPERATIONS = ("tools/call", "resources/read") + + +def _get(value: Any, *names: str) -> Any: + """A field of an MCP model (1.x camelCase or 2.x snake_case) or a plain dict; never raises.""" + for name in names: + try: + found = value.get(name) if isinstance(value, Mapping) else getattr(value, name, None) + except Exception: + found = None + if found is not None: + return found + return None + + +def _active(manager) -> bool: + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return False + return active(manager) + + +def capture_mcp_result(manager, span, result: Any, operation: str = "tools/call", capture=None, + trace_config=None) -> Mapping[str, Any]: + """Capture typed inline media of one MCP ``tools/call`` or ``resources/read`` result. + + ``span`` is the span that represents this call. Tool media is tool output + and potential model input, so input and output hide flags both apply. + Returns the admitted positions (``"/"`` -> occurrence); + empty when no manager, disabled, unsampled or denied. Never raises and + never changes ``result``. + """ + if operation not in _OPERATIONS or not _active(manager): + return _EMPTY + try: + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return _EMPTY + try: + media = ToolMedia(manager, span, "mcp", trace_config=trace_config, capture=capture) + if not media.usable: + return _EMPTY + error = _get(result, "isError", "is_error") is True + parts = _get(result, "content" if operation == "tools/call" else "contents") + if not isinstance(parts, (list, tuple)): + return _EMPTY + for index, part in enumerate(parts[:MAX_TOOL_PARTS]): + if not media.usable: + break + kind = _get(part, "type") + if operation == "resources/read": + holder, field, expect = part, "blob", None + elif kind in ("image", "audio"): + holder, field, expect = part, "data", kind + elif kind == "resource": + holder, field, expect = _get(part, "resource"), "blob", None + else: + continue # text, resource_link and unknown blocks are never dereferenced + if holder is None: + continue + mime = _get(holder, "mimeType", "mime_type") + media.base64("%s/%d" % (operation, index), lambda h=holder, f=field: _get(h, f), mime, + purpose="tool_result", expect=expect, role="tool", part_index=index, error=error) + captured = media.captured + return MappingProxyType(captured) if captured else _EMPTY + except Exception: + return _EMPTY # Malformed or untrusted results must not change the tool response. + + +def _provider_manager(tracer_provider) -> Any: + owner = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + manager = getattr(owner, "attachment_manager", None) + if manager is None: + delegate = getattr(owner, "_real_tracer_provider", None) or getattr(owner, "_tracer_provider", None) + manager = getattr(delegate, "attachment_manager", None) + return manager + + +class _TracedSession(ObjectProxy): # type: ignore[misc] + """Proxies an MCP ``ClientSession``; only ``call_tool`` and ``read_resource`` gain a span.""" + + def __init__(self, wrapped, config): + super().__init__(wrapped) + self._self_config = config + + def _self_traced(self, operation: str, method, args, kwargs): + tracer_provider, attachments, trace_config, capture = self._self_config + + @functools.wraps(method) + async def run(): + span = None + manager = None + try: + manager = attachments if attachments is not None else _provider_manager(tracer_provider) + provider = tracer_provider if tracer_provider is not None else trace_api.get_tracer_provider() + tracer = provider.get_tracer("traceai_mcp", __version__) + span = tracer.start_span("mcp.%s" % operation, kind=SpanKind.CLIENT, + attributes={"gen_ai.span.kind": "TOOL", "mcp.method.name": operation}) + except Exception: + span = None # Tracing never prevents the SDK call. + if span is None: + return await method(*args, **kwargs) + token = otel_context.attach(trace_api.set_span_in_context(span)) + try: + result = await method(*args, **kwargs) + except BaseException: + otel_context.detach(token) + try: + span.set_status(Status(StatusCode.ERROR)) + finally: + span.end() + raise + otel_context.detach(token) + try: + if manager is not None: + capture_mcp_result(manager, span, result, operation, capture, trace_config) + failed = _get(result, "isError", "is_error") is True + span.set_status(Status(StatusCode.ERROR if failed else StatusCode.OK)) + except Exception: + pass # Never log SDK results or exceptions. + finally: + span.end() + return result + + return run() + + def call_tool(self, *args, **kwargs): + method = self.__wrapped__.call_tool + if not inspect.iscoroutinefunction(method): + return method(*args, **kwargs) + return self._self_traced("tools/call", method, args, kwargs) + + def read_resource(self, *args, **kwargs): + method = self.__wrapped__.read_resource + if not inspect.iscoroutinefunction(method): + return method(*args, **kwargs) + return self._self_traced("resources/read", method, args, kwargs) + + +def wrap_mcp_client(session, *, tracer_provider=None, attachments=None, trace_config=None, + attachment_capture=None): + """Trace a ``mcp.ClientSession``'s ``call_tool`` and ``read_resource`` and capture their media. + + ``attachments`` defaults to the provider's shared manager + (``tracer_provider.attachment_manager``, as ``fi_instrumentation.register`` + and ``traceai.init`` set it); no uploader is created here. ``trace_config`` + and ``attachment_capture`` can only add restrictions. The SDK result, + exception and cancellation are returned unchanged. Wrapping a traced session + again returns it unchanged, so spans never nest and media is never uploaded + twice. + """ + # A traced session is returned as is, so spans never nest and media is never captured twice. + # (ObjectProxy delegates ==/hash to the session, so identity caches would see them as equal.) + if isinstance(session, _TracedSession): + return session + proxy = _TracedSession(session, (tracer_provider, attachments, trace_config, attachment_capture)) + return proxy diff --git a/python/frameworks/openai-agents/README.md b/python/frameworks/openai-agents/README.md index bba264856..2712bfe36 100644 --- a/python/frameworks/openai-agents/README.md +++ b/python/frameworks/openai-agents/README.md @@ -59,3 +59,33 @@ print(result.final_output) ``` +## Tool media and computer-use screenshots (attachments) + +Pass the shared manager: `OpenAIAgentsInstrumentor().instrument(tracer_provider=..., attachments=manager)` +(`traceai.init` passes it for you). On the tool's function span, before +`output.value` is written: + +- the computer tool's screenshot (`computer_screenshot`), which the SDK records + on the `computer` function span from openai-agents 0.10.0; +- `ToolOutputImage` / `{"type": "image", "image_url": "data:..."}` and + `ToolOutputFileContent` with `file_data` (openai-agents 0.4.0+; MCP image + results become these from 0.8.0), as `tool_result`. + +Admitted values become `fi-attachment:` in `output.value`. +Nothing is recorded or uploaded when `RunConfig.trace_include_sensitive_data` +is off. Model-call input media is left to the OpenAI instrumentor's own span: +with both enabled, a screenshot is uploaded once as the tool output and once +as the next model input. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/openai-agents/traceai_openai_agents/__init__.py b/python/frameworks/openai-agents/traceai_openai_agents/__init__.py index fa114bfe7..b5f5e0edc 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/__init__.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/__init__.py @@ -12,6 +12,23 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class OpenAIAgentsInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for openai-agents @@ -36,15 +53,22 @@ def _instrument(self, **kwargs: Any) -> None: from traceai_openai_agents._processor import FiTracingProcessor - + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + processor = FiTracingProcessor( + cast(Tracer, tracer), + attachments=_attachment_manager(kwargs), + trace_config=config, + attachment_capture=kwargs.get("attachment_capture"), + ) if exclusive_processor: from agents import set_trace_processors - set_trace_processors([FiTracingProcessor(cast(Tracer, tracer))]) + set_trace_processors([processor]) else: from agents import add_trace_processor - add_trace_processor(FiTracingProcessor(cast(Tracer, tracer))) + add_trace_processor(processor) def _uninstrument(self, **kwargs: Any) -> None: # TODO : OpenAI Agents does not support uninstrumentation currently diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py b/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py new file mode 100644 index 000000000..6b318edb6 --- /dev/null +++ b/python/frameworks/openai-agents/traceai_openai_agents/_attachments.py @@ -0,0 +1,110 @@ +"""Tool media of OpenAI Agents SDK function spans, through the shared attachment manager. + +The SDK records what a tool returned on its ``FunctionSpanData.output`` (only +when ``RunConfig.trace_include_sensitive_data`` is on). These typed outputs +are observed, at the tool span, before ``OUTPUT_VALUE`` is written: + +- the computer tool's screenshot: a ``data:image/png;base64,`` string on the + ``computer`` function span (SDK 0.10.0 and later record it there); +- ``ToolOutputImage`` / ``{"type": "image", "image_url": }`` + (SDK 0.4.0+; MCP image results become these from 0.8.0); +- ``ToolOutputFileContent`` / ``{"type": "file", "file_data": }``; +- a list of those. + +Remote URLs, ``file_id`` values and plain strings on other tools are never +fetched or scraped. Model-call input media (``ResponseSpanData.input``) is +left to the OpenAI provider adapter, which owns that span. Admitted values +become ``fi-attachment:`` in the span's ``OUTPUT_VALUE``; the +SDK's own objects are never changed. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Tuple + +__all__ = ("capture_function_output", "openai_agents_attachment_capabilities") + +COMPUTER_TOOL_NAMES = ("computer", "computer_use_preview") +_IMAGE_MIMES = ("image/png", "image/jpeg", "image/webp") +_FILE_MIMES = ("application/pdf", "application/octet-stream") +_EMPTY: Mapping[str, Any] = MappingProxyType({}) + +openai_agents_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("function.output", "computer.screenshot"), + "sources": ("inline_data_uri",), + "minimum_sdk": {"computer.screenshot": "0.10.0", "tool_output_image": "0.4.0", "mcp_image": "0.8.0"}, + "unsupported": ("remote_url", "file_id", "response_input", "trace_include_sensitive_data_off"), +}) + + +def _field(value: Any, key: str) -> Any: + try: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + except Exception: + return None + + +def _typed(value: Any) -> bool: + return isinstance(value, Mapping) or type(value).__name__ in ("ToolOutputImage", "ToolOutputFileContent") + + +def capture_function_output(manager, span, name: Any, output: Any, *, trace_config=None, + capture=None) -> Tuple[Mapping[str, Any], Any]: + """Capture one function span's typed media; returns (admitted positions, telemetry output). + + The telemetry output is ``output`` itself when nothing was admitted, else a + copy where only the admitted values are ``fi-attachment:`` URIs. Never raises. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia, uri + except ImportError: + return _EMPTY, output + try: + if not active(manager) or output is None: + return _EMPTY, output + media = ToolMedia(manager, span, "openai_agents", trace_config=trace_config, capture=capture) + if not media.usable: + return _EMPTY, output + + def item(value, index): + position = "function/%d" % index + if isinstance(value, str): + if name in COMPUTER_TOOL_NAMES and value.startswith("data:image/"): + found = media.data_uri(position, lambda: value, _IMAGE_MIMES, purpose="computer_screenshot", + role="tool", part_index=index) + return uri(found) if found else value + return value + if not _typed(value): + return value + kind = _field(value, "type") + if kind == "image": + key, mimes = "image_url", _IMAGE_MIMES + elif kind == "file": + key, mimes = "file_data", _FILE_MIMES + else: + return value + encoded = _field(value, key) + if not isinstance(encoded, str) or not encoded.startswith("data:"): + return value # URLs and file ids are never fetched + found = media.data_uri(position, lambda: encoded, mimes, purpose="tool_result", role="tool", + part_index=index) + if not found: + return value + plain = dict(value) if isinstance(value, Mapping) else value.model_dump(exclude_none=True) + plain[key] = uri(found) + return plain + + if isinstance(output, list): + projected = [item(value, index) if index < MAX_TOOL_PARTS else value + for index, value in enumerate(output)] + else: + projected = item(output, 0) + captured = media.captured + if not captured: + return _EMPTY, output + return MappingProxyType(captured), projected + except Exception: + return _EMPTY, output diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py index 48684260b..ea23b690c 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py @@ -67,8 +67,19 @@ class FiTracingProcessor(TracingProcessor): _MAX_HANDOFFS_IN_FLIGHT = 1000 - def __init__(self, tracer: Tracer) -> None: + def __init__( + self, + tracer: Tracer, + *, + attachments: Any = None, + trace_config: Any = None, + attachment_capture: Any = None, + ) -> None: self._tracer = tracer + # The shared attachment manager (None unless an enabled one was passed). + self._attachments = attachments + self._trace_config = trace_config + self._attachment_capture = attachment_capture self._root_spans: dict[str, OtelSpan] = {} self._otel_spans: dict[str, OtelSpan] = {} self._tokens: dict[str, object] = {} @@ -227,7 +238,19 @@ def _set_span_attributes(self, span: Span[Any], otel_span: OtelSpan) -> None: for k, v in _get_attributes_from_generation_span_data(data): otel_span.set_attribute(k, v) elif isinstance(data, FunctionSpanData): - for k, v in _get_attributes_from_function_span_data(data): + output = data.output + if self._attachments is not None and output is not None: + from traceai_openai_agents._attachments import capture_function_output + + _, output = capture_function_output( + self._attachments, + otel_span, + data.name, + output, + trace_config=self._trace_config, + capture=self._attachment_capture, + ) + for k, v in _get_attributes_from_function_span_data(data, output): otel_span.set_attribute(k, v) elif isinstance(data, MCPListToolsSpanData): for k, v in _get_attributes_from_mcp_list_tool_span_data(data): @@ -560,20 +583,27 @@ def _convert_to_primitive(value: Any) -> Union[bool, str, bytes, int, float]: return str(value) +_UNSET = object() + + def _get_attributes_from_function_span_data( obj: FunctionSpanData, + output: Any = _UNSET, ) -> Iterator[tuple[str, AttributeValue]]: + # ``output`` is a telemetry copy in which admitted media are fi-attachment URIs. + if output is _UNSET: + output = obj.output yield GEN_AI_TOOL_NAME, obj.name if obj.input: yield INPUT_VALUE, obj.input yield INPUT_MIME_TYPE, JSON - if obj.output is not None: - yield OUTPUT_VALUE, _convert_to_primitive(obj.output) + if output is not None: + yield OUTPUT_VALUE, _convert_to_primitive(output) if ( - isinstance(obj.output, str) - and len(obj.output) > 1 - and obj.output[0] == "{" - and obj.output[-1] == "}" + isinstance(output, str) + and len(output) > 1 + and output[0] == "{" + and output[-1] == "}" ): yield OUTPUT_MIME_TYPE, JSON diff --git a/python/frameworks/smolagents/README.md b/python/frameworks/smolagents/README.md index f942a29d0..4a59348c1 100644 --- a/python/frameworks/smolagents/README.md +++ b/python/frameworks/smolagents/README.md @@ -80,3 +80,33 @@ manager_agent.run( ) ``` +## Image observations (attachments) + +Pass the shared manager: `SmolagentsInstrumentor().instrument(tracer_provider=..., attachments=manager)`. + +- Images smolagents hands to a model (`{"type": "image", "image": }` + blocks: task images and `ActionStep.observations_images`, such as a browser + screenshot a step callback stored) are uploaded on the model span as + `model_input`, once per image object (memory replays are not re-uploaded). + Every model call that carries a captured image, new or replayed, tells + provider instrumentors below it that the call's media is already owned, so + a replayed image is never uploaded again from the provider request. +- A tool's returned in-memory PIL image or `AgentImage` is uploaded on the + tool span as `tool_result`. + +Images are PNG-encoded only after the manager admitted them, up to +2048 x 2048 pixels. File-backed images that are not loaded yet, image URLs +and every `AgentAudio` are never read. + +Attachments are off unless the provider's attachment manager is enabled +(`fi_instrumentation.register(..., attachment_config=AttachmentConfig(enabled=True))` +or `traceai.init(attachment_config=...)`). Tool media is tool output and +potential model input, so every input and output hide flag +(`hide_inputs`, `hide_outputs` and their message/text variants), +`hide_input_images` for images, and `pii_redaction` deny the upload; they +cannot be relaxed by the server or the core. Only typed inline bytes are +read, and only after the manager admitted them; URLs, file IDs, resource URIs +and local paths are never fetched or opened. Call the manager's `flush()` +before a short process exits. Upload completion means "transferred for +validation"; playback needs an authorized occurrence through the media +gateway. diff --git a/python/frameworks/smolagents/traceai_smolagents/__init__.py b/python/frameworks/smolagents/traceai_smolagents/__init__.py index e6aac0cdd..642e55863 100644 --- a/python/frameworks/smolagents/traceai_smolagents/__init__.py +++ b/python/frameworks/smolagents/traceai_smolagents/__init__.py @@ -4,6 +4,7 @@ from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from traceai_smolagents._wrappers import ( + _Media, _ModelWrapper, _RunWrapper, _StepWrapper, @@ -16,6 +17,23 @@ _instruments = ("smolagents >= 1.2.2",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class SmolagentsInstrumentor(BaseInstrumentor): __slots__ = ( "_original_run_method", @@ -43,6 +61,9 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + # The provider-owned shared manager, passed as ``attachments`` (auto also passes + # ``attachment_manager``). Nothing is captured unless an enabled manager is supplied. + media = _Media(_attachment_manager(kwargs), config, kwargs.get("attachment_capture")) run_wrapper = _RunWrapper(tracer=self._tracer) self._original_run_method = getattr(MultiStepAgent, "run", None) @@ -72,9 +93,10 @@ def _instrument(self, **kwargs: Any) -> None: if method not in vars(model_subclass): continue self._original_model_call_methods[(model_subclass, method)] = vars(model_subclass)[method] - wrap_function_wrapper(model_subclass, method, _ModelWrapper(tracer=self._tracer, method_name=method)) + wrap_function_wrapper(model_subclass, method, + _ModelWrapper(tracer=self._tracer, method_name=method, media=media)) - tool_call_wrapper = _ToolCallWrapper(tracer=self._tracer) + tool_call_wrapper = _ToolCallWrapper(tracer=self._tracer, media=media) self._original_tool_call_method = getattr(Tool, "__call__", None) wrap_function_wrapper( module="smolagents", diff --git a/python/frameworks/smolagents/traceai_smolagents/_attachments.py b/python/frameworks/smolagents/traceai_smolagents/_attachments.py new file mode 100644 index 000000000..5058e85b3 --- /dev/null +++ b/python/frameworks/smolagents/traceai_smolagents/_attachments.py @@ -0,0 +1,164 @@ +"""smolagents typed media observations, through the shared attachment manager. + +Observed, and nothing else: + +- images smolagents hands to a model: ``{"type": "image", "image": }`` + content blocks (task images and ``ActionStep.observations_images``, such as + a browser screenshot a step callback stored), on the model span as model + input, before the model converts them; +- a tool's returned image: an in-memory ``PIL.Image.Image`` or an + ``AgentImage`` whose image is already in memory, on the tool span. + +Images are PNG-encoded only after a pixel-bounded preflight admitted them. +A file-backed ``AgentImage`` or PIL image that is not loaded yet, image URLs, +``image_url`` strings and every ``AgentAudio`` (a path or a tensor) are not +read (``source_unavailable``). smolagents replays memory images on every +model call; the same image object is uploaded once per manager, and every +model call that carries a captured image (new or replayed) is marked as the +owner of its media so a provider adapter below never uploads it again. +""" + +import os +import threading +import weakref +from collections import OrderedDict +from types import MappingProxyType +from typing import Any, Mapping + +__all__ = ("capture_model_images", "capture_tool_image", "owner_context", "smolagents_attachment_capabilities") + +smolagents_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.input.image", "tool.output.image"), + "sources": ("in_memory_pil_image",), + "unsupported": ("agent_audio", "file_backed_image", "image_url", "tensor"), +}) + +_EMPTY: Mapping[str, Any] = MappingProxyType({}) +_MAX_SEEN = 1024 +_MAX_MESSAGES = 256 + + +class _Seen: + """Image objects already uploaded, per manager (identity only; PIL images are unhashable).""" + + def __init__(self) -> None: + self._reset() + + def _reset(self) -> None: + self._pid = os.getpid() + self._lock = threading.Lock() + self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() + + def _owned_lock(self) -> threading.Lock: + # A forked child gets a fresh lock and table: a lock another thread held at fork + # time would never be released there. No fork hook is registered. + if self._pid != os.getpid(): + self._reset() + return self._lock + + def __contains__(self, item) -> bool: + manager, image = item + with self._owned_lock(): + entry = self._managers.get(manager, {}).get(id(image)) + return entry is not None and entry() is image + + def add(self, manager, image) -> None: + with self._owned_lock(): + seen = self._managers.get(manager) + if seen is None: + seen = self._managers[manager] = OrderedDict() + seen[id(image)] = weakref.ref(image) + while len(seen) > _MAX_SEEN: + seen.popitem(last=False) + + +_SEEN = _Seen() + + +def _get(value: Any, key: str) -> Any: + try: + return value.get(key) if isinstance(value, Mapping) else getattr(value, key, None) + except Exception: + return None + + +def _in_memory(value: Any) -> Any: + """The PIL image an ``AgentImage`` already holds in memory, else the value itself.""" + if type(value).__name__ == "AgentImage": + raw = getattr(value, "_raw", None) + return raw # None for path- or tensor-backed values: never loaded here + return value + + +def capture_model_images(manager, span, messages: Any, *, trace_config=None, capture=None) -> int: + """Capture the in-memory images of one model call's input messages. + + Returns the number of images this call owns: admitted now, or already uploaded + (a memory replay). A positive count means the caller marks the model call as owner. + """ + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import MAX_TOOL_PARTS, ToolMedia + except ImportError: + return 0 + admitted = 0 + try: + if not active(manager) or not isinstance(messages, (list, tuple)): + return 0 + media = ToolMedia(manager, span, "smolagents", trace_config=trace_config, capture=capture, + budget=8 * 1024 * 1024, direction="input") + for message_index, message in enumerate(messages[:_MAX_MESSAGES]): + content = _get(message, "content") + if not isinstance(content, list): + continue + for part_index, part in enumerate(content[:MAX_TOOL_PARTS]): + if not media.usable: + return admitted + if _get(part, "type") != "image": + continue + image = _in_memory(_get(part, "image")) + if image is None: + continue + if (manager, image) in _SEEN: + admitted += 1 # Replayed: already uploaded, still owned here. + continue + found = media.image("model/%d/%d" % (message_index, part_index), image, purpose="model_input", + role="user", message_index=message_index, part_index=part_index) + if found is not None: + _SEEN.add(manager, image) + admitted += 1 + except Exception: + return admitted + return admitted + + +def owner_context(): + """The current context marked as owner of this call's media (by the shared key name).""" + from fi_instrumentation.attachments.tool_media import with_ownership + + return with_ownership() + + +def capture_tool_image(manager, span, response: Any, *, trace_config=None, capture=None) -> int: + """Capture a tool's returned in-memory image on its span. Returns 1 when admitted.""" + try: + from fi_instrumentation.attachments.media_capture import active + from fi_instrumentation.attachments.tool_media import ToolMedia + except ImportError: + return 0 + try: + if not active(manager): + return 0 + image = _in_memory(response) + if image is None or (manager, image) in _SEEN: + return 0 + media = ToolMedia(manager, span, "smolagents", trace_config=trace_config, capture=capture, + budget=8 * 1024 * 1024) + found = media.image("tool/0", image, purpose="tool_result", role="tool", part_index=0) + if found is None: + return 0 + _SEEN.add(manager, image) + return 1 + except Exception: + return 0 diff --git a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py index 0c3cd0cb1..5897279b3 100644 --- a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py +++ b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py @@ -418,10 +418,20 @@ def _input_value_and_mime_type( yield INPUT_VALUE, safe_json_dumps(arguments) +class _Media: + """The shared attachment manager and its local restrictions (None manager: capture off).""" + + __slots__ = ("manager", "trace_config", "capture") + + def __init__(self, manager: Any = None, trace_config: Any = None, capture: Any = None) -> None: + self.manager, self.trace_config, self.capture = manager, trace_config, capture + + class _ModelWrapper: - def __init__(self, tracer: trace_api.Tracer, method_name: str = "__call__") -> None: + def __init__(self, tracer: trace_api.Tracer, method_name: str = "__call__", media: Any = None) -> None: self._tracer = tracer self._method_name = method_name + self._media = media if media is not None else _Media() def __call__( self, @@ -446,7 +456,20 @@ def __call__( **dict(_get_raw_input(args, **kwargs)), }, ) as span: - output_message = wrapped(*args, **kwargs) + owner_token = None + if self._media.manager is not None: + from traceai_smolagents._attachments import capture_model_images, owner_context + + if capture_model_images(self._media.manager, span, arguments.get("messages"), + trace_config=self._media.trace_config, capture=self._media.capture): + # These images are captured here (now or on an earlier call); a provider + # adapter below must not upload them again. + owner_token = context_api.attach(owner_context()) + try: + output_message = wrapped(*args, **kwargs) + finally: + if owner_token is not None: + context_api.detach(owner_token) span.set_status(trace_api.StatusCode.OK) span.set_attribute(OUTPUT_VALUE, _get_raw_output(output_message)) # Modern SDKs return usage on ChatMessage; historical releases @@ -473,8 +496,9 @@ def __call__( class _ToolCallWrapper: - def __init__(self, tracer: trace_api.Tracer) -> None: + def __init__(self, tracer: trace_api.Tracer, media: Any = None) -> None: self._tracer = tracer + self._media = media if media is not None else _Media() def __call__( self, @@ -501,6 +525,11 @@ def __call__( }, ) as span: response = wrapped(*args, **kwargs) + if self._media.manager is not None: + from traceai_smolagents._attachments import capture_tool_image + + capture_tool_image(self._media.manager, span, response, + trace_config=self._media.trace_config, capture=self._media.capture) span.set_attribute(OUTPUT_VALUE, _get_raw_output(response)) span.set_status(trace_api.StatusCode.OK) span.set_attributes( diff --git a/python/tests/attachments_mcp_agents/_fake_claude_cli.py b/python/tests/attachments_mcp_agents/_fake_claude_cli.py new file mode 100644 index 000000000..83d83f4b0 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_fake_claude_cli.py @@ -0,0 +1,110 @@ +"""A stand-in for the Claude Code CLI that speaks the SDK's stream-json control protocol. + +The real ``claude_agent_sdk`` spawns this through ``ClaudeAgentOptions(cli_path=...)``. +For every user message it replays one tool call: the assistant ``tool_use``, the +PreToolUse and PostToolUse hook callbacks the SDK registered, the ``tool_result`` +carrying the media in ``FAKE_CLAUDE_RESULT`` (a JSON file), then a final answer. +``FAKE_CLAUDE_ORDER=result-first`` streams the tool result before PostToolUse; +``no-result`` never streams it. ``FAKE_CLAUDE_GAP`` (seconds) waits between +PostToolUse and the tool result; ``FAKE_CLAUDE_HOLD`` (a path) also waits there, +until that file exists (at most 20 s). ``FAKE_TOOL_ID`` names the tool call. No +model, network or file other than the result and hold files is touched. +""" + +import json +import os +import sys +import time + +if "-v" in sys.argv[1:] or "--version" in sys.argv[1:]: + print("2.1.999 (Claude Code)") + sys.exit(0) + +with open(os.environ["FAKE_CLAUDE_RESULT"]) as handle: + TOOL_RESULT = json.load(handle) +ORDER = os.environ.get("FAKE_CLAUDE_ORDER", "hooks-first") +GAP = float(os.environ.get("FAKE_CLAUDE_GAP", "0")) +HOLD = os.environ.get("FAKE_CLAUDE_HOLD", "") +TOOL_ID = os.environ.get("FAKE_TOOL_ID", "toolu_screen_1") +TOOL = "mcp__screen__capture" +hooks = {} +counter = [0] + + +def send(message): + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +def wait_response(request_id): + for line in sys.stdin: + message = json.loads(line) + if message.get("type") == "control_response": + if message.get("response", {}).get("request_id") == request_id: + return message + else: + handle(message) + + +def hook(event, tool_use_id, extra): + for callback_id in hooks.get(event, []): + counter[0] += 1 + request_id = "cli_%d" % counter[0] + send({"type": "control_request", "request_id": request_id, "request": { + "subtype": "hook_callback", "callback_id": callback_id, "tool_use_id": tool_use_id, + "input": dict({"hook_event_name": event, "session_id": "s1", "tool_name": TOOL, "tool_input": {}, + "cwd": "/", "transcript_path": ""}, **extra)}}) + wait_response(request_id) + + +def assistant(content): + send({"type": "assistant", "parent_tool_use_id": None, "session_id": "s1", "message": { + "id": "msg_%d" % counter[0], "type": "message", "role": "assistant", "model": "claude-test", + "content": content, "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 1}}}) + + +def turn(): + tool_use_id = TOOL_ID + time.sleep(0.05) # Model latency: the client is iterating receive_response() before any hook. + assistant([{"type": "tool_use", "id": tool_use_id, "name": TOOL, "input": {}}]) + hook("PreToolUse", tool_use_id, {}) + result = {"type": "user", "parent_tool_use_id": None, "session_id": "s1", "message": { + "role": "user", "content": [dict(TOOL_RESULT, type="tool_result", tool_use_id=tool_use_id)]}} + if ORDER == "result-first": + send(result) + hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) + else: + hook("PostToolUse", tool_use_id, {"tool_response": TOOL_RESULT.get("content")}) + time.sleep(GAP) + deadline = time.monotonic() + 20 + while HOLD and not os.path.exists(HOLD) and time.monotonic() < deadline: + time.sleep(0.02) + if ORDER != "no-result": + send(result) + assistant([{"type": "text", "text": "done"}]) + send({"type": "result", "subtype": "success", "duration_ms": 1, "duration_api_ms": 1, "is_error": False, + "num_turns": 1, "session_id": "s1", "total_cost_usd": 0.0, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}) + + +def handle(message): + kind = message.get("type") + if kind == "control_request": + request = message.get("request", {}) + if request.get("subtype") == "initialize": + for event, matchers in (request.get("hooks") or {}).items(): + hooks[event] = [cid for matcher in matchers for cid in matcher.get("hookCallbackIds", [])] + send({"type": "control_response", "response": { + "subtype": "success", "request_id": message.get("request_id"), + "response": {"commands": [], "models": [], "account": {}} if request.get("subtype") == "initialize" + else {}}}) + elif kind == "user": + turn() + + +send({"type": "system", "subtype": "init", "session_id": "s1", "tools": [], "mcp_servers": [], "model": "claude-test", + "cwd": "/", "permissionMode": "default", "apiKeySource": "none"}) +for line in sys.stdin: + line = line.strip() + if line: + handle(json.loads(line)) diff --git a/python/tests/attachments_mcp_agents/_harness.py b/python/tests/attachments_mcp_agents/_harness.py new file mode 100644 index 000000000..b7e2d2a53 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_harness.py @@ -0,0 +1,44 @@ +"""Shared helpers: the provider-attachments harness, loaded by path (one F01 loopback for every lane).""" + +import importlib.util +import json +import sys +from pathlib import Path + +_NAME = "_attachments_providers_harness" +if _NAME in sys.modules: + harness = sys.modules[_NAME] +else: + _path = Path(__file__).resolve().parent.parent / "attachments_providers" / "conftest.py" + _spec = importlib.util.spec_from_file_location(_NAME, _path) + harness = importlib.util.module_from_spec(_spec) + sys.modules[_NAME] = harness + _spec.loader.exec_module(harness) + +png, wav, b64, refs, wait_uploads, flush, PDF = (harness.png, harness.wav, harness.b64, harness.refs, + harness.wait_uploads, harness.flush, harness.PDF) + + +def data_uri(mime, data): + return "data:%s;base64,%s" % (mime, b64(data)) + + +def uploaded(f01): + """(mime_type, direction, purpose, role, tool_call_id) of each initialize body, sorted.""" + return sorted((u.get("mime_type"), u.get("direction"), u.get("purpose"), u.get("role"), + (u.get("correlation") or {}).get("tool_call_id") or u.get("tool_call_id")) + for u in f01.uploads) + + +def span_named(exporter, name): + found = [s for s in exporter.get_finished_spans() if s.name == name] + assert found, [s.name for s in exporter.get_finished_spans()] + return found[-1] + + +def no_bytes(exporter, data): + """No exported span attribute carries the media's base64 text.""" + text = b64(data)[:40] + for span in exporter.get_finished_spans(): + for value in (span.attributes or {}).values(): + assert text not in json.dumps(value, default=str), span.name diff --git a/python/tests/attachments_mcp_agents/_mcp_media_server.py b/python/tests/attachments_mcp_agents/_mcp_media_server.py new file mode 100644 index 000000000..726dde217 --- /dev/null +++ b/python/tests/attachments_mcp_agents/_mcp_media_server.py @@ -0,0 +1,116 @@ +"""A real MCP server over stdio whose tools return synthetic typed media. + +Used by the MCP (``wrap_mcp_client``) and mcp-use tests. It talks to nothing +else. ``reads`` reports how many times the client read each resource, so a +test can prove a ``resource_link`` URI was never fetched. +""" + +import base64 +import json +import struct +import zlib +from importlib.metadata import version + +from mcp import types + +if int(version("mcp").split(".")[0]) >= 2: + from mcp.server.mcpserver import MCPServer as Server +else: + from mcp.server.fastmcp import FastMCP as Server + + +def png(seed=0, width=2, height=2) -> bytes: + raw = b"".join(b"\x00" + bytes((seed + x + y) % 256 for x in range(width * 3)) for y in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +WAV = b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00@\x1f\x00\x00\x80>\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00" +PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n" +SECRET_FILE = "file:///etc/hostname" +READS = {"shot": 0, "secret": 0} + + +def b64(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def block(kind, **fields): + """A content block from its wire (camelCase) shape, valid on MCP 1.x and 2.x.""" + model = {"image": types.ImageContent, "audio": types.AudioContent, "text": types.TextContent, + "resource": types.EmbeddedResource, "resource_link": types.ResourceLink}[kind] + return model.model_validate({"type": kind, **fields}) + + +server = Server("media-fixture") + + +@server.tool() +def screenshot(): + """A screenshot and a caption.""" + return [block("text", text="CAPTION-MARKER"), block("image", data=b64(png(1)), mimeType="image/png")] + + +@server.tool() +def voice(): + """A short WAV clip.""" + return [block("audio", data=b64(WAV), mimeType="audio/wav")] + + +@server.tool() +def report(): + """An embedded PDF resource blob plus a resource link that must never be fetched.""" + return [ + block("resource", resource={"uri": "file:///report.pdf", "mimeType": "application/pdf", "blob": b64(PDF)}), + block("resource_link", uri=SECRET_FILE, name="hostname", mimeType="image/png"), + ] + + +@server.tool() +def relabeled(): + """An image block whose MIME claims another modality, and one outside the F01 enum.""" + return [block("image", data=b64(png(2)), mimeType="audio/wav"), + block("image", data=b64(png(3)), mimeType="image/gif")] + + +@server.tool() +def malformed(): + """Not valid base64.""" + return [block("image", data="!!!not-base64!!!", mimeType="image/png")] + + +@server.tool() +def many(count: int): + """``count`` distinct images.""" + return [block("image", data=b64(png(10 + n)), mimeType="image/png") for n in range(count)] + + +@server.tool() +def big(): + """An image larger than the 256 KiB per-result budget.""" + return [block("image", data=b64(png(4) + b"\x00" * (300 * 1024)), mimeType="image/png")] + + +@server.tool() +def reads() -> str: + return json.dumps(READS) + + +@server.resource("file:///shot.png", mime_type="image/png") +def shot() -> bytes: + READS["shot"] += 1 + return png(5) + + +@server.resource(SECRET_FILE, mime_type="image/png") +def secret() -> bytes: + READS["secret"] += 1 + return png(6) + + +if __name__ == "__main__": + server.run(transport="stdio") diff --git a/python/tests/attachments_mcp_agents/conftest.py b/python/tests/attachments_mcp_agents/conftest.py new file mode 100644 index 000000000..41d06ea86 --- /dev/null +++ b/python/tests/attachments_mcp_agents/conftest.py @@ -0,0 +1,27 @@ +"""Loopback harness for MCP and agent tool-media tests: synthetic data only. + +Reuses the provider-attachments harness (``tests/attachments_providers``): +``f01`` is a TLS loopback F01 attachment server and ``spans`` an in-memory +OpenTelemetry provider. Nothing here reaches a network beyond 127.0.0.1. +""" + +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("fi_instrumentation.attachments.tool_media") +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _harness import harness # noqa: E402 + +# Fixtures, registered by being attributes of this conftest. +clean_env = harness.clean_env +f01 = harness.f01 +spans = harness.spans + + +@pytest.fixture(autouse=True) +def mock_all_http_requests(): + """Overrides python/tests/conftest.py: these suites talk to real loopback servers through requests.""" + yield None diff --git a/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py new file mode 100644 index 000000000..b336f8428 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_claude_agent_sdk_attachments.py @@ -0,0 +1,280 @@ +"""Real claude_agent_sdk ClaudeSDKClient over its subprocess transport -> loopback F01. + +The CLI is ``_fake_claude_cli.py`` (the SDK's real stream-json control protocol: +initialize, hook callbacks, user/assistant/result messages); no model or key. +""" + +import asyncio +import json +import os +import stat +import sys +import time +from pathlib import Path + +import pytest + +claude_agent_sdk = pytest.importorskip("claude_agent_sdk") +pytest.importorskip("traceai_claude_agent_sdk") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_claude_agent_sdk import ClaudeAgentInstrumentor # noqa: E402 +from traceai_claude_agent_sdk._attributes import ClaudeAgentAttributes # noqa: E402 + +from _harness import PDF, b64, flush, png, refs # noqa: E402 + +CLI = Path(__file__).resolve().parent / "_fake_claude_cli.py" +SHOT = png(seed=51) + + +def image(data, mime="image/png"): + return {"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64(data)}} + + +@pytest.fixture +def cli(tmp_path, monkeypatch): + def make(content, order="hooks-first", is_error=False, gap=0.0): + result = tmp_path / "result.json" + result.write_text(json.dumps({"content": content, "is_error": is_error})) + launcher = tmp_path / "claude" + launcher.write_text("#!%s\nimport runpy\nrunpy.run_path(%r, run_name='__main__')\n" % (sys.executable, str(CLI))) + launcher.chmod(launcher.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("FAKE_CLAUDE_RESULT", str(result)) + monkeypatch.setenv("FAKE_CLAUDE_ORDER", order) + monkeypatch.setenv("FAKE_CLAUDE_GAP", str(gap)) + monkeypatch.setenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "1") + # Releases without ClaudeAgentOptions.cli_path find the CLI as "claude" on PATH. + monkeypatch.setenv("PATH", str(tmp_path) + os.pathsep + os.environ.get("PATH", "")) + return str(launcher) + + return make + + +@pytest.fixture +def instrument(spans): + provider, exporter = spans + state = {} + + def install(manager, config=None): + instrumentor = ClaudeAgentInstrumentor() + instrumentor.instrument(tracer_provider=provider, attachments=manager, config=config or TraceConfig(), + skip_dep_check=True) + state["instrumentor"] = instrumentor + return exporter + + yield install + if "instrumentor" in state: + state["instrumentor"].uninstrument() + + +def client_options(cli_path, **env): + fields = getattr(claude_agent_sdk.ClaudeAgentOptions, "__dataclass_fields__", {}) + kwargs = {"cli_path": cli_path} if "cli_path" in fields else {} + if env: + if "env" not in fields: + pytest.skip("this claude-agent-sdk release has no ClaudeAgentOptions.env") + kwargs["env"] = env + return claude_agent_sdk.ClaudeAgentOptions(**kwargs) + + +def converse(cli_path): + async def go(): + options = client_options(cli_path) + messages = [] + async with claude_agent_sdk.ClaudeSDKClient(options=options) as client: + await client.query("take a screenshot") + async for message in client.receive_response(): + messages.append(message) + return messages + + return asyncio.run(go()) + + +def tool_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.startswith("tool.")] + + +@pytest.mark.parametrize("order", ["hooks-first", "result-first"]) +def test_tool_result_image_and_pdf_upload_before_truncation_on_the_tool_span(f01, instrument, cli, order): + manager = f01.manager() + exporter = instrument(manager) + messages = converse(cli([{"type": "text", "text": "captured"}, image(SHOT), + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", + "data": b64(PDF)}}], order=order)) + flush(manager) + assert type(messages[-1]).__name__ == "ResultMessage" + assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ + ("application/pdf", "output", "tool_result", "toolu_screen_1"), + ("image/png", "output", "tool_result", "toolu_screen_1")] + assert sorted(f01.payloads) == sorted([SHOT, PDF]) + (tool,) = tool_spans(exporter) + parents = {u["association"]["parent"]["span_id"] for u in f01.uploads} + assert parents == {format(tool.context.span_id, "016x")} + # In both orders the tool span is still recording when its result arrives, so every + # uploaded occurrence is projected (fi.attachment.refs) on its upload parent. + assert sorted(r["occurrence_id"] for r in refs(tool)) == sorted(u["occurrence_id"] for u in f01.uploads) + assert tool.status.is_ok + # Bytes never reach a text attribute (tool output, conversation output). + for span in exporter.get_finished_spans(): + for value in span.attributes.values(): + assert b64(SHOT)[:40] not in str(value) + manager.shutdown(5) + + +def _duration_ms(span): + return (span.end_time - span.start_time) / 1e6 + + +def test_hooks_first_tool_span_keeps_the_hook_end_time_while_waiting_for_its_result(f01, instrument, cli): + manager = f01.manager() + exporter = instrument(manager) + converse(cli([image(SHOT)], gap=0.6)) + flush(manager) + (tool,) = tool_spans(exporter) + assert len(refs(tool)) == 1 and len(f01.uploads) == 1 + hook_ms = tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS] + # Ended at the PostToolUse time, not when the result streamed 600 ms later. + assert abs(_duration_ms(tool) - hook_ms) < 150 and _duration_ms(tool) < 450 + manager.shutdown(5) + + +def test_a_result_that_never_arrives_still_ends_the_tool_span_at_its_hook_time(f01, instrument, cli): + manager = f01.manager() + exporter = instrument(manager) + messages = converse(cli([image(SHOT)], order="no-result", gap=0.3)) + flush(manager) + assert type(messages[-1]).__name__ == "ResultMessage" + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + manager.shutdown(5) + + +def test_url_and_file_sources_never_fetch(f01, instrument, cli): + manager = f01.manager() + instrument(manager) + converse(cli([{"type": "image", "source": {"type": "url", "url": "http://127.0.0.1:9/x.png"}}, + {"type": "image", "source": {"type": "file", "file_id": "file_1"}}, + {"type": "image", "source": {"type": "base64", "media_type": "text/plain", "data": b64(SHOT)}}])) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny(f01, instrument, cli, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + converse(cli([image(SHOT)])) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing_and_the_conversation_is_unchanged(f01, instrument, cli): + instrument(create_attachment_manager(config=AttachmentConfig())) + messages = converse(cli([image(SHOT)])) + assert f01.uploads == [] + blocks = [m for m in messages if type(m).__name__ == "UserMessage"][0].content + assert blocks[0].content[0]["source"]["data"] == b64(SHOT) + + +def test_receive_messages_without_disconnect_ends_the_tool_span_at_its_hook(f01, instrument, cli): + """No receive_response() conversation is live, so PostToolUse ends the span as with attachments off.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)], gap=0.3) + clients = [] + + async def go(): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path)) + clients.append(client) + await client.connect() + await client.query("take a screenshot") + async for message in client.receive_messages(): + if type(message).__name__ == "ResultMessage": + break + # Returns without disconnect() or async-with: nothing flushes deferred spans. + + asyncio.run(go()) + flush(manager) + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + assert not clients[0]._tool_deferrals.ends and clients[0]._tool_deferrals.live == 0 + manager.shutdown(5) + + +def test_conversation_end_flushes_its_waiting_tool_span_without_disconnect(f01, instrument, cli): + """A result that never arrives: the span ends when receive_response() ends, not at the 30 s timer.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)], order="no-result", gap=0.2) + clients = [] + + async def go(): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path)) + clients.append(client) + await client.connect() + await client.query("take a screenshot") + async for _ in client.receive_response(): + pass + # Inspected before any disconnect(): only the conversation end can have ended it. + deferrals = clients[0]._tool_deferrals + return len(tool_spans(exporter)), list(deferrals.ends), deferrals.live + + ended, waiting, live = asyncio.run(go()) + assert (ended, waiting, live) == (1, [], 0) + (tool,) = tool_spans(exporter) + assert tool.status.is_ok and not refs(tool) and f01.uploads == [] + assert abs(_duration_ms(tool) - tool.attributes[ClaudeAgentAttributes.TOOL_DURATION_MS]) < 150 + manager.shutdown(5) + + +def test_one_conversation_ending_does_not_end_another_clients_deferred_tool_span(f01, instrument, cli, tmp_path): + """B's tool span waits for its result while A runs to completion; A's end flushes only A's spans.""" + manager = f01.manager() + exporter = instrument(manager) + path = cli([image(SHOT)]) + other = png(seed=52) + a_result = tmp_path / "result-a.json" + a_result.write_text(json.dumps({"content": [image(other)], "is_error": False})) + release = tmp_path / "release-b" + clients = {} + parked = {} + + async def converse_as(name, **env): + client = claude_agent_sdk.ClaudeSDKClient(options=client_options(path, FAKE_TOOL_ID="toolu_" + name, **env)) + clients[name] = client + async with client: + await client.query("take a screenshot") + async for _ in client.receive_response(): + pass + + async def go(): + b = asyncio.ensure_future(converse_as("b", FAKE_CLAUDE_HOLD=str(release))) + deadline = time.monotonic() + 15 + while not ("b" in clients and clients["b"]._tool_deferrals.ends): + assert time.monotonic() < deadline and not b.done(), "B never parked its tool span" + await asyncio.sleep(0.02) + await converse_as("a", FAKE_CLAUDE_RESULT=str(a_result)) + parked["b"] = list(clients["b"]._tool_deferrals.ends) + release.write_text("go") + await b + + asyncio.run(go()) + flush(manager) + assert parked["b"] == ["toolu_b"] + tools = {s.attributes[ClaudeAgentAttributes.TOOL_USE_ID]: s for s in tool_spans(exporter)} + assert sorted(tools) == ["toolu_a", "toolu_b"] + assert sorted(u.get("tool_call_id") for u in f01.uploads) == ["toolu_a", "toolu_b"] + assert sorted(f01.payloads) == sorted([SHOT, other]) + for upload in f01.uploads: + tool = tools[upload["tool_call_id"]] + assert tool.status.is_ok + assert upload["association"]["parent"]["span_id"] == format(tool.context.span_id, "016x") + assert [r["occurrence_id"] for r in refs(tool)] == [upload["occurrence_id"]] + assert not clients["a"]._tool_deferrals.ends and not clients["b"]._tool_deferrals.ends + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py new file mode 100644 index 000000000..f710e16e0 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_firecrawl_attachments.py @@ -0,0 +1,128 @@ +"""Real firecrawl-py v2 clients against a loopback Firecrawl API -> loopback F01.""" + +import asyncio +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("firecrawl") +pytest.importorskip("traceai_firecrawl") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from firecrawl import AsyncFirecrawl, Firecrawl # noqa: E402 +from traceai_firecrawl import FirecrawlInstrumentor # noqa: E402 + +from _harness import data_uri, flush, png, refs, span_named # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "firecrawl" / "tests")) +from _firecrawl_fake import JOB_ID, FakeFirecrawl, crawl_status # noqa: E402 + +SHOT = png(seed=61) +ACTION_SHOT = png(seed=62) +KEY = "fc-SYNTHETIC-PLACEHOLDER" + + +@pytest.fixture +def fake(): + server = FakeFirecrawl() + yield server + server.close() + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + FirecrawlInstrumentor().uninstrument() + + +def page(screenshot, actions=None): + data = {"markdown": "body", "screenshot": screenshot, "metadata": {"sourceURL": "https://example.com"}} + if actions is not None: + data["actions"] = {"screenshots": actions} + return {"success": True, "data": data} + + +def test_inline_screenshots_upload_and_hosted_urls_are_never_fetched(f01, instrument, fake): + manager = f01.manager() + exporter = instrument(manager) + hosted = fake.origin + "/hosted/screenshot.png" + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT), [data_uri("image/jpeg", ACTION_SHOT), hosted])) + document = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert document.screenshot == data_uri("image/png", SHOT) # the SDK result is unchanged + assert sorted((u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads) == [ + ("image/jpeg", "output", "tool_result"), ("image/png", "output", "tool_result")] + assert sorted(f01.payloads) == sorted([SHOT, ACTION_SHOT]) + assert fake.calls_to("GET", "/hosted/screenshot.png") == 0 + assert len(refs(span_named(exporter, "firecrawl.scrape"))) == 2 + manager.shutdown(5) + + +def test_hosted_only_screenshot_uploads_nothing(f01, instrument, fake): + manager = f01.manager() + instrument(manager) + fake.routes[("POST", "/v2/scrape")] = (200, page(fake.origin + "/hosted/x.png")) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert f01.uploads == [] and fake.calls_to("GET", "/hosted/x.png") == 0 + manager.shutdown(5) + + +def test_async_client_scrape(f01, instrument, fake): + manager = f01.manager() + instrument(manager) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + + async def go(): + return await AsyncFirecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape( + "https://example.com", formats=["screenshot"]) + + asyncio.run(go()) + flush(manager) + assert f01.payloads == [SHOT] + manager.shutdown(5) + + +def test_crawl_status_documents_upload_on_the_status_span(f01, instrument, fake): + manager = f01.manager() + exporter = instrument(manager) + body = crawl_status("completed", pages=2) + body["data"] = [dict(page, screenshot=data_uri("image/png", png(seed=63 + n))) for n, page in enumerate(body["data"])] + fake.routes[("GET", "/v2/crawl/" + JOB_ID)] = (200, body) + job = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).get_crawl_status(JOB_ID) + flush(manager) + assert len(job.data) == 2 + assert sorted(f01.payloads) == sorted([png(seed=63), png(seed=64)]) + assert len(refs(span_named(exporter, "firecrawl.get_crawl_status"))) == 2 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_inputs", "pii_redaction"]) +def test_local_restrictions_deny(f01, instrument, fake, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, instrument, fake): + instrument(create_attachment_manager(config=AttachmentConfig())) + fake.routes[("POST", "/v2/scrape")] = (200, page(data_uri("image/png", SHOT))) + Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1).scrape("https://example.com", formats=["screenshot"]) + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_mcp_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_attachments.py new file mode 100644 index 000000000..155cef0e3 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_mcp_attachments.py @@ -0,0 +1,244 @@ +"""Real ``mcp`` ClientSession over stdio -> wrap_mcp_client -> TLS loopback F01 server.""" + +import asyncio +import json +import sys +import threading +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") +traceai_mcp = pytest.importorskip("traceai_mcp") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from mcp import ClientSession, StdioServerParameters # noqa: E402 +from mcp.client.stdio import stdio_client # noqa: E402 + +from _harness import PDF, flush, no_bytes, refs, span_named, uploaded # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _mcp_media_server as server # noqa: E402 + +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" + + +def run(spans, manager, calls, **wrap): + """Open a real stdio session, wrap it, and run ``calls(session)``.""" + provider, _ = spans + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + traced = traceai_mcp.wrap_mcp_client(session, tracer_provider=provider, attachments=manager, **wrap) + return await calls(traced) + + return asyncio.run(go()) + + +def test_tool_image_audio_and_embedded_blob_upload_with_tool_identity(f01, spans): + manager = f01.manager() + _, exporter = spans + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + result = await session.call_tool("report", {}) + reads = await session.call_tool("reads", {}) + return result, reads + + result, reads = run(spans, manager, calls) + flush(manager) + assert sorted(u["mime_type"] for u in f01.uploads) == ["application/pdf", "audio/wav", "image/png"] + assert {(u["direction"], u["purpose"]) for u in f01.uploads} == {("output", "tool_result")} + assert server.png(1) in f01.payloads and server.WAV in f01.payloads and PDF in f01.payloads + # The resource_link URI was never dereferenced. + assert json.loads(reads.content[0].text)["secret"] == 0 + # The SDK result is unchanged. + assert result.content[0].resource.blob == server.b64(PDF) + shot = [s for s in exporter.get_finished_spans() if s.name == "mcp.tools/call"] + assert len(shot) == 4 + assert sum(len(refs(s)) for s in shot) == 3 + assert {r["role"] for s in shot for r in refs(s)} == {"tool"} + no_bytes(exporter, server.png(1)) + manager.shutdown(5) + + +def test_resources_read_blob_uploads_and_text_is_ignored(f01, spans): + manager = f01.manager() + + async def calls(session): + return await session.read_resource("file:///shot.png") + + result = run(spans, manager, calls) + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["image/png"] + assert f01.payloads == [server.png(5)] + assert result.contents[0].blob == server.b64(server.png(5)) + manager.shutdown(5) + + +def test_relabeled_unknown_malformed_and_oversized_media_upload_nothing(f01, spans): + manager = f01.manager() + + async def calls(session): + for name in ("relabeled", "malformed", "big"): + await session.call_tool(name, {}) + + run(spans, manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_part_budget_caps_uploads_per_result(f01, spans): + manager = f01.manager() + + async def calls(session): + await session.call_tool("many", {"count": 70}) + + run(spans, manager, calls) + flush(manager) + assert len(f01.uploads) == 64 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_messages", "hide_output_text", + "pii_redaction"]) +def test_local_hide_flags_and_pii_deny_every_tool_media(f01, spans, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + + run(spans, manager, calls, trace_config=TraceConfig(**{flag: True})) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_hide_input_images_denies_images_but_not_audio(f01, spans): + manager = f01.manager() + + async def calls(session): + await session.call_tool("screenshot", {}) + await session.call_tool("voice", {}) + + run(spans, manager, calls, trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["audio/wav"] + manager.shutdown(5) + + +def test_env_hide_flag_denies(f01, spans, monkeypatch): + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + manager = f01.manager() + + async def calls(session): + await session.call_tool("screenshot", {}) + + run(spans, manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_disabled_manager_and_unsampled_span_upload_nothing_and_start_no_threads(f01, spans): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + + before = {t.name for t in threading.enumerate()} + disabled = create_attachment_manager(config=AttachmentConfig()) + + async def calls(session): + return await session.call_tool("screenshot", {}) + + result = run(spans, disabled, calls) + assert result.content[1].data == server.b64(server.png(1)) + assert not {t.name for t in threading.enumerate()} - before + manager = f01.manager() + unsampled = TracerProvider(sampler=ALWAYS_OFF) + run((unsampled, None), manager, calls) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_provider_manager_is_the_default_and_rewrapping_is_idempotent(f01, spans): + manager = f01.manager() + provider, exporter = spans + provider.attachment_manager = manager + + async def go(): + params = StdioServerParameters(command=sys.executable, args=[str(SERVER)]) + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + traced = traceai_mcp.wrap_mcp_client(session, tracer_provider=provider) + assert traceai_mcp.wrap_mcp_client(traced) is traced + await traced.call_tool("screenshot", {}) + + asyncio.run(go()) + flush(manager) + assert len(f01.uploads) == 1 + assert len([s for s in exporter.get_finished_spans() if s.name == "mcp.tools/call"]) == 1 + manager.shutdown(5) + + +def test_tool_errors_keep_the_sdk_exception_and_end_the_span(f01, spans): + manager = f01.manager() + _, exporter = spans + + async def calls(session): + with pytest.raises(Exception): + await session.read_resource("file:///missing.png") + + run(spans, manager, calls) + span = span_named(exporter, "mcp.resources/read") + assert span.status.status_code.name == "ERROR" + manager.shutdown(5) + + +def test_capture_mcp_result_is_preflight_first_and_never_raises(f01, spans): + manager = f01.manager() + provider, _ = spans + reads = [] + + class Part(dict): + def get(self, key, default=None): + if key == "data": + reads.append(key) + return super().get(key, default) + + tracer = provider.get_tracer("t") + with tracer.start_as_current_span("denied") as span: + traceai_mcp.capture_mcp_result(manager, span, {"content": [Part(type="image", data="AAAA", mimeType="image/png")]}, + trace_config=TraceConfig(hide_outputs=True)) + assert reads == [] + assert traceai_mcp.capture_mcp_result(manager, span, object()) == {} + assert traceai_mcp.capture_mcp_result(None, span, {"content": []}) == {} + manager.shutdown(5) + + +def test_error_results_follow_hide_error_details(f01, spans): + """An isError result's media is error detail: hide_error_details denies it, a success result still uploads.""" + manager = f01.manager() + provider, _ = spans + shot, ok = server.png(31), server.png(32) + + def result(data, error): + return {"isError": error, "content": [{"type": "image", "data": server.b64(data), "mimeType": "image/png"}]} + + config = TraceConfig(hide_error_details=True) + with provider.get_tracer("t").start_as_current_span("failed") as span: + assert traceai_mcp.capture_mcp_result(manager, span, result(shot, True), trace_config=config) == {} + with provider.get_tracer("t").start_as_current_span("succeeded") as span: + assert len(traceai_mcp.capture_mcp_result(manager, span, result(ok, False), trace_config=config)) == 1 + flush(manager) + assert f01.payloads == [ok] + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py new file mode 100644 index 000000000..458ceff2e --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_mcp_use_attachments.py @@ -0,0 +1,102 @@ +"""Real mcp-use MCPAgent (scripted LangChain chat model) + stdio MCP media server -> loopback F01.""" + +import asyncio +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("mcp_use") +pytest.importorskip("traceai_mcp_use") +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "frameworks" / "mcp-use" / "tests")) +import _mcp_use_support as support # noqa: E402 (also turns mcp-use telemetry off) + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_mcp_use import FutureAGICallback # noqa: E402 + +from _harness import flush, refs # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _mcp_media_server as server # noqa: E402 + +SERVER = Path(__file__).resolve().parent / "_mcp_media_server.py" + + +def run(callback, tool, args=None): + from mcp_use import MCPAgent, MCPClient + + async def go(): + client = MCPClient.from_dict({"mcpServers": {"media": {"command": sys.executable, "args": [str(SERVER)]}}}) + try: + agent = MCPAgent(llm=support.ChatFake(script=[support.tool_call(tool, args or {}, "call_media_1"), + support.answer()]), + client=client, callbacks=[callback]) + return await agent.run(support.PROMPT) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +def tool_span(exporter, name): + return [s for s in exporter.get_finished_spans() if s.name == "execute_tool " + name][-1] + + +def test_tool_image_and_blob_upload_on_the_tool_span(f01, spans): + manager = f01.manager() + provider, exporter = spans + callback = FutureAGICallback(tracer_provider=provider, attachments=manager) + assert run(callback, "screenshot") == support.ANSWER + # LangChain turns an audio-only tool result into a string before callbacks see it: no typed block, no upload. + run(callback, "voice") + run(callback, "report") + flush(manager) + assert sorted((u["mime_type"], u["direction"], u["purpose"], u.get("tool_call_id")) for u in f01.uploads) == [ + ("application/pdf", "output", "tool_result", "call_media_1"), + ("image/png", "output", "tool_result", "call_media_1")] + assert server.png(1) in f01.payloads + span = tool_span(exporter, "screenshot") + assert len(refs(span)) == 1 and refs(span)[0]["role"] == "tool" + assert server.b64(server.png(1))[:40] not in str(span.attributes) + manager.shutdown(5) + + +def test_provider_manager_is_the_default(f01, spans): + manager = f01.manager() + provider, _ = spans + provider.attachment_manager = manager + run(FutureAGICallback(tracer_provider=provider), "screenshot") + flush(manager) + assert len(f01.uploads) == 1 + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "pii_redaction"]) +def test_local_restrictions_deny(f01, spans, flag): + manager = f01.manager() + provider, _ = spans + before = len(f01.after_bootstrap()) + run(FutureAGICallback(tracer_provider=provider, config=TraceConfig(**{flag: True}), attachments=manager), + "screenshot") + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_malformed_and_relabeled_media_upload_nothing(f01, spans): + manager = f01.manager() + provider, _ = spans + callback = FutureAGICallback(tracer_provider=provider, attachments=manager) + run(callback, "relabeled") + run(callback, "malformed") + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, spans): + provider, _ = spans + run(FutureAGICallback(tracer_provider=provider, attachments=create_attachment_manager(config=AttachmentConfig())), + "screenshot") + assert f01.uploads == [] diff --git a/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py new file mode 100644 index 000000000..946697b24 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_openai_agents_attachments.py @@ -0,0 +1,215 @@ +"""Real OpenAI Agents SDK Runner (scripted local Model, real ComputerTool and function tools) -> loopback F01.""" + +import asyncio +import json +from importlib.metadata import version + +import pytest + +agents = pytest.importorskip("agents") +pytest.importorskip("traceai_openai_agents") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from packaging.version import Version # noqa: E402 +from traceai_openai_agents import OpenAIAgentsInstrumentor # noqa: E402 + +from _harness import data_uri, flush, png, refs # noqa: E402 + +SDK = Version(version("openai-agents")) +COMPUTER_SPAN = SDK >= Version("0.10.0") # the SDK records computer calls on a function span from 0.10.0 +TYPED_IMAGE = SDK >= Version("0.4.0") # ToolOutputImage exists from 0.4.0 +SHOT = png(seed=41) +TOOL_IMAGE = png(seed=42) + + +class ScriptedModel(agents.Model): + """Replays model outputs; no network.""" + + def __init__(self, turns): + self.turns = list(turns) + self.inputs = [] + + async def get_response(self, *args, **kwargs): + from agents.usage import Usage + + self.inputs.append(kwargs.get("input", args[1] if len(args) > 1 else None)) + output = self.turns.pop(0) + fields = dict(output=output, usage=Usage(), response_id=None) + try: + return agents.ModelResponse(**fields) + except TypeError: + return agents.ModelResponse(output=output, usage=Usage(), response_id=None, request_id=None) + + def stream_response(self, *args, **kwargs): + raise NotImplementedError + + +def message(text="done"): + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + return ResponseOutputMessage(id="m1", type="message", role="assistant", status="completed", + content=[ResponseOutputText(type="output_text", text=text, annotations=[])]) + + +def computer_call(call_id="call_shot"): + from openai.types.responses import ResponseComputerToolCall + + return ResponseComputerToolCall.model_validate({ + "id": "cc1", "type": "computer_call", "call_id": call_id, "status": "completed", + "pending_safety_checks": [], "action": {"type": "screenshot"}}) + + +def function_call(name, call_id="call_fn"): + from openai.types.responses import ResponseFunctionToolCall + + return ResponseFunctionToolCall(id="fc1", type="function_call", call_id=call_id, name=name, arguments="{}") + + +class Screen(agents.AsyncComputer): + shots = 0 + + @property + def environment(self): + return "browser" + + @property + def dimensions(self): + return (1024, 768) + + async def screenshot(self): + Screen.shots += 1 + return data_uri("image/png", SHOT).split(",", 1)[1] + + async def click(self, x, y, button): + return None + + async def double_click(self, x, y): + return None + + async def scroll(self, x, y, scroll_x, scroll_y): + return None + + async def type(self, text): + return None + + async def wait(self): + return None + + async def move(self, x, y): + return None + + async def keypress(self, keys): + return None + + async def drag(self, path): + return None + + +@pytest.fixture +def traced(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + state = {} + + def install(manager, config=None): + OpenAIAgentsInstrumentor().instrument(tracer_provider=provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + OpenAIAgentsInstrumentor().uninstrument() # lets the next test install a fresh processor + agents.set_trace_processors([]) + provider.shutdown() + + +def run_computer(manager, install, config=None): + exporter = install(manager, config) + model = ScriptedModel([[computer_call()], [message()]]) + agent = agents.Agent(name="operator", model=model, tools=[agents.ComputerTool(computer=Screen())]) + result = asyncio.run(agents.Runner.run(agent, "take a screenshot")) + return exporter, model, result + + +def function_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "TOOL"] + + +def test_computer_screenshot_uploads_on_the_computer_tool_span(f01, traced): + manager = f01.manager() + exporter, model, result = run_computer(manager, traced) + flush(manager) + assert result.final_output == "done" + if not COMPUTER_SPAN: + assert f01.uploads == [] # no function span records the computer call before 0.10.0 + return + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "computer_screenshot")] + assert f01.payloads == [SHOT] + tool = [s for s in function_spans(exporter) if refs(s)] + assert len(tool) == 1 + output = tool[0].attributes["output.value"] + assert output.startswith("fi-attachment:") and output[len("fi-attachment:"):] == refs(tool[0])[0]["occurrence_id"] + # The SDK still sent the original screenshot to the model. + sent = json.dumps(model.inputs[-1], default=str) + assert data_uri("image/png", SHOT) in sent + manager.shutdown(5) + + +@pytest.mark.skipif(not TYPED_IMAGE, reason="ToolOutputImage needs openai-agents 0.4.0") +def test_function_tool_typed_image_uploads_and_url_or_file_id_never_fetch(f01, traced): + manager = f01.manager() + exporter = traced(manager) + + @agents.function_tool + def chart() -> list: + """Draw a chart.""" + return [agents.ToolOutputImage(image_url=data_uri("image/png", TOOL_IMAGE)), + agents.ToolOutputImage(image_url="https://example.invalid/chart.png"), + agents.ToolOutputImage(file_id="file-123")] + + model = ScriptedModel([[function_call("chart")], [message()]]) + agent = agents.Agent(name="analyst", model=model, tools=[chart]) + asyncio.run(agents.Runner.run(agent, "chart it")) + flush(manager) + assert [(u["mime_type"], u["purpose"]) for u in f01.uploads] == [("image/png", "tool_result")] + assert f01.payloads == [TOOL_IMAGE] + span = [s for s in function_spans(exporter) if refs(s)][0] + assert "https://example.invalid/chart.png" in span.attributes["output.value"] + assert data_uri("image/png", TOOL_IMAGE) not in span.attributes["output.value"] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_outputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny_the_screenshot(f01, traced, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + run_computer(manager, traced, TraceConfig(**{flag: True})) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_disabled_manager_keeps_the_old_attributes(f01, traced): + exporter, _, result = run_computer(create_attachment_manager(config=AttachmentConfig()), traced) + assert result.final_output == "done" and f01.uploads == [] + for span in function_spans(exporter): + assert not refs(span) + assert not str(span.attributes.get("output.value", "")).startswith("fi-attachment:") + + +def test_sensitive_data_off_records_and_uploads_nothing(f01, traced): + manager = f01.manager() + traced(manager) + model = ScriptedModel([[computer_call()], [message()]]) + agent = agents.Agent(name="operator", model=model, tools=[agents.ComputerTool(computer=Screen())]) + asyncio.run(agents.Runner.run(agent, "shot", run_config=agents.RunConfig(trace_include_sensitive_data=False))) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_mcp_agents/test_smolagents_attachments.py b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py new file mode 100644 index 000000000..1762d5a49 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_smolagents_attachments.py @@ -0,0 +1,231 @@ +"""Real smolagents (OpenAIServerModel on a loopback OpenAI double, real Tool and agent loop) -> loopback F01.""" + +import io + +import pytest + +smolagents = pytest.importorskip("smolagents") +PIL_Image = pytest.importorskip("PIL.Image") +pytest.importorskip("openai") +pytest.importorskip("traceai_smolagents") + +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_smolagents import SmolagentsInstrumentor # noqa: E402 + +from _harness import flush, harness # noqa: E402 + +provider = harness.provider # the loopback OpenAI-compatible API (fixture) + + +def picture(seed=0, size=(8, 6)): + image = PIL_Image.new("RGB", size, (seed % 256, 10, 20)) + return image + + +def png_of(image): + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + SmolagentsInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + SmolagentsInstrumentor().uninstrument() + + +def model(provider): + return smolagents.OpenAIServerModel(model_id="m", api_base=provider.url + "/v1", api_key="SYNTHETIC_KEY") + + +def user_message(*images): + content = [{"type": "text", "text": "look"}] + [{"type": "image", "image": image} for image in images] + if not hasattr(smolagents.OpenAIServerModel, "generate"): + return {"role": "user", "content": content} # releases before generate() take plain dicts + from smolagents.models import ChatMessage, MessageRole + + return ChatMessage(role=MessageRole.USER, content=content) + + +def call(m, messages): + method = getattr(m, "generate", None) or m.__call__ + return method(messages) + + +def test_model_input_images_upload_once_on_the_model_span(f01, instrument, provider): + manager = f01.manager() + exporter = instrument(manager) + shot = picture(1) + m = model(provider) + call(m, [user_message(shot)]) + call(m, [user_message(shot)]) # memory replay of the same observation image + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "input", "model_input")] + assert f01.payloads == [png_of(shot)] + # The model still received the image. + assert any(b"data:image/png;base64," in raw for _, _, raw in provider.requests) + llm = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "LLM"] + assert llm and harness.refs(llm[0]) + manager.shutdown(5) + + +def test_tool_returned_images_upload_on_the_tool_span(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + raw, wrapped_image = picture(2), picture(3) + + class Snap(smolagents.Tool): + name = "snap" + description = "Take a picture." + inputs = {"which": {"type": "string", "description": "which"}} + output_type = "image" + + def forward(self, which): + if which == "raw": + return raw + return smolagents.AgentImage(wrapped_image) + + tool = Snap() + tool(which="raw", sanitize_inputs_outputs=False) + tool(which="agent", sanitize_inputs_outputs=False) + flush(manager) + assert [(u["mime_type"], u["direction"], u["purpose"]) for u in f01.uploads] == [ + ("image/png", "output", "tool_result")] * 2 + assert sorted(f01.payloads) == sorted([png_of(raw), png_of(wrapped_image)]) + tools = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "TOOL"] + assert sum(len(harness.refs(s)) for s in tools) == 2 + manager.shutdown(5) + + +def test_file_backed_and_audio_values_are_never_read(f01, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + picture(4).save(path) + + class Files(smolagents.Tool): + name = "files" + description = "Return files." + inputs = {"which": {"type": "string", "description": "which"}} + output_type = "image" + + def forward(self, which): + if which == "agent-path": + return smolagents.AgentImage(str(path)) + return PIL_Image.open(path) + + tool = Files() + tool(which="agent-path", sanitize_inputs_outputs=False) + tool(which="opened", sanitize_inputs_outputs=False) + flush(manager) + assert f01.uploads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_images", "pii_redaction"]) +def test_local_restrictions_deny_model_images(f01, instrument, provider, flag): + manager = f01.manager() + before = len(f01.after_bootstrap()) + instrument(manager, TraceConfig(**{flag: True})) + call(model(provider), [user_message(picture(5))]) + flush(manager) + assert f01.uploads == [] and len(f01.after_bootstrap()) == before + manager.shutdown(5) + + +def test_agent_run_with_task_images_uploads_each_image_once(f01, instrument, provider): + manager = f01.manager() + instrument(manager) + agent = smolagents.ToolCallingAgent(tools=[], model=model(provider), max_steps=1) + try: + agent.run("describe", images=[picture(6)]) + except Exception: + pass # the double never calls a tool; the loop's own errors are not under test + flush(manager) + assert [u["purpose"] for u in f01.uploads] == ["model_input"] + manager.shutdown(5) + + +def test_disabled_manager_uploads_nothing(f01, instrument, provider): + instrument(create_attachment_manager(config=AttachmentConfig())) + call(model(provider), [user_message(picture(7))]) + assert f01.uploads == [] + + +def test_every_model_call_carrying_a_captured_image_is_marked_owner(f01, instrument, provider, monkeypatch): + """A replayed memory image is already uploaded: the provider adapter below must not upload it again.""" + import openai.resources.chat.completions as completions + from fi_instrumentation.attachments import tool_media + from opentelemetry import context as otel_context + + marks = [] + real = completions.Completions.create + + def create(self, *args, **kwargs): + marks.append(otel_context.get_value(tool_media.OWNER_KEY)) + return real(self, *args, **kwargs) + + monkeypatch.setattr(completions.Completions, "create", create) + manager = f01.manager() + instrument(manager) + shot = picture(11) + m = model(provider) + call(m, [user_message(shot)]) + call(m, [user_message(shot)]) # memory replay: the same image object goes to the model again + call(m, [user_message()]) # no image: no mark + flush(manager) + assert marks == [True, True, None] + assert len(f01.uploads) == 1 + assert sum(raw.count(b"data:image/png;base64,") for _, _, raw in provider.requests) == 2 + manager.shutdown(5) + + +def test_seen_table_is_fresh_in_a_forked_child_even_if_its_lock_was_held(): + import os + import signal + import threading + + from traceai_smolagents import _attachments as sa + + if not hasattr(os, "fork"): + pytest.skip("no fork") + held, release = threading.Event(), threading.Event() + + def holder(): + with sa._SEEN._owned_lock(): + held.set() + release.wait(5) + + thread = threading.Thread(target=holder) + thread.start() + held.wait(5) + + class Manager: + pass + + keep = Manager() + pid = os.fork() + if pid == 0: # child: must not block on the lock the parent's thread held at fork + signal.alarm(5) + try: + image = picture(1) + sa._SEEN.add(keep, image) + os._exit(0 if (keep, image) in sa._SEEN else 4) + except BaseException: + os._exit(3) + release.set() + thread.join() + _, status = os.waitpid(pid, 0) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 diff --git a/python/tests/attachments_mcp_agents/test_tool_media.py b/python/tests/attachments_mcp_agents/test_tool_media.py new file mode 100644 index 000000000..c77c58fd4 --- /dev/null +++ b/python/tests/attachments_mcp_agents/test_tool_media.py @@ -0,0 +1,205 @@ +"""The shared tool-media helper: deny-wins folding, preflight before read, budgets, dedup, ownership.""" + +import struct + +import pytest + +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy, create_attachment_manager +from fi_instrumentation.attachments import tool_media +from opentelemetry import context as otel_context + +from _harness import b64, data_uri, flush, png, refs + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_messages", "hide_input_text", "hide_outputs", + "hide_output_messages", "hide_output_text", "pii_redaction"]) +def test_tool_policy_folds_input_and_output_restrictions(flag): + policy = tool_media.tool_capture_policy(TraceConfig(**{flag: True})) + assert policy.capture_content is False and policy.model_input and policy.contains_output + + +def test_tool_policy_images_and_errors(): + assert tool_media.tool_capture_policy(TraceConfig(hide_input_images=True), image=True).capture_content is False + assert tool_media.tool_capture_policy(TraceConfig(hide_input_images=True)).capture_content is True + assert tool_media.tool_capture_policy(TraceConfig(hide_error_details=True), error=True).capture_content is False + base = CapturePolicy(record_outputs=False) + assert tool_media.tool_capture_policy(None, base).record_outputs is False + + +def test_owner_context_disables_capture(f01, spans): + manager = f01.manager() + provider, _ = spans + token = otel_context.attach(tool_media.with_ownership()) + try: + with provider.get_tracer("t").start_as_current_span("owned") as span: + media = tool_media.ToolMedia(manager, span, "test") + assert not media.usable + assert tool_media.tool_capture_policy().requested is False + finally: + otel_context.detach(token) + manager.shutdown(5) + + +def test_value_is_read_only_after_admission_and_only_once(f01, spans): + manager = f01.manager() + provider, _ = spans + reads = [] + + def read(): + reads.append(1) + return b64(png(seed=1)) + + with provider.get_tracer("t").start_as_current_span("tool") as span: + denied = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_outputs=True)) + assert denied.base64("a", read, "image/png", purpose="tool_result", expect="image") is None + assert reads == [] + media = tool_media.ToolMedia(manager, span, "test") + first = media.base64("b", read, "image/png", purpose="tool_result", expect="image") + again = tool_media.ToolMedia(manager, span, "test").base64("b", read, "image/png", purpose="tool_result") + assert first is not None and again == first and reads == [1] + assert media.base64("c", read, "audio/wav", purpose="tool_result", expect="image") is None + assert media.base64("d", read, "image/gif", purpose="tool_result") is None + assert reads == [1] + flush(manager) + assert len(f01.uploads) == 1 and len(refs(span)) == 1 + manager.shutdown(5) + + +def test_budget_and_part_cap(f01, spans): + manager = f01.manager() + provider, _ = spans + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", budget=200, max_parts=2) + image = png(width=4, height=4, seed=2) + assert len(image) < 200 + assert media.base64("1", lambda: b64(image), "image/png", purpose="tool_result") is not None + assert media.remaining == 200 - len(image) + assert media.data_uri("2", lambda: data_uri("image/png", png(width=40, height=40, seed=3)), ("image/png",), + purpose="tool_result") is None # over the remaining budget + assert media.base64("3", lambda: b64(png(seed=4)), "image/png", purpose="tool_result") is None # part cap + flush(manager) + assert len(f01.uploads) == 1 + manager.shutdown(5) + + +def test_disabled_and_missing_managers_are_inert(spans): + provider, _ = spans + with provider.get_tracer("t").start_as_current_span("tool") as span: + for manager in (None, create_attachment_manager(config=AttachmentConfig())): + media = tool_media.ToolMedia(manager, span, "test") + assert not media.usable + assert media.base64("x", lambda: pytest.fail("read"), "image/png", purpose="tool_result") is None + + +def test_pil_images_are_encoded_only_when_in_memory(f01, spans, tmp_path): + Image = pytest.importorskip("PIL.Image") + manager = f01.manager() + provider, _ = spans + in_memory = Image.new("RGB", (4, 3), (1, 2, 3)) + path = tmp_path / "x.png" + in_memory.save(path) + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test") + assert media.image("file", Image.open(path), purpose="tool_result") is None + assert media.image("mem", in_memory, purpose="tool_result") is not None + assert media.image("huge", Image.new("L", (4096, 4096)), purpose="tool_result") is None + assert media.image("not-an-image", b"\x89PNG", purpose="tool_result") is None + flush(manager) + assert [u["mime_type"] for u in f01.uploads] == ["image/png"] + manager.shutdown(5) + + +def test_signature_of_another_modality_is_denied_and_never_relabelled(f01, spans): + """A PNG declared as audio, a generic file or a PDF is not uploaded (hide_input_images cannot be escaped).""" + from _harness import PDF, wav + + manager = f01.manager() + provider, _ = spans + shot = png(seed=21) + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_input_images=True)) + assert media.base64("a", lambda: b64(shot), "audio/wav", purpose="tool_result", expect="audio") is None + assert media.base64("b", lambda: b64(shot), "application/octet-stream", purpose="tool_result") is None + assert media.base64("c", lambda: b64(shot), "application/pdf", purpose="tool_result") is None + assert media.data_uri("d", lambda: "data:audio/wav;base64," + b64(shot), ("audio/wav",), + purpose="tool_result") is None + # Matching signatures and signature-less bytes still upload. + assert media.base64("e", lambda: b64(wav()), "audio/wav", purpose="tool_result", expect="audio") + assert media.base64("f", lambda: b64(PDF), "application/pdf", purpose="tool_result") + flush(manager) + assert shot not in f01.payloads + assert sorted(u["mime_type"] for u in f01.uploads) == ["application/pdf", "audio/wav"] + manager.shutdown(5) + + +@pytest.mark.parametrize("head,kind", [ + (b"\x89PNG\r\n\x1a\n\x00\x00\x00\x00", "image"), (b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01", "image"), + (b"RIFF\x00\x00\x00\x00WEBP", "image"), (b"RIFF\x00\x00\x00\x00WAVE", "audio"), (b"OggS\x00\x02", "audio"), + (b"fLaC\x00\x00", "audio"), (b"ID3\x04\x00", "audio"), (b"%PDF-1.7\n", "document"), + (b"\x00\x00\x00\x18ftypmp42", None), (b"\x1aE\xdf\xa3\x9fB\x86", None), (b"RIFF\x00\x00\x00\x00AVI ", None), + (b"plain text", None), (b"BM6\x00\x00\x00\x00\x00\x00\x006\x00", "image"), (b"II*\x00\x08\x00\x00\x00", "image"), + (b"MM\x00*\x00\x00\x00\x08", "image"), (b"\x00\x00\x00\x18ftypavif", "image"), + (b"\x00\x00\x00\x18ftypheic", "image"), (b"\x00\x00\x00\x18ftypheix", "image"), + (b"\x00\x00\x00\x18ftypmif1", "image"), (b"\x00\x00\x00\x18ftypmsf1", "image"), + (b"\x00\x00\x00\x18ftypisom", None), (b"\x00\x00\x00\x18ftypM4A ", None), (b"\x00\x00\x00\x14ftypqt ", None), + (b"\x00\x00\x00\x18avifftyp", None)]) +def test_signature_kind(head, kind): + assert tool_media.signature_kind(head) == kind + + +def _bmp(): + pixels = b"\x00\x00\xff\x00" * 4 + header = b"BM" + struct.pack("I", 24) + b"ftyp" + brand + b"\x00\x00\x00\x00" + brand + b"mif1" + b"\x00" * 32 + + +_IMAGE_MISLABELS = [ + ("bmp-as-audio/wav", "audio/wav", _bmp), ("bmp-as-audio/L16", "audio/L16", _bmp), + ("bmp-as-octet-stream", "application/octet-stream", _bmp), ("tiff-le-as-audio/L16", "audio/L16", _tiff), + ("tiff-be-as-audio/wav", "audio/wav", lambda: _tiff(b"MM\x00*")), + ("avif-as-video/mp4", "video/mp4", lambda: _heif(b"avif")), ("heic-as-audio/mp4", "audio/mp4", lambda: _heif(b"heic")), + ("heix-as-octet-stream", "application/octet-stream", lambda: _heif(b"heix")), + ("mif1-as-video/mp4", "video/mp4", lambda: _heif(b"mif1")), ("msf1-as-video/mp4", "video/mp4", lambda: _heif(b"msf1"))] + + +@pytest.mark.parametrize("label,mime,make", _IMAGE_MISLABELS, ids=[case[0] for case in _IMAGE_MISLABELS]) +def test_bmp_tiff_and_heif_images_cannot_escape_hide_input_images(f01, spans, label, mime, make): + """Image bytes declared as audio, video or a generic file are not uploaded under hide_input_images.""" + manager = f01.manager() + provider, _ = spans + data = make() + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test", trace_config=TraceConfig(hide_input_images=True)) + assert media.base64("a", lambda: b64(data), mime, purpose="tool_result") is None + assert media.data_uri("b", lambda: "data:%s;base64,%s" % (mime, b64(data)), (mime,), + purpose="tool_result") is None + flush(manager) + assert f01.uploads == [] and data not in f01.payloads + manager.shutdown(5) + + +def test_octet_stream_with_a_recognised_signature_is_skipped(f01, spans): + """application/octet-stream is the file modality: image, audio or PDF bytes in it are skipped, never relabelled.""" + from _harness import PDF, wav + + manager = f01.manager() + provider, _ = spans + plain = b"opaque tool output, no known signature" * 4 + with provider.get_tracer("t").start_as_current_span("tool") as span: + media = tool_media.ToolMedia(manager, span, "test") + for position, data in (("png", png(seed=31)), ("wav", wav()), ("pdf", PDF), ("bmp", _bmp())): + assert media.base64(position, lambda d=data: b64(d), "application/octet-stream", + purpose="tool_result") is None, position + assert media.base64("plain", lambda: b64(plain), "application/octet-stream", purpose="tool_result") + flush(manager) + assert [(u["mime_type"], p) for u, p in zip(f01.uploads, f01.payloads)] == [("application/octet-stream", plain)] + manager.shutdown(5)